prepare 1.79 (#3281)

* update changelog for 1.79.0

* bump version to 1.79.0

.github/dependabot.yml

@@ -3,7 +3,7 @@ updates:
   - package-ecosystem: "cargo"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "monthly"
     commit-message:
       prefix: "cargo"
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 50

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples
+          args: --workspace --tests --examples --benches
 
   docs:
     name: Rust doc comments
@@ -77,19 +77,19 @@ jobs:
         include:
           # Currently used Rust version, same as in `rust-toolchain` file.
           - os: ubuntu-latest
-            rust: 1.54.0
+            rust: 1.60.0
             python: 3.9
           - os: windows-latest
-            rust: 1.54.0
+            rust: 1.60.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.51.0
+          # Minimum Supported Rust Version = 1.56.0
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.51.0
+            rust: 1.56.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/dependabot.yml

@@ -0,0 +1,21 @@
+name: Dependabot auto-approve
+on: pull_request
+
+permissions:
+  pull-requests: write
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v1.1.1
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Approve a PR
+        run: gh pr review --approve "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/upload-docs.yml

@@ -0,0 +1,28 @@
+name: Build & Deploy Documentation on rs.delta.chat
+
+on:
+  push:
+    branches:
+      - master
+      - docs-gh-action
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions-rs/toolchain@v1
+    - name: Build the documentation with cargo
+      run: |
+        cargo doc --package deltachat --no-deps
+    - name: Upload to rs.delta.chat
+      uses: up9cloud/action-rsync@v1.3
+      env:
+        USER: ${{ secrets.USERNAME }}
+        KEY: ${{ secrets.KEY }}
+        HOST: "delta.chat"
+        SOURCE: "target/doc"
+        TARGET: "/var/www/html/rs/"
+

.github/workflows/upload-ffi-docs.yml

@@ -0,0 +1,28 @@
+name: Build & Deploy Documentation on cffi.delta.chat
+
+on:
+  push:
+    branches:
+      - master
+      - docs-gh-action
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions-rs/toolchain@v1
+    - name: Build the documentation with cargo
+      run: |
+        cargo doc --package deltachat_ffi --no-deps
+    - name: Upload to cffi.delta.chat
+      uses: up9cloud/action-rsync@v1.3
+      env:
+        USER: ${{ secrets.USERNAME }}
+        KEY: ${{ secrets.KEY }}
+        HOST: "delta.chat"
+        SOURCE: "target/doc"
+        TARGET: "/var/www/html/cffi/"
+

.gitignore

@@ -29,3 +29,7 @@ deltachat-ffi/xml
 
 coverage/
 .DS_Store
+.vscode/launch.json
+python/accounts.txt
+python/all-testaccounts.txt
+tmp/

CHANGELOG.md

@@ -1,21 +1,146 @@
 # Changelog
 
-## Unreleased
+## 1.79.0
+
+### Changes
+- Send locations in the background regardless of SMTP loop activity #3247
+- refactorings #3268
+- improve tests and ci #3266 #3271
+
+### Fixes
+- simplify `dc_stop_io()` and remove potential panics and race conditions #3273
+- fix correct message escaping consisting of a dot in SMTP protocol #3265
+
+
+## 1.78.0
+
+### API-Changes
+- replaced stock string `DC_STR_ONE_MOMENT` by `DC_STR_NOT_CONNECTED` #3222
+- add `dc_resend_msgs()` #3238
+- `dc_provider_new_from_email()` does no longer do an DNS lookup for checking custom domains,
+  this is done by `dc_provider_new_from_email_with_dns()` now #3256
+
+### Changes
+- introduce multiple self addresses with the "configured" address always being the primary one #2896
+- Further improve finding the correct server after logging in #3208
+- `get_connectivity_html()` returns HTML as non-scalable #3213
+- add update-serial to `DC_EVENT_WEBXDC_STATUS_UPDATE` #3215
+- Speed up message receiving via IMAP a bit #3225
+- mark messages as seen on IMAP in batches #3223
+- remove Received: based draft detection heuristic #3230
+- Use pkgconfig for building Python package #2590
+- don't start io on unconfigured context #2664
+- do not assign group IDs to ad-hoc groups #2798
+- dynamic libraries use dylib extension on Darwin #3226
+- refactorings #3217 #3219 #3224 #3235 #3239 #3244 #3254
+- improve documentation #3214 #3220 #3237
+- improve tests and ci #3212 #3233 #3241 #3242 #3252 #3250 #3255 #3260
+
+### Fixes
+- Take `delete_device_after` into account when calculating ephemeral loop timeout #3211 #3221
+- Fix a bug where a blocked contact could send a contact request #3218
+- Make sure, videochat-room-names are always URL-safe #3231
+- Try removing account folder multiple times in case of failure #3229
+- Ignore messages from all spam folders if there are many #3246
+- Hide location-only messages instead of displaying empty bubbles #3248
+
+
+## 1.77.0
+
+### API changes
+- change semantics of `dc_get_webxdc_status_updates()` second parameter
+  and remove update-id from `DC_EVENT_WEBXDC_STATUS_UPDATE` #3081
+
+### Changes
+- add more SMTP logging #3093
+- place common headers like `From:` before the large `Autocrypt:` header #3079
+- keep track of securejoin joiner status in database to survive restarts #2920
+- remove never used `SentboxMove` option #3111
+- improve speed by caching config values #3131 #3145
+- optimize `markseen_msgs` #3141
+- automatically accept chats with outgoing messages #3143
+- `dc_receive_imf` refactorings #3154 #3156 #3159
+- add index to speedup deletion of expired ephemeral messages #3155
+- muted chats stay archived on new messages #3184
+- support `min_api` from Webxdc manifests #3206
+- do not read whole webxdc file into memory #3109
+- improve tests, refactorings #3073 #3096 #3102 #3108 #3139 #3128 #3133 #3142 #3153 #3151 #3174 #3170 #3148 #3179 #3185
+- improve documentation #2983 #3112 #3103 #3118 #3120
+
+### Fixes
+- speed up loading of chat messages by a factor of 20 #3171 #3194 #3173
+- fix an issue where the app crashes when trying to export a backup #3195
+- hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
+- do not delete messages without Message-IDs as duplicates #3095
+- assign replies from a different email address to the correct chat #3119
+- assing outgoing private replies to the correct chat #3177
+- start ephemeral timer when seen status is synchronized via IMAP #3122
+- do not create empty contact requests with "setup changed" messages;
+  instead, send a "setup changed" message into all chats we share with the peer #3187
+- do not delete duplicate messages on IMAP immediately to accidentally deleting
+  the last copy #3138
+- clear more columns when message expires due to `delete_device_after` setting #3181
+- do not try to use stale SMTP connections #3180
+- slightly improve finding the correct server after logging in #3207
+- retry message sending automatically if loop is not interrupted #3183
+- fix a bug where sometimes the file extension of a long filename containing a dot was cropped #3098
+
+
+## 1.76.0
+
+### Changes
+- move messages in batches #3058
+- delete messages in batches #3060
+- python: remove arbitrary timeouts from tests #3059
+- refactorings #3026
+
+### Fixes
+- avoid archived, fresh chats #3053
+- Also resync UIDs in folders that are not configured #2289
+- treat "NO" IMAP response to MOVE and COPY commands as an error #3058
+- Fix a bug where messages in the Spam folder created contact requests #3015
+- Fix a bug where drafts disappeared after some days #3067
+- Parse MS Exchange read receipts and mark the original message as read #3075
+- do not retry message sending infinitely in case of permanent SMTP failure #3070
+- set message state to failed when retry limit is exceeded #3072
+
+
+## 1.75.0
+
+### Changes
+- optimize `delete_expired_imap_messages()` #3047
+
+
+## 1.74.0
+
+### Fixes
+- avoid reconnection loop when message without Message-ID is marked as seen #3044
+
+
+## 1.73.0
+
+### API changes
+- added `only_fetch_mvbox` config #3028
 
 ### Changes
 - don't watch Sent folder by default #3025
 - use webxdc app name in chatlist/quotes/replies etc. #3027
-- refactorings #3023
-- remove direct dependency on `byteorder` crate #3031
 - make it possible to cancel message sending by removing the message #3034,
   this was previosuly removed in 1.71.0 #2939
-- always skip Seen flag synchronization when there are no updates #3039
+- synchronize Seen flags only on watched folders to speed up
+  folder scanning #3041
+- remove direct dependency on `byteorder` crate #3031
+- refactorings #3023 #3013
+- update provider database #3043
+- improve documentation #3017 #3018 #3021
 
 ### Fixes
 - fix splitting off text from webxdc messages #3032
 - call slow `delete_expired_imap_messages()` less often #3037
 - make synchronization of Seen status more robust in case unsolicited FETCH
   result without UID is returned #3022
+- fetch Inbox before scanning folders to ensure iOS does
+  not kill the app before it gets to fetch the Inbox in background #3040
 
 
 ## 1.72.0

CMakeLists.txt

@@ -4,10 +4,18 @@ include(GNUInstallDirs)
 
 find_program(CARGO cargo)
 
+if(APPLE)
+       set(DYNAMIC_EXT "dylib")
+elseif(UNIX)
+       set(DYNAMIC_EXT "so")
+else()
+       set(DYNAMIC_EXT "dll")
+endif()
+
 add_custom_command(
 	OUTPUT
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
+	"target/release/libdeltachat.${DYNAMIC_EXT}"
 	"target/release/pkgconfig/deltachat.pc"
         COMMAND
         PREFIX=${CMAKE_INSTALL_PREFIX}
@@ -32,11 +40,11 @@ add_custom_target(
 	ALL
 	DEPENDS
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
+	"target/release/libdeltachat.${DYNAMIC_EXT}"
 	"target/release/pkgconfig/deltachat.pc"
 )
 
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "deltachat"
-version = "1.72.0"
+version = "1.79.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
-edition = "2018"
+edition = "2021"
 license = "MPL-2.0"
-resolver = "2"
+rust-version = "1.56"
 
 [profile.dev]
 debug = 0
@@ -12,9 +12,6 @@ debug = 0
 [profile.release]
 lto = true
 
-[patch.crates-io]
-rusqlite = { git = "https://github.com/rusqlite/rusqlite", branch="master" }
-
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
@@ -22,9 +19,9 @@ ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
 async-imap = { git = "https://github.com/async-email/async-imap" }
 async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
-async-std-resolver = "0.20"
-async-std = { version = "1", features = ["unstable"] }
+async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", default-features=false, features = ["smtp-transport", "socks5"] }
+async-std-resolver = "0.21"
+async-std = { version = "1" }
 async-tar = { version = "0.4", default-features=false }
 async-trait = "0.1"
 backtrace = "0.3"
@@ -37,27 +34,26 @@ encoded-words = { git = "https://github.com/async-email/encoded-words", branch="
 escaper = "0.1"
 futures = "0.3"
 hex = "0.4.0"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-imap-proto = "0.14.3"
+image = { version = "0.24.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
-log = {version = "0.4.8", optional = true }
+log = {version = "0.4.16", optional = true }
 mailparse = "0.13"
 native-tls = "0.2"
 num_cpus = "1.13"
 num-derive = "0.3"
 num-traits = "0.2"
-once_cell = "1.9.0"
+once_cell = "1.10.0"
 percent-encoding = "2.0"
 pgp = { version = "0.7", default-features = false }
 pretty_env_logger = { version = "0.4", optional = true }
 quick-xml = "0.22"
 r2d2 = "0.8"
-r2d2_sqlite = "0.19"
+r2d2_sqlite = "0.20"
 rand = "0.7"
 regex = "1.5"
-rusqlite = { version = "0.26", features = ["sqlcipher"] }
+rusqlite = { version = "0.27", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 rustyline = { version = "9", optional = true }
 sanitize-filename = "0.3"
@@ -66,9 +62,8 @@ serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
 smallvec = "1"
-stop-token = "0.7"
-strum = "0.23"
-strum_macros = "0.23"
+strum = "0.24"
+strum_macros = "0.24"
 surf = { version = "2.3", default-features = false, features = ["h1-client"] }
 thiserror = "1"
 toml = "0.5"
@@ -77,14 +72,14 @@ uuid = { version = "0.8", features = ["serde", "v4"] }
 fast-socks5 = "0.4"
 humansize = "1"
 qrcodegen = "1.7.0"
-tagger = "4.2.1"
-textwrap = "0.14.2"
-zip = { version = "0.5.13", default-features = false, features = ["deflate"] }
+tagger = "4.3.3"
+textwrap = "0.15.0"
+zip = { version = "0.6.2", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
 async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = "0.3"
+criterion = { version = "0.3.4", features = ["async_std"] }
 futures-lite = "1.12"
 log = "0.4"
 pretty_env_logger = "0.4"
@@ -120,6 +115,18 @@ harness = false
 name = "search_msgs"
 harness = false
 
+[[bench]]
+name = "receive_emails"
+harness = false
+
+[[bench]]
+name = "get_chat_msgs"
+harness = false
+
+[[bench]]
+name = "get_chatlist"
+harness = false
+
 [features]
 default = ["vendored"]
 internals = []

README.md

@@ -12,6 +12,8 @@ To download and install the official compiler for the Rust programming language,
 $ curl https://sh.rustup.rs -sSf | sh
 ```
 
+> On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
+
 ## Using the CLI client
 
 Compile and run Delta Chat Core command line utility, using `cargo`:

benches/get_chat_msgs.rs

@@ -0,0 +1,40 @@
+use async_std::path::Path;
+
+use criterion::async_executor::AsyncStdExecutor;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+use deltachat::chat::{self, ChatId};
+use deltachat::chatlist::Chatlist;
+use deltachat::context::Context;
+
+async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
+    let id = 100;
+    let context = Context::new(dbfile.into(), id).await.unwrap();
+
+    for c in chats.iter().take(10) {
+        black_box(chat::get_chat_msgs(&context, *c, 0, None).await.ok());
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let chats: Vec<_> = async_std::task::block_on(async {
+            let context = Context::new((&path).into(), 100).await.unwrap();
+            let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
+            let len = chatlist.len();
+            (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
+        });
+
+        c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
+            b.to_async(AsyncStdExecutor)
+                .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
+        });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/get_chatlist.rs

@@ -0,0 +1,27 @@
+use criterion::async_executor::AsyncStdExecutor;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+use deltachat::chatlist::Chatlist;
+use deltachat::context::Context;
+
+async fn get_chat_list_benchmark(context: &Context) {
+    Chatlist::try_load(context, 0, None, None).await.unwrap();
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let context =
+            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
+        c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
+            b.to_async(AsyncStdExecutor)
+                .iter(|| get_chat_list_benchmark(black_box(&context)))
+        });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/receive_emails.rs

@@ -0,0 +1,84 @@
+use async_std::{path::PathBuf, task::block_on};
+use criterion::{
+    async_executor::AsyncStdExecutor, black_box, criterion_group, criterion_main, BatchSize,
+    Criterion,
+};
+use deltachat::{
+    config::Config,
+    context::Context,
+    dc_receive_imf::dc_receive_imf,
+    imex::{imex, ImexMode},
+};
+use tempfile::tempdir;
+
+async fn recv_all_emails(context: Context) -> Context {
+    for i in 0..100 {
+        let imf_raw = format!(
+            "Subject: Benchmark
+Message-ID: Mr.OssSYnOFkhR.{i}@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+To: alice@example.com
+From: sender@testrun.org
+Chat-Version: 1.0
+Chat-Disposition-Notification-To: sender@testrun.org
+Chat-User-Avatar: 0
+In-Reply-To: Mr.OssSYnOFkhR.{i_dec}@testrun.org
+MIME-Version: 1.0
+
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+Hello {i}",
+            i = i,
+            i_dec = i - 1,
+        );
+        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), false)
+            .await
+            .unwrap();
+    }
+    context
+}
+
+async fn create_context() -> Context {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let id = 100;
+    let context = Context::new(dbfile.into(), id).await.unwrap();
+
+    let backup: PathBuf = std::env::current_dir()
+        .unwrap()
+        .join("delta-chat-backup.tar")
+        .into();
+    if backup.exists().await {
+        println!("Importing backup");
+        imex(&context, ImexMode::ImportBackup, &backup, None)
+            .await
+            .unwrap();
+    }
+
+    let addr = "alice@example.com";
+    context.set_config(Config::Addr, Some(addr)).await.unwrap();
+    context
+        .set_config(Config::ConfiguredAddr, Some(addr))
+        .await
+        .unwrap();
+    context
+        .set_config(Config::Configured, Some("1"))
+        .await
+        .unwrap();
+    context
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Receive messages");
+    group.bench_function("Receive 100 simple text msgs", |b| {
+        b.to_async(AsyncStdExecutor).iter_batched(
+            || block_on(create_context()),
+            |context| recv_all_emails(black_box(context)),
+            BatchSize::LargeInput,
+        );
+    });
+    group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/search_msgs.rs

@@ -20,6 +20,8 @@ fn criterion_benchmark(c: &mut Criterion) {
         c.bench_function("search hello", |b| {
             b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
         });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
     }
 }
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.72.0"
+version = "1.79.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/src/lib.rs

@@ -31,13 +31,13 @@ use rand::Rng;
 
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
-use deltachat::contact::{Contact, Origin};
+use deltachat::contact::{Contact, ContactId, Origin};
 use deltachat::context::Context;
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
 use deltachat::stock_str::StockMessage;
-use deltachat::webxdc::StatusUpdateId;
+use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
 
@@ -493,14 +493,16 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ChatEphemeralTimerModified { chat_id, .. } => chat_id.to_u32() as libc::c_int,
         EventType::ContactsChanged(id) | EventType::LocationChanged(id) => {
             let id = id.unwrap_or_default();
-            id as libc::c_int
+            id.to_u32() as libc::c_int
         }
         EventType::ConfigureProgress { progress, .. } | EventType::ImexProgress(progress) => {
             *progress as libc::c_int
         }
         EventType::ImexFileWritten(_) => 0,
         EventType::SecurejoinInviterProgress { contact_id, .. }
-        | EventType::SecurejoinJoinerProgress { contact_id, .. } => *contact_id as libc::c_int,
+        | EventType::SecurejoinJoinerProgress { contact_id, .. } => {
+            contact_id.to_u32() as libc::c_int
+        }
         EventType::WebxdcStatusUpdate { msg_id, .. } => msg_id.to_u32() as libc::c_int,
     }
 }
@@ -533,8 +535,8 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ImexFileWritten(_)
         | EventType::MsgsNoticed(_)
         | EventType::ConnectivityChanged
-        | EventType::SelfavatarChanged
-        | EventType::ChatModified(_) => 0,
+        | EventType::SelfavatarChanged => 0,
+        EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
         | EventType::MsgDelivered { msg_id, .. }
@@ -544,8 +546,9 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
         EventType::ChatEphemeralTimerModified { timer, .. } => timer.to_u32() as libc::c_int,
         EventType::WebxdcStatusUpdate {
-            status_update_id, ..
-        } => status_update_id.to_u32() as libc::c_int,
+            status_update_serial,
+            ..
+        } => status_update_serial.to_u32() as libc::c_int,
     }
 }
 
@@ -717,7 +720,11 @@ pub unsafe extern "C" fn dc_get_chatlist(
     let ctx = &*context;
     let qs = to_opt_string_lossy(query_str);
 
-    let qi = if query_id == 0 { None } else { Some(query_id) };
+    let qi = if query_id == 0 {
+        None
+    } else {
+        Some(ContactId::new(query_id))
+    };
 
     block_on(async move {
         match chatlist::Chatlist::try_load(ctx, flags as usize, qs.as_deref(), qi)
@@ -745,7 +752,7 @@ pub unsafe extern "C" fn dc_create_chat_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        ChatId::create_for_contact(ctx, contact_id)
+        ChatId::create_for_contact(ctx, ContactId::new(contact_id))
             .await
             .log_err(ctx, "Failed to create chat from contact_id")
             .map(|id| id.to_u32())
@@ -765,7 +772,7 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        ChatId::lookup_by_contact(ctx, contact_id)
+        ChatId::lookup_by_contact(ctx, ContactId::new(contact_id))
             .await
             .log_err(ctx, "Failed to get chat for contact_id")
             .unwrap_or_default() // unwraps the Result
@@ -903,7 +910,7 @@ pub unsafe extern "C" fn dc_send_webxdc_status_update(
 pub unsafe extern "C" fn dc_get_webxdc_status_updates(
     context: *mut dc_context_t,
     msg_id: u32,
-    status_update_id: u32,
+    last_known_serial: u32,
 ) -> *mut libc::c_char {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_get_webxdc_status_updates()");
@@ -913,11 +920,7 @@ pub unsafe extern "C" fn dc_get_webxdc_status_updates(
 
     block_on(ctx.get_webxdc_status_updates(
         MsgId::new(msg_id),
-        if status_update_id == 0 {
-            None
-        } else {
-            Some(StatusUpdateId::new(status_update_id))
-        },
+        StatusUpdateSerial::new(last_known_serial),
     ))
     .unwrap_or_else(|_| "".to_string())
     .strdup()
@@ -1346,7 +1349,10 @@ pub unsafe extern "C" fn dc_get_chat_contacts(
         let arr = dc_array_t::from(
             chat::get_chat_contacts(ctx, ChatId::new(chat_id))
                 .await
-                .unwrap_or_log_default(ctx, "Failed get_chat_contacts"),
+                .unwrap_or_log_default(ctx, "Failed get_chat_contacts")
+                .iter()
+                .map(|id| id.to_u32())
+                .collect::<Vec<u32>>(),
         );
         Box::into_raw(Box::new(arr))
     })
@@ -1456,7 +1462,7 @@ pub unsafe extern "C" fn dc_is_contact_in_chat(
     block_on(chat::is_contact_in_chat(
         ctx,
         ChatId::new(chat_id),
-        contact_id,
+        ContactId::new(contact_id),
     ))
     .log_err(ctx, "is_contact_in_chat failed")
     .unwrap_or_default() as libc::c_int
@@ -1477,7 +1483,7 @@ pub unsafe extern "C" fn dc_add_contact_to_chat(
     block_on(chat::add_contact_to_chat(
         ctx,
         ChatId::new(chat_id),
-        contact_id,
+        ContactId::new(contact_id),
     ))
     .log_err(ctx, "Failed to add contact")
     .is_ok() as libc::c_int
@@ -1498,7 +1504,7 @@ pub unsafe extern "C" fn dc_remove_contact_from_chat(
     block_on(chat::remove_contact_from_chat(
         ctx,
         ChatId::new(chat_id),
-        contact_id,
+        ContactId::new(contact_id),
     ))
     .log_err(ctx, "Failed to remove contact")
     .is_ok() as libc::c_int
@@ -1745,6 +1751,27 @@ pub unsafe extern "C" fn dc_forward_msgs(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_resend_msgs(
+    context: *mut dc_context_t,
+    msg_ids: *const u32,
+    msg_cnt: libc::c_int,
+) -> libc::c_int {
+    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
+        eprintln!("ignoring careless call to dc_resend_msgs()");
+        return 0;
+    }
+    let ctx = &*context;
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
+
+    if let Err(err) = block_on(chat::resend_msgs(ctx, &msg_ids)) {
+        error!(ctx, "Resending failed: {}", err);
+        0
+    } else {
+        1
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_markseen_msgs(
     context: *mut dc_context_t,
@@ -1833,7 +1860,8 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
         Contact::lookup_id_by_addr(ctx, &to_string_lossy(addr), Origin::IncomingReplyTo)
             .await
             .unwrap_or_log_default(ctx, "failed to lookup id")
-            .unwrap_or(0)
+            .map(|id| id.to_u32())
+            .unwrap_or_default()
     })
 }
 
@@ -1853,6 +1881,7 @@ pub unsafe extern "C" fn dc_create_contact(
     block_on(async move {
         Contact::create(ctx, &name, &to_string_lossy(addr))
             .await
+            .map(|id| id.to_u32())
             .unwrap_or(0)
     })
 }
@@ -1891,7 +1920,9 @@ pub unsafe extern "C" fn dc_get_contacts(
 
     block_on(async move {
         match Contact::get_all(ctx, flags, query).await {
-            Ok(contacts) => Box::into_raw(Box::new(dc_array_t::from(contacts))),
+            Ok(contacts) => Box::into_raw(Box::new(dc_array_t::from(
+                contacts.iter().map(|id| id.to_u32()).collect::<Vec<u32>>(),
+            ))),
             Err(_) => ptr::null_mut(),
         }
     })
@@ -1928,7 +1959,10 @@ pub unsafe extern "C" fn dc_get_blocked_contacts(
             Contact::get_all_blocked(ctx)
                 .await
                 .log_err(ctx, "Can't get blocked contacts")
-                .unwrap_or_default(),
+                .unwrap_or_default()
+                .iter()
+                .map(|id| id.to_u32())
+                .collect::<Vec<u32>>(),
         )))
     })
 }
@@ -1939,7 +1973,8 @@ pub unsafe extern "C" fn dc_block_contact(
     contact_id: u32,
     block: libc::c_int,
 ) {
-    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL as u32 {
+    let contact_id = ContactId::new(contact_id);
+    if context.is_null() || contact_id.is_special() {
         eprintln!("ignoring careless call to dc_block_contact()");
         return;
     }
@@ -1969,7 +2004,7 @@ pub unsafe extern "C" fn dc_get_contact_encrinfo(
     let ctx = &*context;
 
     block_on(async move {
-        Contact::get_encrinfo(ctx, contact_id)
+        Contact::get_encrinfo(ctx, ContactId::new(contact_id))
             .await
             .map(|s| s.strdup())
             .unwrap_or_else(|e| {
@@ -1984,7 +2019,8 @@ pub unsafe extern "C" fn dc_delete_contact(
     context: *mut dc_context_t,
     contact_id: u32,
 ) -> libc::c_int {
-    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL as u32 {
+    let contact_id = ContactId::new(contact_id);
+    if context.is_null() || contact_id.is_special() {
         eprintln!("ignoring careless call to dc_delete_contact()");
         return 0;
     }
@@ -2010,7 +2046,7 @@ pub unsafe extern "C" fn dc_get_contact(
     let ctx = &*context;
 
     block_on(async move {
-        Contact::get_by_id(ctx, contact_id)
+        Contact::get_by_id(ctx, ContactId::new(contact_id))
             .await
             .map(|contact| Box::into_raw(Box::new(ContactWrapper { context, contact })))
             .unwrap_or_else(|_| ptr::null_mut())
@@ -2433,7 +2469,7 @@ pub unsafe extern "C" fn dc_array_get_contact_id(
         return 0;
     }
 
-    (*array).get_location(index).contact_id
+    (*array).get_location(index).contact_id.to_u32()
 }
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_msg_id(
@@ -2949,7 +2985,7 @@ pub unsafe extern "C" fn dc_msg_get_from_id(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_from_id()
+    ffi_msg.message.get_from_id().to_u32()
 }
 
 #[no_mangle]
@@ -3660,7 +3696,7 @@ pub unsafe extern "C" fn dc_contact_get_id(contact: *mut dc_contact_t) -> u32 {
         return 0;
     }
     let ffi_contact = &*contact;
-    ffi_contact.contact.get_id()
+    ffi_contact.contact.get_id().to_u32()
 }
 
 #[no_mangle]
@@ -3937,6 +3973,25 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
     }
     let addr = to_string_lossy(addr);
 
+    let ctx = &*context;
+
+    match block_on(provider::get_provider_info(ctx, addr.as_str(), true)) {
+        Some(provider) => provider,
+        None => ptr::null_mut(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_new_from_email_with_dns(
+    context: *const dc_context_t,
+    addr: *const libc::c_char,
+) -> *const dc_provider_t {
+    if context.is_null() || addr.is_null() {
+        eprintln!("ignoring careless call to dc_provider_new_from_email_with_dns()");
+        return ptr::null();
+    }
+    let addr = to_string_lossy(addr);
+
     let ctx = &*context;
     let socks5_enabled = block_on(async move {
         ctx.get_config_bool(config::Config::Socks5Enabled)

deltachat-ffi/src/lot.rs

@@ -111,19 +111,19 @@ impl Lot {
         match self {
             Self::Summary(_) => Default::default(),
             Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { contact_id, .. } => *contact_id,
+                Qr::AskVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::AskVerifyGroup { .. } => Default::default(),
-                Qr::FprOk { contact_id } => *contact_id,
-                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default(),
+                Qr::FprOk { contact_id } => contact_id.to_u32(),
+                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => *contact_id,
+                Qr::Addr { contact_id } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
-                Qr::WithdrawVerifyContact { contact_id, .. } => *contact_id,
+                Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::WithdrawVerifyGroup { .. } => Default::default(),
-                Qr::ReviveVerifyContact { contact_id, .. } => *contact_id,
+                Qr::ReviveVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::ReviveVerifyGroup { .. } => Default::default(),
             },
             Self::Error(_) => Default::default(),

draft/webxdc-dev-reference.md

@@ -29,7 +29,7 @@ window.webxdc.sendUpdate(update, descr);
 Webxdc apps are usually shared in a chat and run independently on each peer.
 To get a shared state, the peers use `sendUpdate()` to send updates to each other.
 
-- `update`: an object with the following fields:  
+- `update`: an object with the following properties:  
     - `update.payload`: any javascript primitive, array or object.
     - `update.info`: optional, short, informational message that will be added to the chat,
        eg. "Alice voted" or "Bob scored 123 in MyGame";
@@ -45,48 +45,38 @@ All peers, including the sending one,
 will receive the update by the callback given to `setUpdateListener()`.
 
 There are situations where the user cannot send messages to a chat,
-eg. contact requests or if the user has left a group.
+eg. if the webxdc instance comes as a contact request or if the user has left a group.
 In these cases, you can still call `sendUpdate()`,
 however, the update won't be sent to other peers
-and you won't get the update by `setUpdateListener()` nor by `getAllUpdates()`.
+and you won't get the update by `setUpdateListener()`.
 
 
 ### setUpdateListener()
 
 ```js
-window.webxdc.setUpdateListener((update) => {});
+let promise = window.webxdc.setUpdateListener((update) => {}, serial);
 ```
 
 With `setUpdateListener()` you define a callback that receives the updates
-sent by `sendUpdate()`.
+sent by `sendUpdate()`. The callback is called for updates sent by you or other peers.
+The `serial` specifies the last serial that you know about (defaults to 0). 
+The returned promise resolves when the listener has processed all the update messages known at the time when  `setUpdateListener` was called. 
 
-- `update`: passed to the callback on updates with the following fields:  
-  `update.payload`: equals the payload given to `sendUpdate()`
+Each `update` which is passed to the callback comes with the following properties: 
 
-The callback is called for updates sent by you or other peers.
+- `update.payload`: equals the payload given to `sendUpdate()`
 
+- `update.serial`: the serial number of this update.
+  Serials are larger `0` and newer serials have higher numbers.
+  There may be gaps in the serials
+  and it is not guaranteed that the next serial is exactly incremented by one.
 
-### getAllUpdates()
+- `update.max_serial`: the maximum serial currently known.
+  If `max_serial` equals `serial` this update is the last update (until new network messages arrive).
 
-```js
-updates = await window.webxdc.getAllUpdates();
-```
+- `update.info`: optional, short, informational message (see `send_update`) 
 
-In case your Webxdc was just started,
-you may want to reconstruct the state from the last run -
-and also incorporate updates that may have arrived while the app was not running.
-
-- `updates`: All previous updates in an array, 
-  eg. `[{payload: "foo"},{payload: "bar"}]`
-  if `webxdc.sendUpdate({payload: "foo"}); webxdc.sendUpdate({payload: "bar"};` was called on the last run.
-
-The updates are wrapped into a Promise that you can `await` for.
-If you are not in an async function and cannot use `await` therefore,
-you can get the updates with `then()`:
-
-```js
-window.webxdc.getAllUpdates().then(updates => {});
-```
+- `update.summary`: optional, short text, shown beside app icon (see `send_update`)
 
 
 ### selfAddr
@@ -162,9 +152,7 @@ The following example shows an input field and  every input is show on all peers
         document.getElementById('output').innerHTML += update.payload + "<br>";
       }
     
-      window.webxdc.setUpdateListener(receiveUpdate);
-      window.webxdc.getAllUpdates().then(updates => updates.forEach(receiveUpdate));
-
+      window.webxdc.setUpdateListener(receiveUpdate, 0);
     </script>
   </body>
 </html>
@@ -189,6 +177,9 @@ just clone and start adapting things to your need.
 
 - older devices might not have the newest js features in their webview,
   you may want to transpile your code down to an older js version eg. with https://babeljs.io
+- viewport and scaling features are implementation specific,
+  if you want to have an explicit behavior, you can add eg.
+  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
 - there are tons of ideas for enhancements of the API and the file format,
   eg. in the future, we will may define icon- and manifest-files,
-  allow to aggregate the state or add metadata.
+  allow to aggregate the state or add metadata.

examples/repl/cmdline.rs

@@ -17,11 +17,10 @@ use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
-use deltachat::message::{self, Message, MessageState, MsgId};
+use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
-use deltachat::EventType;
 use deltachat::{config, provider};
 use std::fs;
 use std::time::{Duration, SystemTime};
@@ -84,6 +83,7 @@ async fn reset_tables(context: &Context, bits: i32) {
             )
             .await
             .unwrap();
+        context.sql().config_cache().write().await.clear();
         context
             .sql()
             .execute("DELETE FROM leftgrps;", paramsv![])
@@ -92,16 +92,13 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_event(EventType::MsgsChanged {
-        chat_id: ChatId::new(0),
-        msg_id: MsgId::new(0),
-    });
+    context.emit_msgs_changed_without_ids();
 }
 
 async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
     let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", false).await {
+    if let Err(err) = dc_receive_imf(context, &data, false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -163,10 +160,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
+        context.emit_msgs_changed_without_ids();
     }
     true
 }
@@ -209,7 +203,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         contact_id,
         msgtext.unwrap_or_default(),
         if msg.has_html() { "[HAS-HTML]️" } else { "" },
-        if msg.get_from_id() == 1 {
+        if msg.get_from_id() == ContactId::SELF {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -267,9 +261,8 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
     Ok(())
 }
 
-async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
+async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()> {
     for contact_id in contacts {
-        let line;
         let mut line2 = "".to_string();
         let contact = Contact::get_by_id(context, *contact_id).await?;
         let name = contact.get_display_name();
@@ -284,24 +277,20 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
         } else {
             ""
         };
-        line = format!(
+        let line = format!(
             "{}{} <{}>",
             if !name.is_empty() {
-                &name
+                name
             } else {
                 "<name unset>"
             },
             verified_str,
-            if !addr.is_empty() {
-                &addr
-            } else {
-                "addr unset"
-            }
+            if !addr.is_empty() { addr } else { "addr unset" }
         );
-        let peerstate = Peerstate::from_addr(context, &addr)
+        let peerstate = Peerstate::from_addr(context, addr)
             .await
             .expect("peerstate error");
-        if peerstate.is_some() && *contact_id != 1 {
+        if peerstate.is_some() && *contact_id != ContactId::SELF {
             line2 = format!(
                 ", prefer-encrypt={}",
                 peerstate.as_ref().unwrap().prefer_encrypt
@@ -410,6 +399,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
+                 resend <msg-id>\n\
                  markseen <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
@@ -430,7 +420,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  joinqr <qr-content>\n\
                  setqr <qr-content>\n\
                  providerinfo <addr>\n\
-                 event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
                  clear -- clear screen\n\
@@ -719,7 +708,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id: u32 = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
 
             println!("Single#{} created successfully.", chat_id,);
@@ -747,7 +736,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id_0: u32 = arg1.parse()?;
+            let contact_id_0 = ContactId::new(arg1.parse()?);
             chat::add_contact_to_chat(&context, sel_chat.as_ref().unwrap().get_id(), contact_id_0)
                 .await?;
             println!("Contact added to chat.");
@@ -755,7 +744,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id_1: u32 = arg1.parse()?;
+            let contact_id_1 = ContactId::new(arg1.parse()?);
             chat::remove_contact_from_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
@@ -771,7 +760,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             chat::set_chat_name(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                &format!("{} {}", arg1, arg2).trim(),
+                format!("{} {}", arg1, arg2).trim(),
             )
             .await?;
 
@@ -937,13 +926,24 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
+            let query = format!("{} {}", arg1, arg2).trim().to_string();
             let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
             let time_start = std::time::SystemTime::now();
-            let msglist = context.search_msgs(chat, arg1).await?;
+            let msglist = context.search_msgs(chat, &query).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
-            println!("{} messages.", msglist.len());
+            println!(
+                "{}{} messages for {}search of \"{}\"",
+                msglist.len(),
+                if msglist.len() == 1000 { "+" } else { "" },
+                if chat.is_none() {
+                    "global "
+                } else {
+                    "in-chat-"
+                },
+                query,
+            );
             println!("{:?} to create this list", time_needed);
         }
         "draft" => {
@@ -1100,6 +1100,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(&context, &msg_ids, chat_id).await?;
         }
+        "resend" => {
+            ensure!(!arg1.is_empty(), "Arguments <msg-id> expected");
+
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
+            chat::resend_msgs(&context, &msg_ids).await?;
+        }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
@@ -1139,7 +1146,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id: u32 = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
@@ -1174,16 +1181,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "delcontact" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            Contact::delete(&context, arg1.parse()?).await?;
+            Contact::delete(&context, ContactId::new(arg1.parse()?)).await?;
         }
         "block" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             Contact::block(&context, contact_id).await?;
         }
         "unblock" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             Contact::unblock(&context, contact_id).await?;
         }
         "listblocked" => {
@@ -1224,17 +1231,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 }
             }
         }
-        // TODO: implement this again, unclear how to match this through though, without writing a parser.
-        // "event" => {
-        //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
-        //     let event = arg1.parse()?;
-        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
-        //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
-        //     println!(
-        //         "Sending event {:?}({}), received value {}.",
-        //         event, event as usize, r,
-        //     );
-        // }
         "fileinfo" => {
             ensure!(!arg1.is_empty(), "Argument <file> missing.");
 

examples/repl/main.rs

@@ -207,11 +207,12 @@ const CHAT_COMMANDS: [&str; 36] = [
     "accept",
     "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 7] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
+    "resend",
     "markseen",
     "delmsg",
     "download",
@@ -227,13 +228,12 @@ const CONTACT_COMMANDS: [&str; 9] = [
     "unblock",
     "listblocked",
 ];
-const MISC_COMMANDS: [&str; 12] = [
+const MISC_COMMANDS: [&str; 11] = [
     "getqr",
     "getqrsvg",
     "getbadqr",
     "checkqr",
     "joinqr",
-    "event",
     "fileinfo",
     "clear",
     "exit",
@@ -416,7 +416,7 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            let group = arg1.parse::<u32>().ok().map(ChatId::new);
             let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
             if !qr.is_empty() {
                 if arg0 == "getbadqr" && qr.len() > 40 {
@@ -433,7 +433,7 @@ async fn handle_cmd(
         }
         "getqrsvg" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            let group = arg1.parse::<u32>().ok().map(ChatId::new);
             let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
             match get_securejoin_qr_svg(&ctx, group).await {
                 Ok(svg) => {

python/install_python_bindings.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
 
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
         subprocess.check_call([

python/mypy.ini

@@ -17,3 +17,7 @@ ignore_missing_imports = True
 
 [mypy-_pytest.*]
 ignore_missing_imports = True
+
+[mypy-imap_tools.*]
+ignore_missing_imports = True
+

python/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
+requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]

python/setup.py

@@ -11,8 +11,11 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
-        setup_requires=['setuptools_scm'], # required for compatibility with `python3 setup.py sdist`
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
+        setup_requires=[
+            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
+            'pkgconfig',
+        ],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/_build.py

@@ -8,9 +8,9 @@ import shutil
 import subprocess
 import tempfile
 import textwrap
-import types
 
 import cffi
+import pkgconfig  # type: ignore
 
 
 def local_build_flags(projdir, target):
@@ -19,36 +19,31 @@ def local_build_flags(projdir, target):
     :param projdir: The root directory of the deltachat-core-rust project.
     :param target: The rust build target, `debug` or `release`.
     """
-    flags = types.SimpleNamespace()
+    flags = {}
     if platform.system() == 'Darwin':
-        flags.libs = ['resolv', 'dl']
-        flags.extra_link_args = [
+        flags['libraries'] = ['resolv', 'dl']
+        flags['extra_link_args'] = [
                 '-framework', 'CoreFoundation',
                 '-framework', 'CoreServices',
                 '-framework', 'Security',
         ]
     elif platform.system() == 'Linux':
-        flags.libs = ['rt', 'dl', 'm']
-        flags.extra_link_args = []
+        flags['libraries'] = ['rt', 'dl', 'm']
+        flags['extra_link_args'] = []
     else:
         raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
     target_dir = os.environ.get("CARGO_TARGET_DIR")
     if target_dir is None:
         target_dir = os.path.join(projdir, 'target')
-    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags.objs[0]), flags.objs
-    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    flags['extra_objects'] = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags['extra_objects'][0]), flags['extra_objects']
+    flags['include_dirs'] = [os.path.join(projdir, 'deltachat-ffi')]
     return flags
 
 
 def system_build_flags():
     """Construct build flags for building against an installed libdeltachat."""
-    flags = types.SimpleNamespace()
-    flags.libs = ['deltachat']
-    flags.objs = []
-    flags.incs = []
-    flags.extra_link_args = []
-    return flags
+    return pkgconfig.parse('deltachat')
 
 
 def extract_functions(flags):
@@ -69,7 +64,7 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags.incs,
+                      include_dirs=flags['include_dirs'],
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
             return dst_fp.read()
@@ -105,7 +100,7 @@ def find_header(flags):
         try:
             os.chdir(tmpdir)
             cc.compile(sources=["where.c"],
-                       include_dirs=flags.incs,
+                       include_dirs=flags['include_dirs'],
                        macros=[("PY_CFFI_INC", "1")])
         finally:
             os.chdir(cwd)
@@ -183,10 +178,7 @@ def ffibuilder():
                 return DC_EVENT_DATA2_IS_STRING(e);
             }
         """,
-        include_dirs=flags.incs,
-        libraries=flags.libs,
-        extra_objects=flags.objs,
-        extra_link_args=flags.extra_link_args,
+        **flags,
     )
     builder.cdef("""
         typedef int... time_t;

python/src/deltachat/chat.py

@@ -5,7 +5,7 @@ import calendar
 import json
 from datetime import datetime, timezone
 import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
+from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
@@ -517,7 +517,7 @@ class Chat(object):
                     lib.dc_array_get_timestamp(dc_array, i),
                     timezone.utc
                 ),
-                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+                marker=from_optional_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]

python/src/deltachat/direct_imap.py

@@ -4,25 +4,22 @@ and for cleaning up inbox/mvbox for each test function run.
 """
 
 import io
-import email
 import ssl
 import pathlib
-from imapclient import IMAPClient
-from imapclient.exceptions import IMAPClientError
+from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
 import imaplib
 import deltachat
 from deltachat import const, Account
+from typing import List
 
 
-SEEN = b'\\Seen'
-DELETED = b'\\Deleted'
 FLAGS = b'FLAGS'
 FETCH = b'FETCH'
 ALL = "1:*"
 
 
 @deltachat.global_hookimpl
-def dc_account_extra_configure(account):
+def dc_account_extra_configure(account: Account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
@@ -36,12 +33,10 @@ def dc_account_extra_configure(account):
                     assert imap.select_folder(folder)
                     imap.delete(ALL, expunge=True)
                 else:
-                    imap.conn.delete_folder(folder)
+                    imap.conn.folder.delete(folder)
                     # We just deleted the folder, so we have to make DC forget about it, too
                     if account.get_config("configured_sentbox_folder") == folder:
                         account.set_config("configured_sentbox_folder", None)
-                    if account.get_config("configured_spam_folder") == folder:
-                        account.set_config("configured_spam_folder", None)
 
             setattr(account, "direct_imap", imap)
 
@@ -88,37 +83,34 @@ class DirectImap:
             ssl_context.verify_mode = ssl.CERT_NONE
 
         if security == const.DC_SOCKET_STARTTLS:
-            self.conn = IMAPClient(host, port, ssl=False)
-            self.conn.starttls(ssl_context)
-        elif security == const.DC_SOCKET_PLAIN:
-            self.conn = IMAPClient(host, port, ssl=False)
-        elif security == const.DC_SOCKET_SSL:
-            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
+            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
+        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
+            self.conn = MailBox(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
 
     def shutdown(self):
         try:
-            self.conn.idle_done()
-        except (OSError, IMAPClientError):
+            self.idle_done()
+        except (OSError, imaplib.IMAP4.abort):
             pass
         try:
             self.conn.logout()
-        except (OSError, IMAPClientError):
+        except (OSError, imaplib.IMAP4.abort):
             print("Could not logout direct_imap conn")
 
     def create_folder(self, foldername):
         try:
-            self.conn.create_folder(foldername)
-        except imaplib.IMAP4.error as e:
+            self.conn.folder.create(foldername)
+        except errors.MailboxFolderCreateError as e:
             print("Can't create", foldername, "probably it already exists:", str(e))
 
-    def select_folder(self, foldername):
+    def select_folder(self, foldername: str) -> tuple:
         assert not self._idling
-        return self.conn.select_folder(foldername)
+        return self.conn.folder.set(foldername)
 
-    def select_config_folder(self, config_name):
+    def select_config_folder(self, config_name: str):
         """ Return info about selected folder if it is
         configured, otherwise None. """
         if "_" not in config_name:
@@ -127,50 +119,36 @@ class DirectImap:
         if foldername:
             return self.select_folder(foldername)
 
-    def list_folders(self):
+    def list_folders(self) -> List[str]:
         """ return list of all existing folder names"""
         assert not self._idling
-        folders = []
-        for meta, sep, foldername in self.conn.list_folders():
-            folders.append(foldername)
-        return folders
+        return [folder.name for folder in self.conn.folder.list()]
 
-    def delete(self, range, expunge=True):
+    def delete(self, uid_list: str, expunge=True):
         """ delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.set_flags(range, [DELETED])
+        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
         if expunge:
             self.conn.expunge()
 
-    def get_all_messages(self):
+    def get_all_messages(self) -> List[MailMessage]:
         assert not self._idling
+        return [mail for mail in self.conn.fetch()]
 
-        # Flush unsolicited responses. IMAPClient has problems
-        # dealing with them: https://github.com/mjs/imapclient/issues/334
-        # When this NOOP was introduced, next FETCH returned empty
-        # result instead of a single message, even though IMAP server
-        # can only return more untagged responses than required, not
-        # less.
-        self.conn.noop()
-
-        return self.conn.fetch(ALL, [FLAGS])
-
-    def get_unread_messages(self):
+    def get_unread_messages(self) -> List[str]:
         assert not self._idling
-        res = self.conn.fetch(ALL, [FLAGS])
-        return [uid for uid in res
-                if SEEN not in res[uid][FLAGS]]
+        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
         if messages:
-            res = self.conn.set_flags(messages, [SEEN])
+            res = self.conn.flag(messages, MailMessageFlags.SEEN, True)
             print("marked seen:", messages, res)
 
-    def get_unread_cnt(self):
+    def get_unread_cnt(self) -> int:
         return len(self.get_unread_messages())
 
     def dump_imap_structures(self, dir, logfile):
@@ -192,21 +170,20 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[]', FLAGS]
-            for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[]']
-                if not body_bytes:
-                    log("Message", uid, "has empty body")
+            for msg in self.conn.fetch(mark_seen=False):
+                body = getattr(msg.obj, "text", None)
+                if not body:
+                    body = getattr(msg.obj, "html", None)
+                if not body:
+                    log("Message", msg.uid, "has empty body")
                     continue
 
-                flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(uid))
-                fn.write_bytes(body_bytes)
-                log("Message", uid, fn)
-                email_message = email.message_from_bytes(body_bytes)
-                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+                fn = path.joinpath(str(msg.uid))
+                fn.write_bytes(body)
+                log("Message", msg.uid, fn)
+                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
@@ -216,51 +193,58 @@ class DirectImap:
     def idle_start(self):
         """ switch this connection to idle mode. non-blocking. """
         assert not self._idling
-        res = self.conn.idle()
+        res = self.conn.idle.start()
         self._idling = True
         return res
 
-    def idle_check(self, terminate=False):
+    def idle_check(self, terminate=False, timeout=None) -> List[bytes]:
         """ (blocking) wait for next idle message from server. """
         assert self._idling
         self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check(timeout=30)
-        if len(res) == 0:
-            raise TimeoutError
+        res = self.conn.idle.poll(timeout=timeout)
         if terminate:
             self.idle_done()
         self.account.log("imap-direct: idle_check returned {!r}".format(res))
         return res
 
-    def idle_wait_for_seen(self):
-        """ Return first message with SEEN flag
-        from a running idle-stream REtiurn.
+    def idle_wait_for_new_message(self, terminate=False, timeout=None) -> bytes:
+        while 1:
+            for item in self.idle_check(timeout=timeout):
+                if b'EXISTS' in item or b'RECENT' in item:
+                    if terminate:
+                        self.idle_done()
+                    return item
+
+    def idle_wait_for_seen(self, terminate=False, timeout=None) -> int:
+        """ Return first message with SEEN flag from a running idle-stream.
         """
         while 1:
-            for item in self.idle_check():
-                if item[1] == FETCH:
-                    if item[2][0] == FLAGS:
-                        if SEEN in item[2][1]:
-                            return item[0]
+            for item in self.idle_check(timeout=timeout):
+                if FETCH in item:
+                    self.account.log(str(item))
+                    if FLAGS in item and rb'\Seen' in item:
+                        if terminate:
+                            self.idle_done()
+                        return int(item.split(b' ')[1])
 
     def idle_done(self):
         """ send idle-done to server if we are currently in idle mode. """
         if self._idling:
-            res = self.conn.idle_done()
+            res = self.conn.idle.stop()
             self._idling = False
             return res
 
-    def append(self, folder, msg):
+    def append(self, folder: str, msg: str):
         """Upload a message to *folder*.
         Trailing whitespace or a linebreak at the beginning will be removed automatically.
         """
         if msg.startswith("\n"):
             msg = msg[1:]
         msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(folder, msg)
+        self.conn.append(bytes(msg, encoding='ascii'), folder)
 
-    def get_uid_by_message_id(self, message_id):
-        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
+    def get_uid_by_message_id(self, message_id) -> str:
+        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
         if len(msgs) == 0:
             raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
         return msgs[0]

python/src/deltachat/testplugin.py

@@ -241,7 +241,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._evtracker.set_timeout(30)
             ac.addr = ac.get_self_contact().addr
             ac.set_config("displayname", logid)
             if not quiet:
@@ -483,7 +482,7 @@ class BotProcess:
     def kill(self) -> None:
         self.popen.kill()
 
-    def wait(self, timeout=30) -> None:
+    def wait(self, timeout=None) -> None:
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
@@ -492,7 +491,7 @@ class BotProcess:
             print("+++FNMATCH:", next_pattern)
             ignored = []
             while 1:
-                line = self.stdout_queue.get(timeout=15)
+                line = self.stdout_queue.get()
                 if line is None:
                     if ignored:
                         print("BOT stdout terminated after these lines")

python/tests/bench_test_setup.py

@@ -0,0 +1,18 @@
+"""
+run with:
+
+     pytest -vv --durations=10 bench_empty.py
+
+to see timings of test setups.
+"""
+
+import pytest
+
+
+class TestEmpty:
+    def test_prepare_setup_measurings(self, acfactory):
+        acfactory.get_many_online_accounts(5)
+
+    @pytest.mark.parametrize("num", range(0, 5))
+    def test_setup_online_accounts(self, acfactory, num):
+        acfactory.get_many_online_accounts(num)

python/tests/test_account.py

@@ -11,6 +11,7 @@ from deltachat.hookspec import account_hookimpl
 from deltachat.capi import ffi, lib
 from deltachat.cutil import iter_array
 from datetime import datetime, timedelta, timezone
+from imap_tools import AND, U
 
 
 @pytest.mark.parametrize("msgtext,res", [
@@ -652,8 +653,6 @@ class TestOnlineAccount:
             pre_generated_key=False,
             config={"key_gen_type": str(const.DC_KEY_GEN_ED25519)}
         )
-        # rsa key gen can be slow especially on CI, adjust timeout
-        ac1._evtracker.set_timeout(240)
         acfactory.wait_configure_and_start_io()
         chat = acfactory.get_accepted_chat(ac1, ac2)
 
@@ -892,11 +891,11 @@ class TestOnlineAccount:
 
         chat = acfactory.get_accepted_chat(ac1, ac2)
         chat.send_text("message1")
+        ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
         chat.send_text("message2")
+        ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
         chat.send_text("message3")
         ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-        ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-        ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
     def test_forward_messages(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -961,6 +960,18 @@ class TestOnlineAccount:
         chat.send_text("hello")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    def test_send_dot(self, acfactory, lp):
+        """Test that a single dot is properly escaped in SMTP protocol"""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("sending message")
+        msg_out = chat.send_text(".")
+
+        lp.sec("receiving message")
+        msg_in = ac2._evtracker.wait_next_incoming_message()
+        assert msg_in.text == msg_out.text
+
     def test_send_and_receive_message_markseen(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -1015,7 +1026,7 @@ class TestOnlineAccount:
         assert ev.data1 == msg2.chat.id
         assert ev.data2 == 0
 
-        ac2.direct_imap.idle_check(terminate=True)
+        ac2.direct_imap.idle_wait_for_new_message(terminate=True)
         lp.step("1")
         for i in range(2):
             ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
@@ -1046,8 +1057,7 @@ class TestOnlineAccount:
 
         ac1.create_chat(ac2).send_text("Hello!")
 
-        # Wait for the message to arrive.
-        ac2.direct_imap.idle_check(terminate=True)
+        ac2.direct_imap.idle_wait_for_new_message(terminate=True)
 
         # Emulate moving of the message to DeltaChat folder by Sieve rule.
         # mailcow server contains this rule by default.
@@ -1061,13 +1071,9 @@ class TestOnlineAccount:
         # Accept the contact request.
         msg.chat.accept()
         ac2.mark_seen_messages([msg])
-        ac2.direct_imap.idle_wait_for_seen()
-        ac2.direct_imap.idle_done()
+        uid = ac2.direct_imap.idle_wait_for_seen(terminate=True)
 
-        fetch = list(ac2.direct_imap.conn.fetch("*", b'FLAGS').values())
-        flags = fetch[-1][b'FLAGS']
-        is_seen = b'\\Seen' in flags
-        assert is_seen
+        assert len([a for a in ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*")))]) == 1
 
     def test_multidevice_sync_seen(self, acfactory, lp):
         """Test that message marked as seen on one device is marked as seen on another."""
@@ -1101,6 +1107,30 @@ class TestOnlineAccount:
         assert ev.data1 == ac1_clone_chat.id
         assert ac1_clone_message.is_in_seen
 
+        lp.sec("Send an ephemeral message from ac2 to ac1")
+        ac2_chat.set_ephemeral_timer(60)
+        ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        ac1._evtracker.wait_next_incoming_message()
+        ac1_clone._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        ac1_clone._evtracker.wait_next_incoming_message()
+
+        ac2_chat.send_text("Foobar")
+        ac1_message = ac1._evtracker.wait_next_incoming_message()
+        ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
+        assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
+        assert "Expires: " not in ac1_clone_message.get_message_info()
+        assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
+        assert "Expires: " not in ac1_clone_message.get_message_info()
+
+        ac1.mark_seen_messages([ac1_message])
+        assert ac1_message.is_in_seen
+        assert "Expires: " in ac1_message.get_message_info()
+        ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+        assert ev.data1 == ac1_clone_chat.id
+        assert ac1_clone_message.is_in_seen
+        # Test that the timer is started on the second device after synchronizing the seen status.
+        assert "Expires: " in ac1_clone_message.get_message_info()
+
     def test_message_override_sender_name(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         chat = acfactory.get_accepted_chat(ac1, ac2)
@@ -1400,11 +1430,13 @@ class TestOnlineAccount:
         assert not device_chat.can_send()
         assert device_chat.get_draft() is None
 
-    def test_dont_show_emails_in_draft_folder(self, acfactory, lp):
+    def test_dont_show_emails(self, acfactory, lp):
         """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them.
         So: If it's outgoing AND there is no Received header AND it's not in the sentbox, then ignore the email.
 
-        If the draft email is sent out later (i.e. moved to "Sent"), it must be shown."""
+        If the draft email is sent out later (i.e. moved to "Sent"), it must be shown.
+
+        Also, test that unknown emails in the Spam folder are not shown."""
         ac1 = acfactory.get_online_configuring_account()
         ac1.set_config("show_emails", "2")
         ac1.create_contact("alice@example.org").create_chat()
@@ -1412,6 +1444,8 @@ class TestOnlineAccount:
         acfactory.wait_configure(ac1)
         ac1.direct_imap.create_folder("Drafts")
         ac1.direct_imap.create_folder("Sent")
+        ac1.direct_imap.create_folder("Spam")
+        ac1.direct_imap.create_folder("Junk")
 
         acfactory.wait_configure_and_start_io()
         # Wait until each folder was selected once and we are IDLEing again:
@@ -1436,6 +1470,24 @@ class TestOnlineAccount:
 
             message in Sent
         """.format(ac1.get_config("configured_addr")))
+        ac1.direct_imap.append("Spam", """
+            From: unknown.address@junk.org
+            Subject: subj
+            To: {}
+            Message-ID: <spam.message@junk.org>
+            Content-Type: text/plain; charset=utf-8
+
+            Unknown message in Spam
+        """.format(ac1.get_config("configured_addr")))
+        ac1.direct_imap.append("Junk", """
+            From: unknown.address@junk.org
+            Subject: subj
+            To: {}
+            Message-ID: <spam.message@junk.org>
+            Content-Type: text/plain; charset=utf-8
+
+            Unknown message in Junk
+        """.format(ac1.get_config("configured_addr")))
 
         ac1.set_config("scan_all_folders_debounce_secs", "0")
         lp.sec("All prepared, now let DC find the message")
@@ -1449,6 +1501,10 @@ class TestOnlineAccount:
         assert msg.text == "subj – message in Sent"
         assert len(msg.chat.get_messages()) == 1
 
+        assert not any("unknown.address" in c.get_name() for c in ac1.get_chats())
+        ac1.direct_imap.select_folder("Spam")
+        assert ac1.direct_imap.get_uid_by_message_id("spam.message@junk.org")
+
         ac1.stop_io()
         lp.sec("'Send out' the draft, i.e. move it to the Sent folder, and wait for DC to display it this time")
         ac1.direct_imap.select_folder("Drafts")
@@ -1834,7 +1890,6 @@ class TestOnlineAccount:
         lp.sec("trigger ac setup message and return setupcode")
         assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]
         setup_code = ac1.initiate_key_transfer()
-        ac2._evtracker.set_timeout(30)
         ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
         msg = ac2.get_message_by_id(ev.data2)
         assert msg.is_setup_message()
@@ -1851,7 +1906,6 @@ class TestOnlineAccount:
     def test_ac_setup_message_twice(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
         ac2 = acfactory.clone_online_account(ac1)
-        ac2._evtracker.set_timeout(30)
         acfactory.wait_configure_and_start_io()
 
         lp.sec("trigger ac setup message but ignore")
@@ -2025,7 +2079,7 @@ class TestOnlineAccount:
 
         lp.sec("ac1: send a message to group chat to promote the group")
         chat.send_text("afterwards promoted")
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
         assert chat.is_promoted()
         assert sorted(x.addr for x in chat.get_contacts()) == \
@@ -2035,29 +2089,29 @@ class TestOnlineAccount:
         # note that if the above create_chat() would not
         # happen we would not receive a proper member_added event
         contact2 = chat.add_contact("devnull@testrun.org")
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "added"
         assert ev.message.get_sender_contact().addr == ac1_addr
         assert ev.contact.addr == "devnull@testrun.org"
 
         lp.sec("ac1: remove address2")
         chat.remove_contact(contact2)
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "removed"
         assert ev.contact.addr == contact2.addr
         assert ev.message.get_sender_contact().addr == ac1_addr
 
         lp.sec("ac1: remove ac2 contact from chat")
         chat.remove_contact(ac2)
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "removed"
         assert ev.message.get_sender_contact().addr == ac1_addr
 
@@ -2175,7 +2229,7 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.create_chat(ac1).send_text("Hi")
 
-        ac1.direct_imap.idle_check(terminate=False)
+        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
         ac1.maybe_network()
 
         ac1._evtracker.wait_for_all_work_done()
@@ -2187,8 +2241,6 @@ class TestOnlineAccount:
 
         ac2.create_chat(ac1).send_text("Hi 2")
 
-        ac1.direct_imap.idle_check(terminate=True)
-        ac1.maybe_network()
         ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTED, const.DC_CONNECTIVITY_WORKING)
         ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_WORKING, const.DC_CONNECTIVITY_CONNECTED)
 
@@ -2212,7 +2264,7 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.create_chat(ac1).send_text("Hi")
 
-        ac1.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
         ac1.maybe_network()
 
         while 1:
@@ -2402,25 +2454,23 @@ class TestOnlineAccount:
 
         acfactory.wait_configure_and_start_io()
 
-        imap2 = ac2.direct_imap
-        imap2.idle_start()
-
         lp.sec("ac1: create chat with ac2")
         chat1 = ac1.create_chat(ac2)
         ac2.create_chat(ac1)
 
+        lp.sec("ac1: send message to ac2")
         sent_msg = chat1.send_text("hello")
-        imap2.idle_check(terminate=False)
 
         msg = ac2._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        imap2.idle_check(terminate=True)
+        lp.sec("ac2: wait for close/expunge on autodelete")
         ac2._evtracker.get_info_contains("close/expunge succeeded")
 
-        assert len(imap2.get_all_messages()) == 0
+        lp.sec("ac2: check that message was autodeleted on server")
+        assert len(ac2.direct_imap.get_all_messages()) == 0
 
-        # Mark deleted message as seen and check that read receipt arrives
+        lp.sec("ac2: Mark deleted message as seen and check that read receipt arrives")
         msg.mark_seen()
         ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
         assert ev.data1 == chat1.id
@@ -2504,15 +2554,13 @@ class TestOnlineAccount:
         lp.sec("ac2: deleting all messages except third")
         assert len(to_delete) == len(texts) - 1
         ac2.delete_messages(to_delete)
-        for msg in to_delete:
-            ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+        ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
 
         ac2._evtracker.get_info_contains("close/expunge succeeded")
 
-        lp.sec("imap2: test that only one message is left")
-        imap2 = ac2.direct_imap
-
-        assert len(imap2.get_all_messages()) == 1
+        lp.sec("ac2: test that only one message is left")
+        ac2.direct_imap.select_config_folder("inbox")
+        assert len(ac2.direct_imap.get_all_messages()) == 1
 
     def test_configure_error_msgs(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2682,7 +2730,7 @@ class TestOnlineAccount:
         ac1.direct_imap.select_config_folder("inbox")
         ac1.direct_imap.idle_start()
         acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
-        ac1.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
         ac1.direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
 
         lp.sec("Everything prepared, now see if DeltaChat finds the message (" + variant + ")")
@@ -2715,7 +2763,6 @@ class TestOnlineAccount:
                 assert ac.get_config("configured_mvbox_folder")
 
         ac1 = acfactory.get_online_configuring_account(move=mvbox_move)
-        ac1.set_config("sentbox_move", "1")
         ac2 = acfactory.get_online_configuring_account()
 
         acfactory.wait_configure(ac1)
@@ -2732,10 +2779,7 @@ class TestOnlineAccount:
         acfactory.wait_configure_and_start_io()
         assert_folders_configured(ac1)
 
-        if mvbox_move:
-            ac1.direct_imap.select_config_folder("mvbox")
-        else:
-            ac1.direct_imap.select_config_folder("sentbox")
+        assert ac1.direct_imap.select_config_folder("mvbox" if mvbox_move else "inbox")
         ac1.direct_imap.idle_start()
 
         lp.sec("send out message with bcc to ourselves")
@@ -2744,22 +2788,24 @@ class TestOnlineAccount:
         chat.send_text("message text")
         assert_folders_configured(ac1)
 
-        # now wait until the bcc_self message arrives
-        # Also test that bcc_self messages moved to the mvbox are marked as read.
+        lp.sec("wait until the bcc_self message arrives in correct folder and is marked seen")
         assert ac1.direct_imap.idle_wait_for_seen()
         assert_folders_configured(ac1)
 
+        lp.sec("create a cloned ac1 and fetch contact history during configure")
         ac1_clone = acfactory.clone_online_account(ac1)
         ac1_clone.set_config("fetch_existing_msgs", "1")
         ac1_clone._configtracker.wait_finish()
         ac1_clone.start_io()
         assert_folders_configured(ac1_clone)
 
+        lp.sec("check that ac2 contact was fetchted during configure")
         ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
         ac2_addr = ac2.get_config("addr")
         assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
         assert_folders_configured(ac1_clone)
 
+        lp.sec("check that messages changed events arrive for the correct message")
         msg = ac1_clone._evtracker.wait_next_messages_changed()
         assert msg.text == "message text"
         assert_folders_configured(ac1)
@@ -2818,15 +2864,15 @@ class TestOnlineAccount:
         ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure(ac1)
 
-        ac1.direct_imap.conn.delete_folder("DeltaChat")
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 0
+        ac1.direct_imap.conn.folder.delete("DeltaChat")
+        assert "DeltaChat" not in ac1.direct_imap.list_folders()
         acfactory.wait_configure_and_start_io()
 
         ac2.create_chat(ac1).send_text("hello")
         msg = ac1._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 1
+        assert "DeltaChat" in ac1.direct_imap.list_folders()
 
 
 class TestGroupStressTests:

python/tox.ini

@@ -8,7 +8,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n1 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
@@ -78,8 +78,8 @@ commands =
 addopts = -v -ra --strict-markers
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 90
-timeout_method = thread
+timeout = 150
+timeout_func_only = True
 markers = 
     ignored: ignore this test in default test runs, use --ignored to run.
 

rust-toolchain

@@ -1 +1 @@
-1.54.0
+1.60.0

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.54.0
+RUST_VERSION=1.60.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.54.0
+RUST_VERSION=1.60.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

src/accounts.rs

@@ -54,7 +54,11 @@ impl Accounts {
         ensure!(dir.exists().await, "directory does not exist");
 
         let config_file = dir.join(CONFIG_NAME);
-        ensure!(config_file.exists().await, "accounts.toml does not exist");
+        ensure!(
+            config_file.exists().await,
+            "{:?} does not exist",
+            config_file
+        );
 
         let config = Config::from_file(config_file)
             .await
@@ -144,9 +148,27 @@ impl Accounts {
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id).await {
-            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
-                .await
-                .context("failed to remove account data")?;
+            // Spend up to 1 minute trying to remove the files.
+            // Files may remain locked up to 30 seconds due to r2d2 bug:
+            // https://github.com/sfackler/r2d2/issues/99
+            let mut counter = 0;
+            loop {
+                counter += 1;
+
+                if let Err(err) = fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                    .await
+                    .context("failed to remove account data")
+                {
+                    if counter > 60 {
+                        return Err(err);
+                    }
+
+                    // Wait 1 second and try again.
+                    async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
+                } else {
+                    break;
+                }
+            }
         }
         self.config.remove_account(id).await?;
 

src/blob.rs

@@ -3,29 +3,26 @@
 use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
+use std::io::Cursor;
 
 use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use anyhow::format_err;
-use anyhow::Context as _;
-use anyhow::Error;
-use image::DynamicImage;
-use image::GenericImageView;
-use image::ImageFormat;
+use anyhow::{format_err, Context as _, Error};
+use image::{DynamicImage, ImageFormat};
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::{
-    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
-    WORSE_IMAGE_SIZE,
+    MediaQuality, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE, WORSE_IMAGE_SIZE,
 };
 use crate::context::Context;
 use crate::events::EventType;
 use crate::log::LogExt;
 use crate::message;
+use crate::message::Viewtype;
 
 /// Represents a file in the blob directory.
 ///
@@ -292,7 +289,7 @@ impl<'a> BlobObject<'a> {
 
     /// Returns the filename of the blob.
     pub fn as_file_name(&self) -> &str {
-        self.name.rsplitn(2, '/').next().unwrap()
+        self.name.rsplit('/').next().unwrap()
     }
 
     /// The path relative in the blob directory.
@@ -305,7 +302,7 @@ impl<'a> BlobObject<'a> {
     /// If a blob's filename has an extension, it is always guaranteed
     /// to be lowercase.
     pub fn suffix(&self) -> Option<&str> {
-        let ext = self.name.rsplitn(2, '.').next();
+        let ext = self.name.rsplit('.').next();
         if ext == Some(&self.name) {
             None
         } else {
@@ -348,13 +345,30 @@ impl<'a> BlobObject<'a> {
         };
 
         let clean = sanitize_filename::sanitize_with_options(name, opts);
+        // Let's take the tricky filename
+        // "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz" as an example.
+        // Split it into "file" and "with_lots_of_characters_behind_point_and_double_ending.tar.gz":
         let mut iter = clean.splitn(2, '.');
+
         let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
-        let ext: String = iter.next().unwrap_or_default().chars().take(32).collect();
+        // stem == "file"
+
+        let ext_chars = iter.next().unwrap_or_default().chars();
+        let ext: String = ext_chars
+            .rev()
+            .take(32)
+            .collect::<Vec<_>>()
+            .iter()
+            .rev()
+            .collect();
+        // ext == "d_point_and_double_ending.tar.gz"
+
         if ext.is_empty() {
             (stem, "".to_string())
         } else {
             (stem, format!(".{}", ext).to_lowercase())
+            // Return ("file", ".d_point_and_double_ending.tar.gz")
+            // which is not perfect but acceptable.
         }
     }
 
@@ -449,7 +463,8 @@ impl<'a> BlobObject<'a> {
 
         fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
             encoded.clear();
-            img.write_to(encoded, image::ImageFormat::Jpeg)?;
+            let mut buf = Cursor::new(encoded);
+            img.write_to(&mut buf, image::ImageFormat::Jpeg)?;
             Ok(())
         }
         fn encoded_img_exceeds_bytes(
@@ -619,16 +634,14 @@ pub enum BlobError {
 mod tests {
     use fs::File;
 
-    use super::*;
-
-    use crate::chat::{create_group_chat, ProtectionStatus};
-    use crate::{
-        chat,
-        message::Message,
-        test_utils::{self, TestContext},
-    };
     use anyhow::Result;
-    use image::Pixel;
+    use image::{GenericImageView, Pixel};
+
+    use crate::chat::{self, create_group_chat, ProtectionStatus};
+    use crate::message::Message;
+    use crate::test_utils::{self, TestContext};
+
+    use super::*;
 
     #[async_std::test]
     async fn test_create() {
@@ -927,7 +940,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_recode_image() {
+    async fn test_recode_image_1() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
         send_image_check_mediaquality(Some("0"), bytes, 1000, 1000, 0, 1000, 1000)
@@ -944,7 +957,10 @@ mod tests {
         )
         .await
         .unwrap();
+    }
 
+    #[async_std::test]
+    async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(
@@ -960,22 +976,29 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
-        let mut bytes = vec![];
+        let mut buf = Cursor::new(vec![]);
         img_rotated
-            .write_to(&mut bytes, image::ImageFormat::Jpeg)
+            .write_to(&mut buf, image::ImageFormat::Jpeg)
             .unwrap();
-        let img_rotated = send_image_check_mediaquality(
-            Some("0"),
-            &bytes,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-            0,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-        assert_correct_rotation(&img_rotated);
+        let bytes = buf.into_inner();
+
+        // Do this in parallel to speed up the test a bit
+        // (it still takes very long though)
+        let bytes2 = bytes.clone();
+        let join_handle = async_std::task::spawn(async move {
+            let img_rotated = send_image_check_mediaquality(
+                Some("0"),
+                &bytes2,
+                BALANCED_IMAGE_SIZE * 1800 / 2000,
+                BALANCED_IMAGE_SIZE,
+                0,
+                BALANCED_IMAGE_SIZE * 1800 / 2000,
+                BALANCED_IMAGE_SIZE,
+            )
+            .await
+            .unwrap();
+            assert_correct_rotation(&img_rotated);
+        });
 
         let img_rotated = send_image_check_mediaquality(
             Some("1"),
@@ -990,6 +1013,11 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
+        join_handle.await;
+    }
+
+    #[async_std::test]
+    async fn test_recode_image_3() {
         let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
             .await

src/chatlist.rs

@@ -4,13 +4,11 @@ use anyhow::{ensure, Context as _, Result};
 
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
 use crate::constants::{
-    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CONTACT_ID_DEVICE,
-    DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT, DC_GCL_ARCHIVED_ONLY,
-    DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_GCL_ADD_ALLDONE_HINT,
+    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
 };
-use crate::contact::Contact;
+use crate::contact::{Contact, ContactId};
 use crate::context::Context;
-use crate::ephemeral::delete_expired_messages;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock_str;
 use crate::summary::Summary;
@@ -85,19 +83,13 @@ impl Chatlist {
         context: &Context,
         listflags: usize,
         query: Option<&str>,
-        query_contact_id: Option<u32>,
+        query_contact_id: Option<ContactId>,
     ) -> Result<Self> {
         let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
         let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
         let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
         let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
 
-        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
-        // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_expired_messages(context).await {
-            warn!(context, "Failed to hide expired messages: {}", err);
-        }
-
         let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
@@ -112,7 +104,7 @@ impl Chatlist {
         };
 
         let skip_id = if flag_for_forwarding {
-            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+            ChatId::lookup_by_contact(context, ContactId::DEVICE)
                 .await?
                 .unwrap_or_default()
         } else {
@@ -147,7 +139,7 @@ impl Chatlist {
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
                  GROUP BY c.id
                  ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                paramsv![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
+                paramsv![MessageState::OutDraft, query_contact_id, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             ).await?
@@ -216,7 +208,7 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+                ChatId::lookup_by_contact(context, ContactId::SELF)
                     .await?
                     .unwrap_or_default()
             } else {
@@ -326,7 +318,7 @@ impl Chatlist {
 
         let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
             let lastmsg = Message::load_from_db(context, lastmsg_id).await?;
-            if lastmsg.from_id == DC_CONTACT_ID_SELF {
+            if lastmsg.from_id == ContactId::SELF {
                 (Some(lastmsg), None)
             } else {
                 match chat.typ {
@@ -343,7 +335,7 @@ impl Chatlist {
 
         if chat.id.is_archived_link() {
             Ok(Default::default())
-        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED) {
+        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != ContactId::UNDEFINED) {
             Ok(Summary::new(context, &lastmsg, chat, lastcontact.as_ref()).await)
         } else {
             Ok(Summary {
@@ -356,6 +348,10 @@ impl Chatlist {
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
         self.ids.iter().position(|(chat_id, _)| chat_id == &id)
     }
+
+    pub fn iter(&self) -> impl Iterator<Item = &(ChatId, Option<MsgId>)> {
+        self.ids.iter()
+    }
 }
 
 /// Returns the number of archived chats
@@ -375,8 +371,8 @@ mod tests {
     use super::*;
 
     use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus};
-    use crate::constants::Viewtype;
     use crate::dc_receive_imf::dc_receive_imf;
+    use crate::message::Viewtype;
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
 
@@ -507,7 +503,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
                  \n\
                  hello foo\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -568,7 +563,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:38:57 +0000\n\
                  \n\
                  hello foo\n",
-            "INBOX",
             false,
         )
         .await?;

src/config.rs

@@ -1,16 +1,15 @@
 //! # Key-value configuration management.
 
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Context as _, Result};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
-use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
-use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
 
@@ -71,8 +70,12 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MvboxMove,
 
+    /// Watch for new messages in the "Mvbox" (aka DeltaChat folder) only.
+    ///
+    /// This will not entirely disable other folders, e.g. the spam folder will also still
+    /// be watched for new messages.
     #[strum(props(default = "0"))]
-    SentboxMove, // If `MvboxMove` is true, this config is ignored. Currently only used in tests.
+    OnlyFetchMvbox,
 
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
@@ -109,6 +112,7 @@ pub enum Config {
     DeleteDeviceAfter,
 
     SaveMimeHeaders,
+    /// The primary email address. Also see `SecondaryAddrs`.
     ConfiguredAddr,
     ConfiguredMailServer,
     ConfiguredMailUser,
@@ -127,11 +131,14 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
-    ConfiguredSpamFolder,
     ConfiguredTimestamp,
     ConfiguredProvider,
     Configured,
 
+    /// All secondary self addresses separated by spaces
+    /// (`addr1@example.org addr2@exapmle.org addr3@example.org`)
+    SecondaryAddrs,
+
     #[strum(serialize = "sys.version")]
     SysVersion,
 
@@ -225,6 +232,11 @@ impl Context {
         Ok(self.get_config_int(key).await? != 0)
     }
 
+    pub(crate) async fn should_watch_mvbox(&self) -> Result<bool> {
+        Ok(self.get_config_bool(Config::MvboxMove).await?
+            || self.get_config_bool(Config::OnlyFetchMvbox).await?)
+    }
+
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
@@ -233,7 +245,7 @@ impl Context {
         match self.get_config_int(Config::DeleteServerAfter).await? {
             0 => Ok(None),
             1 => Ok(Some(0)),
-            x => Ok(Some(x as i64)),
+            x => Ok(Some(i64::from(x))),
         }
     }
 
@@ -255,7 +267,7 @@ impl Context {
     pub async fn get_config_delete_device_after(&self) -> Result<Option<i64>> {
         match self.get_config_int(Config::DeleteDeviceAfter).await? {
             0 => Ok(None),
-            x => Ok(Some(x as i64)),
+            x => Ok(Some(i64::from(x))),
         }
     }
 
@@ -281,31 +293,22 @@ impl Context {
                     }
                 }
                 self.emit_event(EventType::SelfavatarChanged);
-                Ok(())
             }
             Config::DeleteDeviceAfter => {
-                let ret = self
-                    .sql
-                    .set_raw_config(key, value)
-                    .await
-                    .map_err(Into::into);
-                // Force chatlist reload to delete old messages immediately.
-                self.emit_event(EventType::MsgsChanged {
-                    msg_id: MsgId::new(0),
-                    chat_id: ChatId::new(0),
-                });
-                ret
+                let ret = self.sql.set_raw_config(key, value).await;
+                // Interrupt ephemeral loop to delete old messages immediately.
+                self.interrupt_ephemeral_task().await;
+                ret?
             }
             Config::Displayname => {
                 let value = value.map(improve_single_line_input);
                 self.sql.set_raw_config(key, value.as_deref()).await?;
-                Ok(())
             }
             _ => {
                 self.sql.set_raw_config(key, value).await?;
-                Ok(())
             }
         }
+        Ok(())
     }
 
     pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
@@ -330,6 +333,73 @@ impl Context {
     }
 }
 
+// Separate impl block for self address handling
+impl Context {
+    /// Determine whether the specified addr maps to the/a self addr.
+    /// Returns `false` if no addresses are configured.
+    pub(crate) async fn is_self_addr(&self, addr: &str) -> Result<bool> {
+        Ok(self
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .iter()
+            .any(|a| addr_cmp(addr, a))
+            || self
+                .get_secondary_self_addrs()
+                .await?
+                .iter()
+                .any(|a| addr_cmp(addr, a)))
+    }
+
+    /// Sets `primary_new` as the new primary self address and saves the old
+    /// primary address (if exists) as a secondary address.
+    ///
+    /// This should only be used by test code and during configure.
+    pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
+        // add old primary address (if exists) to secondary addresses
+        let mut secondary_addrs = self.get_all_self_addrs().await?;
+        // never store a primary address also as a secondary
+        secondary_addrs.retain(|a| !addr_cmp(a, primary_new));
+        self.set_config(
+            Config::SecondaryAddrs,
+            Some(secondary_addrs.join(" ").as_str()),
+        )
+        .await?;
+
+        self.set_config(Config::ConfiguredAddr, Some(primary_new))
+            .await?;
+
+        Ok(())
+    }
+
+    /// Returns all primary and secondary self addresses.
+    pub(crate) async fn get_all_self_addrs(&self) -> Result<Vec<String>> {
+        let primary_addrs = self.get_config(Config::ConfiguredAddr).await?.into_iter();
+        let secondary_addrs = self.get_secondary_self_addrs().await?.into_iter();
+
+        Ok(primary_addrs.chain(secondary_addrs).collect())
+    }
+
+    /// Returns all secondary self addresses.
+    pub(crate) async fn get_secondary_self_addrs(&self) -> Result<Vec<String>> {
+        let secondary_addrs = self
+            .get_config(Config::SecondaryAddrs)
+            .await?
+            .unwrap_or_default();
+        Ok(secondary_addrs
+            .split_ascii_whitespace()
+            .map(|s| s.to_string())
+            .collect())
+    }
+
+    /// Returns the primary self address.
+    /// Returns an error if no self addr is configured.
+    pub async fn get_primary_self_addr(&self) -> Result<String> {
+        self.get_config(Config::ConfiguredAddr)
+            .await?
+            .context("No self addr configured")
+    }
+}
+
 /// Returns all available configuration keys concated together.
 fn get_config_keys_string() -> String {
     let keys = Config::iter().fold(String::new(), |mut acc, key| {
@@ -414,4 +484,57 @@ mod tests {
         assert_eq!(t.get_config_bool(c).await?, false);
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_self_addrs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        assert!(alice.is_self_addr("alice@example.org").await?);
+        assert_eq!(alice.get_all_self_addrs().await?, vec!["alice@example.org"]);
+        assert!(!alice.is_self_addr("alice@alice.com").await?);
+
+        // Test adding the same primary address
+        alice.set_primary_self_addr("alice@example.org").await?;
+        alice.set_primary_self_addr("Alice@Example.Org").await?;
+        assert_eq!(alice.get_all_self_addrs().await?, vec!["Alice@Example.Org"]);
+
+        // Test adding a new (primary) self address
+        // The address is trimmed during by `LoginParam::from_database()`,
+        // so `set_primary_self_addr()` doesn't have to trim it.
+        alice.set_primary_self_addr(" Alice@alice.com ").await?;
+        assert!(alice.is_self_addr("    aliCe@example.org").await?);
+        assert!(alice.is_self_addr("alice@alice.com").await?);
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec![" Alice@alice.com ", "Alice@Example.Org"]
+        );
+
+        // Check that the entry is not duplicated
+        alice.set_primary_self_addr("alice@alice.com").await?;
+        alice.set_primary_self_addr("alice@alice.com").await?;
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec!["alice@alice.com", "Alice@Example.Org"]
+        );
+
+        // Test switching back
+        alice.set_primary_self_addr("alice@example.org").await?;
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec!["alice@example.org", "alice@alice.com"]
+        );
+
+        // Test setting a new primary self address, the previous self address
+        // should be kept as a secondary self address
+        alice.set_primary_self_addr("alice@alice.xyz").await?;
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec!["alice@alice.xyz", "alice@example.org", "alice@alice.com"]
+        );
+        assert!(alice.is_self_addr("alice@example.org").await?);
+        assert!(alice.is_self_addr("alice@alice.com").await?);
+        assert!(alice.is_self_addr("Alice@alice.xyz").await?);
+
+        Ok(())
+    }
 }

src/configure.rs

@@ -11,21 +11,20 @@ use async_std::task;
 use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
-use crate::dc_tools::EmailAddress;
+use crate::config::Config;
+use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2};
+use crate::context::Context;
+use crate::dc_tools::{time, EmailAddress};
 use crate::imap::Imap;
+use crate::job;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
-use crate::message::Message;
+use crate::message::{Message, Viewtype};
 use crate::oauth2::dc_get_oauth2_addr;
+use crate::param::Params;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
 use crate::stock_str;
 use crate::{chat, e2ee, provider};
-use crate::{config::Config, dc_tools::time};
-use crate::{
-    constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2},
-    job,
-};
-use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
@@ -86,7 +85,7 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let mut param = LoginParam::from_database(self, "").await?;
+        let mut param = LoginParam::load_candidate_params(self).await?;
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
@@ -443,7 +442,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 900);
 
-    let create_mvbox = ctx.get_config_bool(Config::MvboxMove).await?;
+    let create_mvbox = ctx.should_watch_mvbox().await?;
 
     imap.configure_folders(ctx, create_mvbox).await?;
 
@@ -454,8 +453,14 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     drop(imap);
 
     progress!(ctx, 910);
+
+    if ctx.get_config(Config::ConfiguredAddr).await?.as_deref() != Some(&param.addr) {
+        // Switched account, all server UIDs we know are invalid
+        job::schedule_resync(ctx).await?;
+    }
+
     // the trailing underscore is correct
-    param.save_to_database(ctx, "configured_").await?;
+    param.save_as_configured_params(ctx).await?;
     ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
@@ -700,11 +705,11 @@ pub enum Error {
         error: quick_xml::Error,
     },
 
-    #[error("Failed to get URL: {0}")]
-    ReadUrl(#[from] self::read_url::Error),
-
     #[error("Number of redirection is exceeded")]
     Redirection,
+
+    #[error("{0:#}")]
+    Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]

src/configure/read_url.rs

@@ -1,20 +1,40 @@
 use crate::context::Context;
 
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("URL request error")]
-    GetError(surf::Error),
-}
+use anyhow::format_err;
+use anyhow::Context as _;
 
-pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
-    info!(context, "Requesting URL {}", url);
-
-    match surf::get(url).recv_string().await {
-        Ok(res) => Ok(res),
-        Err(err) => {
-            info!(context, "Can\'t read URL {}: {}", url, err);
-
-            Err(Error::GetError(err))
+pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
+    match read_url_inner(context, url).await {
+        Ok(s) => {
+            info!(context, "Successfully read url {}", url);
+            Ok(s)
+        }
+        Err(e) => {
+            info!(context, "Can't read URL {}: {:#}", url, e);
+            Err(format_err!("Can't read URL {}: {:#}", url, e))
         }
     }
 }
+
+pub async fn read_url_inner(context: &Context, mut url: &str) -> anyhow::Result<String> {
+    let mut _temp; // For the borrow checker
+
+    // Follow up to 10 http-redirects
+    for _i in 0..10 {
+        let mut response = surf::get(url).send().await.map_err(|e| e.into_inner())?;
+        if response.status().is_redirection() {
+            _temp = response
+                .header("location")
+                .context("Redirection doesn't have a target location")?
+                .last()
+                .to_string();
+            info!(context, "Following redirect to {}", _temp);
+            url = &_temp;
+            continue;
+        }
+
+        return response.body_string().await.map_err(|e| e.into_inner());
+    }
+
+    Err(format_err!("Followed 10 redirections"))
+}

src/configure/server_params.rs

@@ -52,10 +52,8 @@ impl ServerParams {
     fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
         if self.hostname.is_empty() {
             vec![
-                Self {
-                    hostname: param_domain.to_string(),
-                    ..self.clone()
-                },
+                // Try "imap.ex.org"/"smtp.ex.org" and "mail.ex.org" first because if a server exists
+                // under this address, it's likely the correct one.
                 Self {
                     hostname: match self.protocol {
                         Protocol::Imap => "imap.".to_string() + param_domain,
@@ -65,6 +63,12 @@ impl ServerParams {
                 },
                 Self {
                     hostname: "mail.".to_string() + param_domain,
+                    ..self.clone()
+                },
+                // Try "ex.org" last because if it's wrong and the server is configured to
+                // not answer at all, configuration may be stuck for several minutes.
+                Self {
+                    hostname: param_domain.to_string(),
                     ..self
                 },
             ]
@@ -296,5 +300,48 @@ mod tests {
                 strict_tls: Some(true)
             }],
         );
+
+        // Test that "example.net" is tried after "*.example.net".
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Imap,
+                hostname: "".to_string(),
+                port: 10480,
+                socket: Socket::Ssl,
+                username: "foobar".to_string(),
+                strict_tls: Some(true),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+        assert_eq!(
+            v,
+            vec![
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "imap.example.net".to_string(),
+                    port: 10480,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                },
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "mail.example.net".to_string(),
+                    port: 10480,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                },
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "example.net".to_string(),
+                    port: 10480,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                }
+            ],
+        );
     }
 }

src/constants.rs

@@ -179,16 +179,6 @@ pub const DC_ELLIPSIS: &str = "[...]";
 /// `char`s), not Unicode Grapheme Clusters.
 pub const DC_DESIRED_TEXT_LEN: usize = 5000;
 
-pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
-pub const DC_CONTACT_ID_SELF: u32 = 1;
-pub const DC_CONTACT_ID_INFO: u32 = 2;
-pub const DC_CONTACT_ID_DEVICE: u32 = 5;
-pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
-
-// decorative address that is used for DC_CONTACT_ID_DEVICE
-// when an api that returns an email is called.
-pub const DC_CONTACT_ID_DEVICE_ADDR: &str = "device@localhost";
-
 // Flags for empty server job
 
 pub const DC_EMPTY_MVBOX: u32 = 0x01;
@@ -230,82 +220,6 @@ pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 // can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
 pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
 
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
-#[repr(u32)]
-pub enum Viewtype {
-    Unknown = 0,
-
-    /// Text message.
-    /// The text of the message is set using dc_msg_set_text()
-    /// and retrieved with dc_msg_get_text().
-    Text = 10,
-
-    /// Image message.
-    /// If the image is an animated GIF, the type DC_MSG_GIF should be used.
-    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension
-    /// and retrieved via dc_msg_set_file(), dc_msg_set_dimension().
-    Image = 20,
-
-    /// Animated GIF message.
-    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension()
-    /// and retrieved via dc_msg_get_file(), dc_msg_get_width(), dc_msg_get_height().
-    Gif = 21,
-
-    /// Message containing a sticker, similar to image.
-    /// If possible, the ui should display the image without borders in a transparent way.
-    /// A click on a sticker will offer to install the sticker set in some future.
-    Sticker = 23,
-
-    /// Message containing an Audio file.
-    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
-    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration().
-    Audio = 40,
-
-    /// A voice message that was directly recorded by the user.
-    /// For all other audio messages, the type #DC_MSG_AUDIO should be used.
-    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
-    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration()
-    Voice = 41,
-
-    /// Video messages.
-    /// File, width, height and durarion
-    /// are set via dc_msg_set_file(), dc_msg_set_dimension(), dc_msg_set_duration()
-    /// and retrieved via
-    /// dc_msg_get_file(), dc_msg_get_width(),
-    /// dc_msg_get_height(), dc_msg_get_duration().
-    Video = 50,
-
-    /// Message containing any file, eg. a PDF.
-    /// The file is set via dc_msg_set_file()
-    /// and retrieved via dc_msg_get_file().
-    File = 60,
-
-    /// Message is an invitation to a videochat.
-    VideochatInvitation = 70,
-
-    /// Message is an webxdc instance.
-    Webxdc = 80,
-}
-
-impl Default for Viewtype {
-    fn default() -> Self {
-        Viewtype::Unknown
-    }
-}
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
@@ -317,33 +231,9 @@ pub enum KeyType {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use num_traits::FromPrimitive;
 
-    #[test]
-    fn test_derive_display_works_as_expected() {
-        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
-    }
-
-    #[test]
-    fn test_viewtype_values() {
-        // values may be written to disk and must not change
-        assert_eq!(Viewtype::Unknown, Viewtype::default());
-        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
-        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
-        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
-        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
-        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
-        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
-        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
-        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
-        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
-        assert_eq!(
-            Viewtype::VideochatInvitation,
-            Viewtype::from_i32(70).unwrap()
-        );
-        assert_eq!(Viewtype::Webxdc, Viewtype::from_i32(80).unwrap());
-    }
+    use super::*;
 
     #[test]
     fn test_chattype_values() {

src/contact.rs

@@ -1,21 +1,20 @@
 //! Contacts module
 
 use std::convert::{TryFrom, TryInto};
+use std::fmt;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::path::PathBuf;
 use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use regex::Regex;
+use serde::{Deserialize, Serialize};
 
 use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
 use crate::color::str_to_color;
 use crate::config::Config;
-use crate::constants::{
-    Blocked, Chattype, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_DEVICE_ADDR, DC_CONTACT_ID_LAST_SPECIAL,
-    DC_CONTACT_ID_SELF, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY,
-};
+use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input, EmailAddress};
 use crate::events::EventType;
@@ -25,8 +24,94 @@ use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
+use crate::sql::{self, params_iter};
 use crate::{chat, stock_str};
 
+/// Contact ID, including reserved IDs.
+///
+/// Some contact IDs are reserved to identify special contacts.  This
+/// type can represent both the special as well as normal contacts.
+#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct ContactId(u32);
+
+impl ContactId {
+    pub const UNDEFINED: ContactId = ContactId::new(0);
+    /// The owner of the account.
+    ///
+    /// The email-address is set by `dc_set_config` using "addr".
+    pub const SELF: ContactId = ContactId::new(1);
+    pub const INFO: ContactId = ContactId::new(2);
+    pub const DEVICE: ContactId = ContactId::new(5);
+    const LAST_SPECIAL: ContactId = ContactId::new(9);
+
+    /// Address to go with [`ContactId::DEVICE`].
+    ///
+    /// This is used by APIs which need to return an email address for this contact.
+    pub const DEVICE_ADDR: &'static str = "device@localhost";
+
+    /// Creates a new [`ContactId`].
+    pub const fn new(id: u32) -> ContactId {
+        ContactId(id)
+    }
+
+    /// Whether this is a special [`ContactId`].
+    ///
+    /// Some [`ContactId`]s are reserved for special contacts like [`ContactId::SELF`],
+    /// [`ContactId::INFO`] and [`ContactId::DEVICE`].  This function indicates whether this
+    /// [`ContactId`] is any of the reserved special [`ContactId`]s (`true`) or whether it
+    /// is the [`ContactId`] of a real contact (`false`).
+    pub fn is_special(&self) -> bool {
+        self.0 <= Self::LAST_SPECIAL.0
+    }
+
+    /// Numerical representation of the [`ContactId`].
+    ///
+    /// Each contact ID has a unique numerical representation which is used in the database
+    /// (via [`rusqlite::ToSql`]) and also for FFI purposes.  In Rust code you should never
+    /// need to use this directly.
+    pub const fn to_u32(&self) -> u32 {
+        self.0
+    }
+}
+
+impl fmt::Display for ContactId {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        if *self == ContactId::UNDEFINED {
+            write!(f, "Contact#Undefined")
+        } else if *self == ContactId::SELF {
+            write!(f, "Contact#Self")
+        } else if *self == ContactId::INFO {
+            write!(f, "Contact#Info")
+        } else if *self == ContactId::DEVICE {
+            write!(f, "Contact#Device")
+        } else if self.is_special() {
+            write!(f, "Contact#Special{}", self.0)
+        } else {
+            write!(f, "Contact#{}", self.0)
+        }
+    }
+}
+
+/// Allow converting [`ContactId`] to an SQLite type.
+impl rusqlite::types::ToSql for ContactId {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Integer(i64::from(self.0));
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+/// Allow converting an SQLite integer directly into [`ContactId`].
+impl rusqlite::types::FromSql for ContactId {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|val| {
+            val.try_into()
+                .map(ContactId::new)
+                .map_err(|_| rusqlite::types::FromSqlError::OutOfRange(val))
+        })
+    }
+}
+
 /// An object representing a single contact in memory.
 ///
 /// The contact object is not updated.
@@ -42,13 +127,7 @@ use crate::{chat, stock_str};
 #[derive(Debug)]
 pub struct Contact {
     /// The contact ID.
-    ///
-    /// Special message IDs:
-    /// - DC_CONTACT_ID_SELF (1) - this is the owner of the account with the email-address set by
-    ///   `dc_set_config` using "addr".
-    ///
-    /// Normal contact IDs are larger than these special ones (larger than DC_CONTACT_ID_LAST_SPECIAL).
-    pub id: u32,
+    pub id: ContactId,
 
     /// Contact name. It is recommended to use `Contact::get_name`,
     /// `Contact::get_display_name` or `Contact::get_name_n_addr` to access this field.
@@ -183,7 +262,7 @@ impl Default for VerifiedStatus {
 }
 
 impl Contact {
-    pub async fn load_from_db(context: &Context, contact_id: u32) -> Result<Self> {
+    pub async fn load_from_db(context: &Context, contact_id: ContactId) -> Result<Self> {
         let mut contact = context
             .sql
             .query_row(
@@ -191,7 +270,7 @@ impl Contact {
                 c.authname, c.param, c.status
                FROM contacts c
               WHERE c.id=?;",
-                paramsv![contact_id as i32],
+                paramsv![contact_id],
                 |row| {
                     let name: String = row.get(0)?;
                     let addr: String = row.get(1)?;
@@ -216,7 +295,7 @@ impl Contact {
                 },
             )
             .await?;
-        if contact_id == DC_CONTACT_ID_SELF {
+        if contact_id == ContactId::SELF {
             contact.name = stock_str::self_msg(context).await;
             contact.addr = context
                 .get_config(Config::ConfiguredAddr)
@@ -226,9 +305,9 @@ impl Contact {
                 .get_config(Config::Selfstatus)
                 .await?
                 .unwrap_or_default();
-        } else if contact_id == DC_CONTACT_ID_DEVICE {
+        } else if contact_id == ContactId::DEVICE {
             contact.name = stock_str::device_messages(context).await;
-            contact.addr = DC_CONTACT_ID_DEVICE_ADDR.to_string();
+            contact.addr = ContactId::DEVICE_ADDR.to_string();
             contact.status = stock_str::device_messages_hint(context).await;
         }
         Ok(contact)
@@ -245,18 +324,18 @@ impl Contact {
     }
 
     /// Check if a contact is blocked.
-    pub async fn is_blocked_load(context: &Context, id: u32) -> Result<bool> {
+    pub async fn is_blocked_load(context: &Context, id: ContactId) -> Result<bool> {
         let blocked = Self::load_from_db(context, id).await?.blocked;
         Ok(blocked)
     }
 
     /// Block the given contact.
-    pub async fn block(context: &Context, id: u32) -> Result<()> {
+    pub async fn block(context: &Context, id: ContactId) -> Result<()> {
         set_block_contact(context, id, true).await
     }
 
     /// Unblock the given contact.
-    pub async fn unblock(context: &Context, id: u32) -> Result<()> {
+    pub async fn unblock(context: &Context, id: ContactId) -> Result<()> {
         set_block_contact(context, id, false).await
     }
 
@@ -269,7 +348,7 @@ impl Contact {
     /// a bunch of addresses.
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
-    pub async fn create(context: &Context, name: &str, addr: &str) -> Result<u32> {
+    pub async fn create(context: &Context, name: &str, addr: &str) -> Result<ContactId> {
         let name = improve_single_line_input(name);
         ensure!(!addr.is_empty(), "Cannot create contact with empty address");
 
@@ -292,12 +371,12 @@ impl Contact {
     }
 
     /// Mark messages from a contact as noticed.
-    pub async fn mark_noticed(context: &Context, id: u32) -> Result<()> {
+    pub async fn mark_noticed(context: &Context, id: ContactId) -> Result<()> {
         context
             .sql
             .execute(
                 "UPDATE msgs SET state=? WHERE from_id=? AND state=?;",
-                paramsv![MessageState::InNoticed, id as i32, MessageState::InFresh],
+                paramsv![MessageState::InNoticed, id, MessageState::InFresh],
             )
             .await?;
         Ok(())
@@ -313,29 +392,24 @@ impl Contact {
         context: &Context,
         addr: &str,
         min_origin: Origin,
-    ) -> Result<Option<u32>> {
+    ) -> Result<Option<ContactId>> {
         if addr.is_empty() {
             bail!("lookup_id_by_addr: empty address");
         }
 
         let addr_normalized = addr_normalize(addr);
 
-        if let Some(addr_self) = context.get_config(Config::ConfiguredAddr).await? {
-            if addr_cmp(addr_normalized, &addr_self) {
-                return Ok(Some(DC_CONTACT_ID_SELF));
-            }
+        if context.is_self_addr(addr_normalized).await? {
+            return Ok(Some(ContactId::SELF));
         }
+
         let id = context
             .sql
             .query_get_value(
                 "SELECT id FROM contacts \
             WHERE addr=?1 COLLATE NOCASE \
             AND id>?2 AND origin>=?3 AND blocked=0;",
-                paramsv![
-                    addr_normalized,
-                    DC_CONTACT_ID_LAST_SPECIAL as i32,
-                    min_origin as u32,
-                ],
+                paramsv![addr_normalized, ContactId::LAST_SPECIAL, min_origin as u32,],
             )
             .await?;
         Ok(id)
@@ -371,20 +445,16 @@ impl Contact {
         name: &str,
         addr: &str,
         mut origin: Origin,
-    ) -> Result<(u32, Modifier)> {
+    ) -> Result<(ContactId, Modifier)> {
         let mut sth_modified = Modifier::None;
 
         ensure!(!addr.is_empty(), "Can not add_or_lookup empty address");
         ensure!(origin != Origin::Unknown, "Missing valid origin");
 
         let addr = addr_normalize(addr).to_string();
-        let addr_self = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
 
-        if addr_cmp(&addr, &addr_self) {
-            return Ok((DC_CONTACT_ID_SELF, sth_modified));
+        if context.is_self_addr(&addr).await? {
+            return Ok((ContactId::SELF, sth_modified));
         }
 
         if !may_be_valid_addr(&addr) {
@@ -500,7 +570,7 @@ impl Contact {
                         paramsv![Chattype::Single, isize::try_from(row_id)?]
                     ).await?;
                     if let Some(chat_id) = chat_id {
-                        let contact = Contact::get_by_id(context, row_id as u32).await?;
+                        let contact = Contact::get_by_id(context, ContactId::new(row_id)).await?;
                         let chat_name = contact.get_display_name();
                         match context
                             .sql
@@ -557,7 +627,7 @@ impl Contact {
             }
         }
 
-        Ok((row_id, sth_modified))
+        Ok((ContactId::new(row_id), sth_modified))
     }
 
     /// Add a number of contacts.
@@ -617,12 +687,8 @@ impl Contact {
         context: &Context,
         listflags: u32,
         query: Option<impl AsRef<str>>,
-    ) -> Result<Vec<u32>> {
-        let self_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
-
+    ) -> Result<Vec<ContactId>> {
+        let self_addrs = context.get_all_self_addrs().await?;
         let mut add_self = false;
         let mut ret = Vec::new();
         let flag_verified_only = (listflags & DC_GCL_VERIFIED_ONLY) != 0;
@@ -633,40 +699,46 @@ impl Contact {
             context
                 .sql
                 .query_map(
-                    "SELECT c.id FROM contacts c \
+                    format!(
+                        "SELECT c.id FROM contacts c \
                  LEFT JOIN acpeerstates ps ON c.addr=ps.addr  \
-                 WHERE c.addr!=?1 \
-                 AND c.id>?2 \
-                 AND c.origin>=?3 \
+                 WHERE c.addr NOT IN ({})
+                 AND c.id>? \
+                 AND c.origin>=? \
                  AND c.blocked=0 \
-                 AND (iif(c.name='',c.authname,c.name) LIKE ?4 OR c.addr LIKE ?5) \
-                 AND (1=?6 OR LENGTH(ps.verified_key_fingerprint)!=0)  \
+                 AND (iif(c.name='',c.authname,c.name) LIKE ? OR c.addr LIKE ?) \
+                 AND (1=? OR LENGTH(ps.verified_key_fingerprint)!=0)  \
                  ORDER BY LOWER(iif(c.name='',c.authname,c.name)||c.addr),c.id;",
-                    paramsv![
-                        self_addr,
-                        DC_CONTACT_ID_LAST_SPECIAL as i32,
+                        sql::repeat_vars(self_addrs.len())
+                    ),
+                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
+                        ContactId::LAST_SPECIAL,
                         Origin::IncomingReplyTo,
                         s3str_like_cmd,
                         s3str_like_cmd,
                         if flag_verified_only { 0i32 } else { 1i32 },
-                    ],
-                    |row| row.get::<_, i32>(0),
+                    ])),
+                    |row| row.get::<_, ContactId>(0),
                     |ids| {
                         for id in ids {
-                            ret.push(id? as u32);
+                            ret.push(id?);
                         }
                         Ok(())
                     },
                 )
                 .await?;
 
-            let self_name = context
-                .get_config(Config::Displayname)
-                .await?
-                .unwrap_or_default();
-            let self_name2 = stock_str::self_msg(context);
-
             if let Some(query) = query {
+                let self_addr = context
+                    .get_config(Config::ConfiguredAddr)
+                    .await?
+                    .unwrap_or_default();
+                let self_name = context
+                    .get_config(Config::Displayname)
+                    .await?
+                    .unwrap_or_default();
+                let self_name2 = stock_str::self_msg(context);
+
                 if self_addr.contains(query.as_ref())
                     || self_name.contains(query.as_ref())
                     || self_name2.await.contains(query.as_ref())
@@ -682,21 +754,23 @@ impl Contact {
             context
                 .sql
                 .query_map(
-                    "SELECT id FROM contacts
-                 WHERE addr!=?1
-                 AND id>?2
-                 AND origin>=?3
+                    format!(
+                        "SELECT id FROM contacts
+                 WHERE addr NOT IN ({})
+                 AND id>?
+                 AND origin>=?
                  AND blocked=0
                  ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                    paramsv![
-                        self_addr,
-                        DC_CONTACT_ID_LAST_SPECIAL as i32,
+                        sql::repeat_vars(self_addrs.len())
+                    ),
+                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
+                        ContactId::LAST_SPECIAL,
                         Origin::IncomingReplyTo
-                    ],
-                    |row| row.get::<_, i32>(0),
+                    ])),
+                    |row| row.get::<_, ContactId>(0),
                     |ids| {
                         for id in ids {
-                            ret.push(id? as u32);
+                            ret.push(id?);
                         }
                         Ok(())
                     },
@@ -705,7 +779,7 @@ impl Contact {
         }
 
         if flag_add_self && add_self {
-            ret.push(DC_CONTACT_ID_SELF);
+            ret.push(ContactId::SELF);
         }
 
         Ok(ret)
@@ -760,14 +834,14 @@ impl Contact {
             .sql
             .count(
                 "SELECT COUNT(*) FROM contacts WHERE id>? AND blocked!=0",
-                paramsv![DC_CONTACT_ID_LAST_SPECIAL],
+                paramsv![ContactId::LAST_SPECIAL],
             )
             .await?;
         Ok(count as usize)
     }
 
     /// Get blocked contacts.
-    pub async fn get_all_blocked(context: &Context) -> Result<Vec<u32>> {
+    pub async fn get_all_blocked(context: &Context) -> Result<Vec<ContactId>> {
         Contact::update_blocked_mailinglist_contacts(context)
             .await
             .context("cannot update blocked mailinglist contacts")?;
@@ -776,8 +850,8 @@ impl Contact {
             .sql
             .query_map(
                 "SELECT id FROM contacts WHERE id>? AND blocked!=0 ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                paramsv![DC_CONTACT_ID_LAST_SPECIAL as i32],
-                |row| row.get::<_, u32>(0),
+                paramsv![ContactId::LAST_SPECIAL],
+                |row| row.get::<_, ContactId>(0),
                 |ids| {
                     ids.collect::<std::result::Result<Vec<_>, _>>()
                         .map_err(Into::into)
@@ -792,15 +866,15 @@ impl Contact {
     /// This function returns a string explaining the encryption state
     /// of the contact and if the connection is encrypted the
     /// fingerprints of the keys involved.
-    pub async fn get_encrinfo(context: &Context, contact_id: u32) -> Result<String> {
+    pub async fn get_encrinfo(context: &Context, contact_id: ContactId) -> Result<String> {
         ensure!(
-            contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+            !contact_id.is_special(),
             "Can not provide encryption info for special contact"
         );
 
         let mut ret = String::new();
         if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            let loginparam = LoginParam::from_database(context, "configured_").await?;
+            let loginparam = LoginParam::load_configured_params(context).await?;
             let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
 
             if let Some(peerstate) = peerstate.filter(|peerstate| {
@@ -861,27 +935,21 @@ impl Contact {
     /// possible as the contact is in use. In this case, the contact can be blocked.
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
-    pub async fn delete(context: &Context, contact_id: u32) -> Result<()> {
-        ensure!(
-            contact_id > DC_CONTACT_ID_LAST_SPECIAL,
-            "Can not delete special contact"
-        );
+    pub async fn delete(context: &Context, contact_id: ContactId) -> Result<()> {
+        ensure!(!contact_id.is_special(), "Can not delete special contact");
 
         let count_chats = context
             .sql
             .count(
                 "SELECT COUNT(*) FROM chats_contacts WHERE contact_id=?;",
-                paramsv![contact_id as i32],
+                paramsv![contact_id],
             )
             .await?;
 
         if count_chats == 0 {
             match context
                 .sql
-                .execute(
-                    "DELETE FROM contacts WHERE id=?;",
-                    paramsv![contact_id as i32],
-                )
+                .execute("DELETE FROM contacts WHERE id=?;", paramsv![contact_id])
                 .await
             {
                 Ok(_) => {
@@ -904,10 +972,10 @@ impl Contact {
 
     /// Get a single contact object.  For a list, see eg. dc_get_contacts().
     ///
-    /// For contact DC_CONTACT_ID_SELF (1), the function returns sth.
+    /// For contact ContactId::SELF (1), the function returns sth.
     /// like "Me" in the selected language and the email address
     /// defined by dc_set_config().
-    pub async fn get_by_id(context: &Context, contact_id: u32) -> Result<Contact> {
+    pub async fn get_by_id(context: &Context, contact_id: ContactId) -> Result<Contact> {
         let contact = Contact::load_from_db(context, contact_id).await?;
 
         Ok(contact)
@@ -919,7 +987,7 @@ impl Contact {
             .sql
             .execute(
                 "UPDATE contacts SET param=? WHERE id=?",
-                paramsv![self.param.to_string(), self.id as i32],
+                paramsv![self.param.to_string(), self.id],
             )
             .await?;
         Ok(())
@@ -931,14 +999,14 @@ impl Contact {
             .sql
             .execute(
                 "UPDATE contacts SET status=? WHERE id=?",
-                paramsv![self.status, self.id as i32],
+                paramsv![self.status, self.id],
             )
             .await?;
         Ok(())
     }
 
     /// Get the ID of the contact.
-    pub fn get_id(&self) -> u32 {
+    pub fn get_id(&self) -> ContactId {
         self.id
     }
 
@@ -997,7 +1065,7 @@ impl Contact {
     /// This is the image set by each remote user on their own
     /// using dc_set_config(context, "selfavatar", image).
     pub async fn get_profile_image(&self, context: &Context) -> Result<Option<PathBuf>> {
-        if self.id == DC_CONTACT_ID_SELF {
+        if self.id == ContactId::SELF {
             if let Some(p) = context.get_config(Config::Selfavatar).await? {
                 return Ok(Some(PathBuf::from(p)));
             }
@@ -1043,7 +1111,7 @@ impl Contact {
     ) -> Result<VerifiedStatus> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
-        if self.id == DC_CONTACT_ID_SELF {
+        if self.id == ContactId::SELF {
             return Ok(VerifiedStatus::BidirectVerified);
         }
 
@@ -1062,26 +1130,6 @@ impl Contact {
         Ok(VerifiedStatus::Unverified)
     }
 
-    pub async fn addr_equals_contact(
-        context: &Context,
-        addr: &str,
-        contact_id: u32,
-    ) -> Result<bool> {
-        if addr.is_empty() {
-            return Ok(false);
-        }
-
-        let contact = Contact::load_from_db(context, contact_id).await?;
-        if !contact.addr.is_empty() {
-            let normalized_addr = addr_normalize(addr);
-            if contact.addr == normalized_addr {
-                return Ok(true);
-            }
-        }
-
-        Ok(false)
-    }
-
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
         if !context.sql.is_open().await {
             return Ok(0);
@@ -1091,14 +1139,14 @@ impl Contact {
             .sql
             .count(
                 "SELECT COUNT(*) FROM contacts WHERE id>?;",
-                paramsv![DC_CONTACT_ID_LAST_SPECIAL as i32],
+                paramsv![ContactId::LAST_SPECIAL],
             )
             .await?;
         Ok(count)
     }
 
-    pub async fn real_exists_by_id(context: &Context, contact_id: u32) -> Result<bool> {
-        if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+    pub async fn real_exists_by_id(context: &Context, contact_id: ContactId) -> Result<bool> {
+        if contact_id.is_special() {
             return Ok(false);
         }
 
@@ -1106,7 +1154,7 @@ impl Contact {
             .sql
             .exists(
                 "SELECT COUNT(*) FROM contacts WHERE id=?;",
-                paramsv![contact_id as i32],
+                paramsv![contact_id],
             )
             .await?;
         Ok(exists)
@@ -1114,14 +1162,14 @@ impl Contact {
 
     pub async fn scaleup_origin_by_id(
         context: &Context,
-        contact_id: u32,
+        contact_id: ContactId,
         origin: Origin,
     ) -> Result<()> {
         context
             .sql
             .execute(
                 "UPDATE contacts SET origin=? WHERE id=? AND origin<?;",
-                paramsv![origin, contact_id as i32, origin],
+                paramsv![origin, contact_id, origin],
             )
             .await?;
         Ok(())
@@ -1165,9 +1213,13 @@ fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     }
 }
 
-async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) -> Result<()> {
+async fn set_block_contact(
+    context: &Context,
+    contact_id: ContactId,
+    new_blocking: bool,
+) -> Result<()> {
     ensure!(
-        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+        !contact_id.is_special(),
         "Can't block special contact {}",
         contact_id
     );
@@ -1179,7 +1231,7 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
             .sql
             .execute(
                 "UPDATE contacts SET blocked=? WHERE id=?;",
-                paramsv![new_blocking as i32, contact_id as i32],
+                paramsv![i32::from(new_blocking), contact_id],
             )
             .await?;
 
@@ -1230,14 +1282,14 @@ WHERE type=? AND id IN (
 /// this typically happens if we see message with our own profile image, sent from another device.
 pub(crate) async fn set_profile_image(
     context: &Context,
-    contact_id: u32,
+    contact_id: ContactId,
     profile_image: &AvatarAction,
     was_encrypted: bool,
 ) -> Result<()> {
     let mut contact = Contact::load_from_db(context, contact_id).await?;
     let changed = match profile_image {
         AvatarAction::Change(profile_image) => {
-            if contact_id == DC_CONTACT_ID_SELF {
+            if contact_id == ContactId::SELF {
                 if was_encrypted {
                     context
                         .set_config(Config::Selfavatar, Some(profile_image))
@@ -1251,7 +1303,7 @@ pub(crate) async fn set_profile_image(
             true
         }
         AvatarAction::Delete => {
-            if contact_id == DC_CONTACT_ID_SELF {
+            if contact_id == ContactId::SELF {
                 if was_encrypted {
                     context.set_config(Config::Selfavatar, None).await?;
                 } else {
@@ -1277,12 +1329,12 @@ pub(crate) async fn set_profile_image(
 /// between Delta Chat devices.
 pub(crate) async fn set_status(
     context: &Context,
-    contact_id: u32,
+    contact_id: ContactId,
     status: String,
     encrypted: bool,
     has_chat_version: bool,
 ) -> Result<()> {
-    if contact_id == DC_CONTACT_ID_SELF {
+    if contact_id == ContactId::SELF {
         if encrypted && has_chat_version {
             context
                 .set_config(Config::Selfstatus, Some(&status))
@@ -1303,11 +1355,11 @@ pub(crate) async fn set_status(
 /// Updates last seen timestamp of the contact if it is earlier than the given `timestamp`.
 pub(crate) async fn update_last_seen(
     context: &Context,
-    contact_id: u32,
+    contact_id: ContactId,
     timestamp: i64,
 ) -> Result<()> {
     ensure!(
-        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+        !contact_id.is_special(),
         "Can not update special contact last seen timestamp"
     );
 
@@ -1368,17 +1420,6 @@ fn cat_fingerprint(
     }
 }
 
-impl Context {
-    /// determine whether the specified addr maps to the/a self addr
-    pub async fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        if let Some(self_addr) = self.get_config(Config::ConfiguredAddr).await? {
-            Ok(addr_cmp(&self_addr, addr))
-        } else {
-            Ok(false)
-        }
-    }
-}
-
 pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
     let norm1 = addr_normalize(addr1).to_lowercase();
     let norm2 = addr_normalize(addr2).to_lowercase();
@@ -1411,6 +1452,17 @@ mod tests {
     use crate::message::Message;
     use crate::test_utils::{self, TestContext};
 
+    #[test]
+    fn test_contact_id_values() {
+        // Some FFI users need to have the values of these fixed, how naughty.  But let's
+        // make sure we don't modify them anyway.
+        assert_eq!(ContactId::UNDEFINED.to_u32(), 0);
+        assert_eq!(ContactId::SELF.to_u32(), 1);
+        assert_eq!(ContactId::INFO.to_u32(), 2);
+        assert_eq!(ContactId::DEVICE.to_u32(), 5);
+        assert_eq!(ContactId::LAST_SPECIAL.to_u32(), 9);
+    }
+
     #[test]
     fn test_may_be_valid_addr() {
         assert_eq!(may_be_valid_addr(""), false);
@@ -1463,6 +1515,8 @@ mod tests {
     async fn test_get_contacts() -> Result<()> {
         let context = TestContext::new().await;
 
+        assert!(context.get_all_self_addrs().await?.is_empty());
+
         // Bob is not in the contacts yet.
         let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).await?;
         assert_eq!(contacts.len(), 0);
@@ -1474,7 +1528,7 @@ mod tests {
             Origin::IncomingReplyTo,
         )
         .await?;
-        assert_ne!(id, 0);
+        assert_ne!(id, ContactId::UNDEFINED);
 
         let contact = Contact::load_from_db(&context.ctx, id).await.unwrap();
         assert_eq!(contact.get_name(), "");
@@ -1552,7 +1606,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bla foo", "one@eins.org", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_id(), contact_id);
@@ -1579,7 +1633,7 @@ mod tests {
             Contact::add_or_lookup(&t, "", "three@drei.sam", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_name(), "");
@@ -1619,7 +1673,7 @@ mod tests {
             Contact::add_or_lookup(&t, "", "alice@w.de", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_name(), "Wonderland, Alice");
@@ -1628,8 +1682,7 @@ mod tests {
         assert_eq!(contact.get_name_n_addr(), "Wonderland, Alice (alice@w.de)");
 
         // check SELF
-        let contact = Contact::load_from_db(&t, DC_CONTACT_ID_SELF).await.unwrap();
-        assert_eq!(DC_CONTACT_ID_SELF, 1);
+        let contact = Contact::load_from_db(&t, ContactId::SELF).await.unwrap();
         assert_eq!(contact.get_name(), stock_str::self_msg(&t).await);
         assert_eq!(contact.get_addr(), ""); // we're not configured
         assert!(!contact.is_blocked());
@@ -1639,7 +1692,7 @@ mod tests {
     async fn test_delete() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
-        assert!(Contact::delete(&alice, DC_CONTACT_ID_SELF).await.is_err());
+        assert!(Contact::delete(&alice, ContactId::SELF).await.is_err());
 
         // Create Bob contact
         let (contact_id, _) =
@@ -1672,7 +1725,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob1", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::Created);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob1");
@@ -1684,7 +1737,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob2", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob2");
@@ -1695,7 +1748,7 @@ mod tests {
         let contact_id = Contact::create(&t, "bob3", "bob@example.org")
             .await
             .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob2");
         assert_eq!(contact.get_name(), "bob3");
@@ -1706,7 +1759,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob4", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob4");
@@ -1720,7 +1773,7 @@ mod tests {
 
         // manually create "claire@example.org" without a given name
         let contact_id = Contact::create(&t, "", "claire@example.org").await.unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "");
         assert_eq!(contact.get_name(), "");
@@ -1894,7 +1947,7 @@ mod tests {
         let id = Contact::lookup_id_by_addr(&alice.ctx, "alice@example.org", Origin::Unknown)
             .await
             .unwrap();
-        assert_eq!(id, Some(DC_CONTACT_ID_SELF));
+        assert_eq!(id, Some(ContactId::SELF));
     }
 
     #[async_std::test]
@@ -1902,9 +1955,9 @@ mod tests {
         let alice = TestContext::new_alice().await;
 
         // Return error for special IDs
-        let encrinfo = Contact::get_encrinfo(&alice, DC_CONTACT_ID_SELF).await;
+        let encrinfo = Contact::get_encrinfo(&alice, ContactId::SELF).await;
         assert!(encrinfo.is_err());
-        let encrinfo = Contact::get_encrinfo(&alice, DC_CONTACT_ID_DEVICE).await;
+        let encrinfo = Contact::get_encrinfo(&alice, ContactId::DEVICE).await;
         assert!(encrinfo.is_err());
 
         let (contact_bob_id, _modified) =
@@ -2085,7 +2138,7 @@ Chat-Version: 1.0
 Date: Sun, 22 Mar 2020 22:37:55 +0000
 
 Hi."#;
-        dc_receive_imf(&alice, mime, "Inbox", false).await?;
+        dc_receive_imf(&alice, mime, false).await?;
         let msg = alice.get_last_msg().await;
 
         let timestamp = msg.get_timestamp();

src/context.rs

@@ -10,7 +10,6 @@ use async_std::{
     channel::{self, Receiver, Sender},
     path::{Path, PathBuf},
     sync::{Arc, Mutex, RwLock},
-    task,
 };
 
 use crate::chat::{get_chat_cnt, ChatId};
@@ -24,7 +23,6 @@ use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
 use crate::quota::QuotaInfo;
 use crate::scheduler::Scheduler;
-use crate::securejoin::Bob;
 use crate::sql::Sql;
 
 #[derive(Clone, Debug)]
@@ -45,7 +43,6 @@ pub struct InnerContext {
     /// Blob directory path
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
-    pub(crate) bob: Bob,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -58,7 +55,6 @@ pub struct InnerContext {
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
-    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
     /// Recently loaded quota information, if any.
     /// Set to `None` if quota was never tried to load.
@@ -171,7 +167,6 @@ impl Context {
             blobdir,
             running_state: RwLock::new(Default::default()),
             sql: Sql::new(dbfile),
-            bob: Default::default(),
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
@@ -179,7 +174,6 @@ impl Context {
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
-            ephemeral_task: RwLock::new(None),
             quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
@@ -195,17 +189,14 @@ impl Context {
 
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
-        info!(self, "starting IO");
-        if self.inner.is_io_running().await {
-            info!(self, "IO is already running");
+        if let Ok(false) = self.is_configured().await {
+            warn!(self, "can not start io on a context that is not configured");
             return;
         }
 
-        {
-            let l = &mut *self.inner.scheduler.write().await;
-            if let Err(err) = l.start(self.clone()).await {
-                error!(self, "Failed to start IO: {}", err)
-            }
+        info!(self, "starting IO");
+        if let Err(err) = self.inner.scheduler.write().await.start(self.clone()).await {
+            error!(self, "Failed to start IO: {}", err)
         }
     }
 
@@ -213,7 +204,9 @@ impl Context {
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
 
-        self.inner.stop_io().await;
+        if let Err(err) = self.inner.stop_io().await {
+            warn!(self, "failed to stop IO: {}", err);
+        }
     }
 
     /// Returns a reference to the underlying SQL instance.
@@ -242,6 +235,24 @@ impl Context {
         });
     }
 
+    /// Emits a generic MsgsChanged event (without chat or message id)
+    pub fn emit_msgs_changed_without_ids(&self) {
+        self.emit_event(EventType::MsgsChanged {
+            chat_id: ChatId::new(0),
+            msg_id: MsgId::new(0),
+        });
+    }
+
+    /// Emits a MsgsChanged event with specified chat and message ids
+    pub fn emit_msgs_changed(&self, chat_id: ChatId, msg_id: MsgId) {
+        self.emit_event(EventType::MsgsChanged { chat_id, msg_id });
+    }
+
+    /// Emits an IncomingMsg event with specified chat and message ids
+    pub fn emit_incoming_msg(&self, chat_id: ChatId, msg_id: MsgId) {
+        self.emit_event(EventType::IncomingMsg { chat_id, msg_id });
+    }
+
     /// Returns a receiver for emitted events.
     ///
     /// Multiple emitters can be created, but note that in this case each emitted event will
@@ -317,8 +328,9 @@ impl Context {
 
     pub async fn get_info(&self) -> Result<BTreeMap<&'static str, String>> {
         let unset = "0";
-        let l = LoginParam::from_database(self, "").await?;
-        let l2 = LoginParam::from_database(self, "configured_").await?;
+        let l = LoginParam::load_candidate_params(self).await?;
+        let l2 = LoginParam::load_configured_params(self).await?;
+        let secondary_addrs = self.get_secondary_self_addrs().await?.join(", ");
         let displayname = self.get_config(Config::Displayname).await?;
         let chats = get_chat_cnt(self).await? as usize;
         let unblocked_msgs = message::get_unblocked_msg_cnt(self).await as usize;
@@ -357,7 +369,7 @@ impl Context {
 
         let sentbox_watch = self.get_config_int(Config::SentboxWatch).await?;
         let mvbox_move = self.get_config_int(Config::MvboxMove).await?;
-        let sentbox_move = self.get_config_int(Config::SentboxMove).await?;
+        let only_fetch_mvbox = self.get_config_int(Config::OnlyFetchMvbox).await?;
         let folders_configured = self
             .sql
             .get_raw_config_int("folders_configured")
@@ -403,6 +415,7 @@ impl Context {
         res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
+        res.insert("secondary_addrs", secondary_addrs);
         res.insert(
             "fetch_existing_msgs",
             self.get_config_int(Config::FetchExistingMsgs)
@@ -421,7 +434,7 @@ impl Context {
         );
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
-        res.insert("sentbox_move", sentbox_move.to_string());
+        res.insert("only_fetch_mvbox", only_fetch_mvbox.to_string());
         res.insert("folders_configured", folders_configured.to_string());
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
@@ -609,11 +622,6 @@ impl Context {
         Ok(mvbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_spam_folder(&self, folder_name: &str) -> Result<bool> {
-        let spam = self.get_config(Config::ConfiguredSpamFolder).await?;
-        Ok(spam.as_deref() == Some(folder_name))
-    }
-
     pub(crate) fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -630,25 +638,9 @@ impl Context {
 }
 
 impl InnerContext {
-    async fn is_io_running(&self) -> bool {
-        self.scheduler.read().await.is_running()
-    }
-
-    async fn stop_io(&self) {
-        if self.is_io_running().await {
-            let token = {
-                let lock = &*self.scheduler.read().await;
-                lock.pre_stop().await
-            };
-            {
-                let lock = &mut *self.scheduler.write().await;
-                lock.stop(token).await;
-            }
-        }
-
-        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
-            ephemeral_task.cancel().await;
-        }
+    async fn stop_io(&self) -> Result<()> {
+        self.scheduler.write().await.stop().await?;
+        Ok(())
     }
 }
 
@@ -673,10 +665,10 @@ mod tests {
     use crate::chat::{
         get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
     };
-    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+    use crate::contact::ContactId;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::dc_tools::dc_create_outgoing_rfc724_mid;
-    use crate::message::Message;
+    use crate::message::{Message, Viewtype};
     use crate::test_utils::TestContext;
     use anyhow::Context as _;
     use std::time::Duration;
@@ -719,9 +711,7 @@ mod tests {
             dc_create_outgoing_rfc724_mid(None, contact.get_addr())
         );
         println!("{}", msg);
-        dc_receive_imf(t, msg.as_bytes(), "INBOX", false)
-            .await
-            .unwrap();
+        dc_receive_imf(t, msg.as_bytes(), false).await.unwrap();
     }
 
     #[async_std::test]
@@ -959,7 +949,7 @@ mod tests {
     #[async_std::test]
     async fn test_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
-        let self_talk = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
+        let self_talk = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
         let chat = alice
             .create_chat_with_contact("Bob", "bob@example.org")
             .await;

src/dc_tools.rs

@@ -21,10 +21,10 @@ use mailparse::MailHeaderMap;
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
-use crate::constants::{Viewtype, DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
+use crate::constants::{DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::events::EventType;
-use crate::message::Message;
+use crate::message::{Message, Viewtype};
 use crate::provider::get_provider_update_timestamp;
 use crate::stock_str;
 
@@ -71,7 +71,7 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
     let lt = Local::now();
-    lt.offset().local_minus_utc() as i64
+    i64::from(lt.offset().local_minus_utc())
 }
 
 // timesmearing
@@ -86,7 +86,7 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
 //   `last_smeared_timestamp` is again in sync with the normal time.
 // - however, we do not do all this for the far future,
 //   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
-const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
+pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 
 /// Returns the current smeared timestamp,
 ///
@@ -214,7 +214,10 @@ pub(crate) fn dc_create_id() -> String {
     rng.fill(&mut arr[..]);
 
     // Take 11 base64 characters containing 66 random bits.
-    base64::encode(&arr).chars().take(11).collect()
+    base64::encode_config(&arr, base64::URL_SAFE)
+        .chars()
+        .take(11)
+        .collect()
 }
 
 /// Function generates a Message-ID that can be used for a new outgoing message.
@@ -534,8 +537,8 @@ impl rusqlite::types::ToSql for EmailAddress {
 /// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
 pub(crate) fn improve_single_line_input(input: &str) -> String {
     input
-        .replace("\n", " ")
-        .replace("\r", " ")
+        .replace('\n', " ")
+        .replace('\r', " ")
         .trim()
         .to_string()
 }
@@ -700,7 +703,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
     async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
-        dc_receive_imf(&t, raw, "INBOX", false).await.unwrap();
+        dc_receive_imf(&t, raw, false).await.unwrap();
         let msg = t.get_last_msg().await;
         let msg_info = get_msg_info(&t, msg.id).await.unwrap();
 
@@ -762,6 +765,15 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         assert_eq!(buf.len(), 11);
     }
 
+    #[test]
+    fn test_dc_create_id_invalid_chars() {
+        for _ in 1..1000 {
+            let buf = dc_create_id();
+            assert!(!buf.contains('/')); // `/` must not be used to be URL-safe
+            assert!(!buf.contains('.')); // `.` is used as a delimiter when extracting grpid from Message-ID
+        }
+    }
+
     #[test]
     fn test_dc_extract_grpid_from_rfc724_mid() {
         // Should return None if we pass invalid mid

src/download.rs

@@ -3,14 +3,14 @@
 use anyhow::{anyhow, Result};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
 
 use crate::config::Config;
-use crate::constants::Viewtype;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::imap::{Imap, ImapActionResult};
 use crate::job::{self, Action, Job, Status};
-use crate::message::{Message, MsgId};
+use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
 use crate::param::Params;
 use crate::{job_try, stock_str, EventType};
@@ -146,7 +146,7 @@ impl Job {
 
         if let Some((server_uid, server_folder)) = row {
             match imap
-                .fetch_single_msg(context, &server_folder, server_uid)
+                .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone())
                 .await
             {
                 ImapActionResult::RetryLater | ImapActionResult::Failed => {
@@ -185,6 +185,7 @@ impl Imap {
         context: &Context,
         folder: &str,
         uid: u32,
+        rfc724_mid: String,
     ) -> ImapActionResult {
         if let Some(imapresult) = self
             .prepare_imap_operation_on_msg(context, folder, uid)
@@ -196,8 +197,10 @@ impl Imap {
         // we are connected, and the folder is selected
         info!(context, "Downloading message {}/{} fully...", folder, uid);
 
+        let mut uid_message_ids: BTreeMap<u32, String> = BTreeMap::new();
+        uid_message_ids.insert(uid, rfc724_mid);
         let (last_uid, _received) = match self
-            .fetch_many_msgs(context, folder, vec![uid], false, false)
+            .fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, false, false)
             .await
         {
             Ok(res) => res,
@@ -251,13 +254,15 @@ impl MimeMessage {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use num_traits::FromPrimitive;
+
     use crate::chat::send_msg;
-    use crate::constants::Viewtype;
     use crate::dc_receive_imf::dc_receive_imf_inner;
     use crate::ephemeral::Timer;
+    use crate::message::Viewtype;
     use crate::test_utils::TestContext;
-    use num_traits::FromPrimitive;
+
+    use super::*;
 
     #[test]
     fn test_downloadstate_values() {
@@ -337,7 +342,15 @@ mod tests {
              Date: Sun, 22 Mar 2020 22:37:57 +0000\
              Content-Type: text/plain";
 
-        dc_receive_imf_inner(&t, header.as_bytes(), "INBOX", false, Some(100000), false).await?;
+        dc_receive_imf_inner(
+            &t,
+            "Mr.12345678901@example.com",
+            header.as_bytes(),
+            false,
+            Some(100000),
+            false,
+        )
+        .await?;
         let msg = t.get_last_msg().await;
         assert_eq!(msg.download_state(), DownloadState::Available);
         assert_eq!(msg.get_subject(), "foo");
@@ -348,8 +361,8 @@ mod tests {
 
         dc_receive_imf_inner(
             &t,
+            "Mr.12345678901@example.com",
             format!("{}\n\n100k text...", header).as_bytes(),
-            "INBOX",
             false,
             None,
             false,
@@ -377,6 +390,7 @@ mod tests {
         // download message from bob partially, this must not change the ephemeral timer
         dc_receive_imf_inner(
             &t,
+            "first@example.org",
             b"From: Bob <bob@example.org>\n\
                     To: Alice <alice@example.org>\n\
                     Chat-Version: 1.0\n\
@@ -384,7 +398,6 @@ mod tests {
                     Message-ID: <first@example.org>\n\
                     Date: Sun, 14 Nov 2021 00:10:00 +0000\
                     Content-Type: text/plain",
-            "INBOX",
             false,
             Some(100000),
             false,

src/e2ee.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashSet;
 
-use anyhow::{bail, format_err, Context as _, Result};
+use anyhow::{format_err, Context as _, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
@@ -28,13 +28,7 @@ impl EncryptHelper {
         let prefer_encrypt =
             EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await?)
                 .unwrap_or_default();
-        let addr = match context.get_config(Config::ConfiguredAddr).await? {
-            None => {
-                bail!("addr not configured!");
-            }
-            Some(addr) => addr,
-        };
-
+        let addr = context.get_primary_self_addr().await?;
         let public_key = SignedPublicKey::load_self(context).await?;
 
         Ok(EncryptHelper {
@@ -387,28 +381,21 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 /// [Config::ConfiguredAddr] is configured, this address is returned.
 // TODO, remove this once deltachat::key::Key no longer exists.
 pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await?
-        .context(concat!(
-            "Failed to get self address, ",
-            "cannot ensure secret key if not configured."
-        ))?;
+    let self_addr = context.get_primary_self_addr().await?;
     SignedPublicKey::load_self(context).await?;
     Ok(self_addr)
 }
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-
     use crate::chat;
-    use crate::constants::Viewtype;
-    use crate::message::Message;
+    use crate::message::{Message, Viewtype};
     use crate::param::Param;
     use crate::peerstate::ToSave;
     use crate::test_utils::{bob_keypair, TestContext};
 
+    use super::*;
+
     mod ensure_secret_key_exists {
         use super::*;
 

src/ephemeral.rs

@@ -48,9 +48,9 @@
 //!
 //! ## When messages are deleted
 //!
-//! Local deletion happens when the chatlist or chat is loaded. A
-//! `MsgsChanged` event is emitted when a message deletion is due, to
-//! make UI reload displayed messages and cause actual deletion.
+//! The `ephemeral_loop` task schedules the next due running of
+//! `delete_expired_messages` which in turn emits `MsgsChanged` events
+//! when deleting local messages to make UIs reload displayed messages.
 //!
 //! Server deletion happens by updating the `imap` table based on
 //! the database entries which are expired either according to their
@@ -62,19 +62,21 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{ensure, Context as _, Result};
-use async_std::task;
+use async_std::channel::Receiver;
+use async_std::future::timeout;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{send_msg, ChatId};
-use crate::constants::{
-    Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
-};
+use crate::constants::{DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH};
+use crate::contact::ContactId;
 use crate::context::Context;
-use crate::dc_tools::time;
+use crate::dc_tools::{duration_to_str, time};
 use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
-use crate::message::{Message, MessageState, MsgId};
+use crate::log::LogExt;
+use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
+use crate::sql;
 use crate::stock_str;
 use std::cmp::max;
 
@@ -196,7 +198,7 @@ impl ChatId {
         }
         self.inner_set_ephemeral_timer(context, timer).await?;
         let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, ContactId::SELF).await);
         msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
         if let Err(err) = send_msg(context, self, &mut msg).await {
             error!(
@@ -212,7 +214,7 @@ impl ChatId {
 pub(crate) async fn stock_ephemeral_timer_changed(
     context: &Context,
     timer: Timer,
-    from_id: u32,
+    from_id: ContactId,
 ) -> String {
     match timer {
         Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
@@ -291,12 +293,43 @@ impl MsgId {
                     paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
                 )
                 .await?;
-            schedule_ephemeral_task(context).await;
+            context.interrupt_ephemeral_task().await;
         }
         Ok(())
     }
 }
 
+pub(crate) async fn start_ephemeral_timers_msgids(
+    context: &Context,
+    msg_ids: &[MsgId],
+) -> Result<()> {
+    let msg_ids: Vec<&dyn crate::ToSql> = msg_ids
+        .iter()
+        .map(|msg_id| msg_id as &dyn crate::ToSql)
+        .collect();
+    let now = time();
+    let count = context
+        .sql
+        .execute(
+            format!(
+                "UPDATE msgs SET ephemeral_timestamp = ? + ephemeral_timer
+         WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ? + ephemeral_timer) AND ephemeral_timer > 0
+         AND id IN ({})",
+                sql::repeat_vars(msg_ids.len())
+            ),
+            rusqlite::params_from_iter(
+                std::iter::once(&now as &dyn crate::ToSql)
+                    .chain(std::iter::once(&now as &dyn crate::ToSql))
+                    .chain(msg_ids),
+            ),
+        )
+        .await?;
+    if count > 0 {
+        context.interrupt_ephemeral_task().await;
+    }
+    Ok(())
+}
+
 /// Deletes messages which are expired according to
 /// `delete_device_after` setting or `ephemeral_timestamp` column.
 ///
@@ -305,7 +338,7 @@ impl MsgId {
 /// false. This function does not emit the MsgsChanged event itself,
 /// because it is also called when chatlist is reloaded, and emitting
 /// MsgsChanged there will cause infinite reload loop.
-pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool> {
+pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Result<()> {
     let mut updated = context
         .sql
         .execute(
@@ -321,21 +354,21 @@ WHERE
   AND ephemeral_timestamp <= ?
   AND chat_id != ?
 "#,
-            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
+            paramsv![DC_CHAT_ID_TRASH, now, DC_CHAT_ID_TRASH],
         )
         .await
         .context("update failed")?
         > 0;
 
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
             .await?
             .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
             .await?
             .unwrap_or_default();
 
-        let threshold_timestamp = time() - delete_device_after;
+        let threshold_timestamp = now.saturating_sub(delete_device_after);
 
         // Delete expired messages
         //
@@ -345,7 +378,8 @@ WHERE
             .sql
             .execute(
                 "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
+             SET chat_id = ?, txt = '', subject='', txt_raw='', \
+                 mime_headers='', from_id=0, to_id=0, param='' \
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
@@ -364,72 +398,116 @@ WHERE
         updated |= rows_modified > 0;
     }
 
-    schedule_ephemeral_task(context).await;
-    Ok(updated)
+    if updated {
+        context.emit_msgs_changed_without_ids();
+    }
+
+    Ok(())
 }
 
-/// Schedule a task to emit MsgsChanged event when the next local
-/// deletion happens. Existing task is cancelled to make sure at most
-/// one such task is scheduled at a time.
+/// Calculates the next timestamp when a message will be deleted due to
+/// `delete_device_after` setting being set.
+async fn next_delete_device_after_timestamp(context: &Context) -> Result<Option<i64>> {
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
+        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+            .await?
+            .unwrap_or_default();
+        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
+            .await?
+            .unwrap_or_default();
+
+        let oldest_message_timestamp: Option<i64> = context
+            .sql
+            .query_get_value(
+                r#"
+                SELECT min(timestamp)
+                FROM msgs
+                WHERE chat_id > ?
+                  AND chat_id != ?
+                  AND chat_id != ?;
+                "#,
+                paramsv![DC_CHAT_ID_TRASH, self_chat_id, device_chat_id],
+            )
+            .await?;
+
+        Ok(oldest_message_timestamp.map(|x| x.saturating_add(delete_device_after)))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Calculates next timestamp when expiration of some message will happen.
 ///
-/// UI is expected to reload the chatlist or the chat in response to
-/// MsgsChanged event, this will trigger actual deletion.
-///
-/// This takes into account only per-chat timeouts, because global device
-/// timeouts are at least one hour long and deletion is triggered often enough
-/// by user actions.
-pub async fn schedule_ephemeral_task(context: &Context) {
+/// Expiration can happen either because user has set `delete_device_after` setting or because the
+/// message itself has an ephemeral timer.
+async fn next_expiration_timestamp(context: &Context) -> Option<i64> {
     let ephemeral_timestamp: Option<i64> = match context
         .sql
         .query_get_value(
             r#"
-    SELECT ephemeral_timestamp
-    FROM msgs
-    WHERE ephemeral_timestamp != 0
-      AND chat_id != ?
-    ORDER BY ephemeral_timestamp ASC
-    LIMIT 1;
-    "#,
+            SELECT min(ephemeral_timestamp)
+            FROM msgs
+            WHERE ephemeral_timestamp != 0
+              AND chat_id != ?;
+            "#,
             paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
         )
         .await
     {
         Err(err) => {
             warn!(context, "Can't calculate next ephemeral timeout: {}", err);
-            return;
+            None
         }
         Ok(ephemeral_timestamp) => ephemeral_timestamp,
     };
 
-    // Cancel existing task, if any
-    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
-        ephemeral_task.cancel().await;
-    }
+    let delete_device_after_timestamp: Option<i64> =
+        match next_delete_device_after_timestamp(context).await {
+            Err(err) => {
+                warn!(
+                    context,
+                    "Can't calculate timestamp of the next message expiration: {}", err
+                );
+                None
+            }
+            Ok(timestamp) => timestamp,
+        };
+
+    ephemeral_timestamp
+        .into_iter()
+        .chain(delete_device_after_timestamp.into_iter())
+        .min()
+}
+
+pub(crate) async fn ephemeral_loop(context: &Context, interrupt_receiver: Receiver<()>) {
+    loop {
+        let ephemeral_timestamp = next_expiration_timestamp(context).await;
 
-    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
         let now = SystemTime::now();
-        let until = UNIX_EPOCH
-            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
-            + Duration::from_secs(1);
+        let until = if let Some(ephemeral_timestamp) = ephemeral_timestamp {
+            UNIX_EPOCH
+                + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+                + Duration::from_secs(1)
+        } else {
+            // no messages to be deleted for now, wait long for one to occur
+            now + Duration::from_secs(86400)
+        };
 
         if let Ok(duration) = until.duration_since(now) {
-            // Schedule a task, ephemeral_timestamp is in the future
-            let context1 = context.clone();
-            let ephemeral_task = task::spawn(async move {
-                async_std::task::sleep(duration).await;
-                context1.emit_event(EventType::MsgsChanged {
-                    chat_id: ChatId::new(0),
-                    msg_id: MsgId::new(0),
-                });
-            });
-            *context.ephemeral_task.write().await = Some(ephemeral_task);
-        } else {
-            // Emit event immediately
-            context.emit_event(EventType::MsgsChanged {
-                chat_id: ChatId::new(0),
-                msg_id: MsgId::new(0),
-            });
+            info!(
+                context,
+                "Ephemeral loop waiting for deletion in {} or interrupt",
+                duration_to_str(duration)
+            );
+            if timeout(duration, interrupt_receiver.recv()).await.is_ok() {
+                // received an interruption signal, recompute waiting time (if any)
+                continue;
+            }
         }
+
+        delete_expired_messages(context, time())
+            .await
+            .ok_or_log(context);
     }
 }
 
@@ -451,12 +529,11 @@ pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<()
         .execute(
             "UPDATE imap
              SET target=''
-             WHERE EXISTS (
-               SELECT * FROM msgs
-               WHERE rfc724_mid=imap.rfc724_mid
-               AND ((download_state = 0 AND timestamp < ?) OR
-                    (download_state != 0 AND timestamp < ?) OR
-                    (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?))
+             WHERE rfc724_mid IN (
+               SELECT rfc724_mid FROM msgs
+               WHERE ((download_state = 0 AND timestamp < ?) OR
+                      (download_state != 0 AND timestamp < ?) OR
+                      (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?))
              )",
             paramsv![threshold_timestamp, threshold_timestamp_extended, now],
         )
@@ -500,6 +577,7 @@ mod tests {
     use super::*;
     use crate::config::Config;
     use crate::dc_receive_imf::dc_receive_imf;
+    use crate::dc_tools::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
@@ -512,7 +590,7 @@ mod tests {
         let context = TestContext::new().await;
 
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, ContactId::SELF).await,
             "Message deletion timer is disabled by me."
         );
 
@@ -520,7 +598,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 1 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 s by me."
@@ -529,7 +607,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 30 s by me."
@@ -538,7 +616,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 minute by me."
@@ -547,7 +625,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 90 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1.5 minutes by me."
@@ -556,7 +634,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 * 60 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 30 minutes by me."
@@ -565,7 +643,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 * 60 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 hour by me."
@@ -574,7 +652,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 5400 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1.5 hours by me."
@@ -585,7 +663,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 2 hours by me."
@@ -596,7 +674,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 day by me."
@@ -607,7 +685,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 2 days by me."
@@ -618,7 +696,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 week by me."
@@ -629,7 +707,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 4 weeks by me."
@@ -788,31 +866,106 @@ mod tests {
     #[async_std::test]
     async fn test_ephemeral_delete_msgs() -> Result<()> {
         let t = TestContext::new_alice().await;
-        let chat = t.get_self_chat().await;
+        let self_chat = t.get_self_chat().await;
 
-        t.send_text(chat.id, "Saved message, which we delete manually")
+        assert_eq!(next_expiration_timestamp(&t).await, None);
+
+        t.send_text(self_chat.id, "Saved message, which we delete manually")
             .await;
-        let msg = t.get_last_msg_in(chat.id).await;
+        let msg = t.get_last_msg_in(self_chat.id).await;
         msg.id.delete_from_db(&t).await?;
-        check_msg_was_deleted(&t, &chat, msg.id).await;
+        check_msg_is_deleted(&t, &self_chat, msg.id).await;
 
-        chat.id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
+        self_chat
+            .id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 3600 })
             .await
             .unwrap();
-        let msg = t
-            .send_text(chat.id, "Saved message, disappearing after 1s")
-            .await;
 
-        async_std::task::sleep(Duration::from_millis(1100)).await;
+        // Send a saved message which will be deleted after 3600s
+        let now = time();
+        let msg = t.send_text(self_chat.id, "Message text").await;
 
-        // Check that the msg was deleted locally.
-        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
+        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3599, time() + 3601)
+            .await
+            .unwrap();
+
+        // Set DeleteDeviceAfter to 1800s. Thend send a saved message which will
+        // still be deleted after 3600s because DeleteDeviceAfter doesn't apply to saved messages.
+        t.set_config(Config::DeleteDeviceAfter, Some("1800"))
+            .await?;
+
+        let now = time();
+        let msg = t.send_text(self_chat.id, "Message text").await;
+
+        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3559, time() + 3601)
+            .await
+            .unwrap();
+
+        // Send a message to Bob which will be deleted after 1800s because of DeleteDeviceAfter.
+        let bob_chat = t.create_chat_with_contact("", "bob@example.net").await;
+        let now = time();
+        let msg = t.send_text(bob_chat.id, "Message text").await;
+
+        check_msg_will_be_deleted(
+            &t,
+            msg.sender_msg_id,
+            &bob_chat,
+            now + 1799,
+            // The message may appear to be sent MAX_SECONDS_TO_LEND_FROM_FUTURE later and
+            // therefore be deleted MAX_SECONDS_TO_LEND_FROM_FUTURE later.
+            time() + 1801 + MAX_SECONDS_TO_LEND_FROM_FUTURE,
+        )
+        .await
+        .unwrap();
+
+        // Enable ephemeral messages with Bob -> message will be deleted after 60s.
+        // This tests that the message is deleted at min(ephemeral deletion time, DeleteDeviceAfter deletion time).
+        bob_chat
+            .id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
+            .await?;
+
+        let now = time();
+        let msg = t.send_text(bob_chat.id, "Message text").await;
+
+        check_msg_will_be_deleted(&t, msg.sender_msg_id, &bob_chat, now + 59, time() + 61)
+            .await
+            .unwrap();
 
         Ok(())
     }
 
-    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+    async fn check_msg_will_be_deleted(
+        t: &TestContext,
+        msg_id: MsgId,
+        chat: &Chat,
+        not_deleted_at: i64,
+        deleted_at: i64,
+    ) -> Result<()> {
+        let next_expiration = next_expiration_timestamp(t).await.unwrap();
+
+        assert!(next_expiration > not_deleted_at);
+        delete_expired_messages(t, not_deleted_at).await?;
+
+        let loaded = Message::load_from_db(t, msg_id).await?;
+        assert_eq!(loaded.text.unwrap(), "Message text");
+        assert_eq!(loaded.chat_id, chat.id);
+
+        assert!(next_expiration < deleted_at);
+        delete_expired_messages(t, deleted_at).await?;
+
+        let loaded = Message::load_from_db(t, msg_id).await?;
+        assert_eq!(loaded.text.unwrap(), "");
+        assert_eq!(loaded.chat_id, DC_CHAT_ID_TRASH);
+
+        // Check that the msg was deleted locally.
+        check_msg_is_deleted(t, chat, msg_id).await;
+
+        Ok(())
+    }
+
+    async fn check_msg_is_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
         let chat_items = chat::get_chat_msgs(t, chat.id, 0, None).await.unwrap();
         // Check that the chat is empty except for possibly info messages:
         for item in &chat_items {
@@ -824,8 +977,8 @@ mod tests {
 
         // Check that if there is a message left, the text and metadata are gone
         if let Ok(msg) = Message::load_from_db(t, msg_id).await {
-            assert_eq!(msg.from_id, 0);
-            assert_eq!(msg.to_id, 0);
+            assert_eq!(msg.from_id, ContactId::UNDEFINED);
+            assert_eq!(msg.to_id, ContactId::UNDEFINED);
             assert!(msg.text.is_none_or_empty(), "{:?}", msg.text);
             let rawtxt: Option<String> = t
                 .sql
@@ -962,7 +1115,6 @@ mod tests {
                     Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
                     \n\
                     hello\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -983,7 +1135,6 @@ mod tests {
                     Ephemeral-Timer: 60\n\
                     \n\
                     second message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -1020,7 +1171,6 @@ mod tests {
                     In-Reply-To: <first@example.com>\n\
                     \n\
                     > hello\n",
-            "INBOX",
             false,
         )
         .await?;

src/events.rs

@@ -7,9 +7,10 @@ use async_std::path::PathBuf;
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
+use crate::contact::ContactId;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
-use crate::webxdc::StatusUpdateId;
+use crate::webxdc::StatusUpdateSerial;
 
 #[derive(Debug)]
 pub struct Events {
@@ -253,7 +254,7 @@ pub enum EventType {
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
     #[strum(props(id = "2030"))]
-    ContactsChanged(Option<u32>),
+    ContactsChanged(Option<ContactId>),
 
     /// Location of one or more contact has changed.
     ///
@@ -261,7 +262,7 @@ pub enum EventType {
     ///     If the locations of several contacts have been changed,
     ///     eg. after calling dc_delete_all_locations(), this parameter is set to `None`.
     #[strum(props(id = "2035"))]
-    LocationChanged(Option<u32>),
+    LocationChanged(Option<ContactId>),
 
     /// Inform about the configuration progress started by configure().
     #[strum(props(id = "2041"))]
@@ -305,7 +306,10 @@ pub enum EventType {
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
     #[strum(props(id = "2060"))]
-    SecurejoinInviterProgress { contact_id: u32, progress: usize },
+    SecurejoinInviterProgress {
+        contact_id: ContactId,
+        progress: usize,
+    },
 
     /// Progress information of a secure-join handshake from the view of the joiner
     /// (Bob, the person who scans the QR code).
@@ -316,7 +320,10 @@ pub enum EventType {
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[strum(props(id = "2061"))]
-    SecurejoinJoinerProgress { contact_id: u32, progress: usize },
+    SecurejoinJoinerProgress {
+        contact_id: ContactId,
+        progress: usize,
+    },
 
     /// The connectivity to the server changed.
     /// This means that you should refresh the connectivity view
@@ -331,6 +338,6 @@ pub enum EventType {
     #[strum(props(id = "2120"))]
     WebxdcStatusUpdate {
         msg_id: MsgId,
-        status_update_id: StatusUpdateId,
+        status_update_serial: StatusUpdateSerial,
     },
 }

src/html.rs

@@ -279,9 +279,9 @@ mod tests {
     use crate::chat;
     use crate::chat::forward_msgs;
     use crate::config::Config;
-    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+    use crate::contact::ContactId;
     use crate::dc_receive_imf::dc_receive_imf;
-    use crate::message::MessengerMessage;
+    use crate::message::{MessengerMessage, Viewtype};
     use crate::test_utils::TestContext;
 
     #[async_std::test]
@@ -365,7 +365,7 @@ test some special html-characters as < > and & but also " and &#x
         // however, rust multiline-strings use just `\n`;
         // therefore, we just remove `\r` before comparison.
         assert_eq!(
-            parser.html.replace("\r", ""),
+            parser.html.replace('\r', ""),
             r##"
 <html>
   <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
@@ -379,7 +379,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         let raw = include_bytes!("../test-data/message/text_alt_html.eml");
         let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
-            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
             r##"<html>
   <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
 </html>
@@ -394,7 +394,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
         let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
-            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
             r##"<html>
   <p>
     this is <b>html</b>
@@ -440,9 +440,9 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
+        dc_receive_imf(&alice, raw, false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_ne!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
         assert!(!msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -456,7 +456,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .await
             .unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -469,7 +469,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         let chat = bob.create_chat_with_contact("", "alice@example.org").await;
         bob.recv_msg(&alice.pop_sent_msg().await).await;
         let msg = bob.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_ne!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -489,7 +489,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
+        dc_receive_imf(&alice, raw, false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
 
         // forward the message to saved-messages,
@@ -506,7 +506,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         alice.recv_msg(&msg).await;
         let chat = alice.get_self_chat().await;
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.get_showpadlock());
         assert!(msg.is_forwarded());
@@ -555,7 +555,6 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/cp1252-html.eml"),
-            "INBOX",
             false,
         )
         .await?;

src/imap/idle.rs

@@ -3,7 +3,6 @@ use super::Imap;
 use anyhow::{bail, Context as _, Result};
 use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
-use imap_proto::types::{AttributeValue, Response};
 use std::time::{Duration, SystemTime};
 
 use crate::{context::Context, scheduler::InterruptInfo};
@@ -72,13 +71,6 @@ impl Imap {
             match fut.await {
                 Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
                     info!(context, "Idle has NewData {:?}", x);
-                    if let Response::Fetch(_message, attrs) = x.parsed() {
-                        for attr in attrs {
-                            if let AttributeValue::ModSeq(modseq) = attr {
-                                self.update_modseq(*modseq);
-                            }
-                        }
-                    }
                 }
                 Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
                     info!(context, "Idle-wait timeout or interruption");
@@ -165,8 +157,10 @@ impl Imap {
                     // in anything.  If so, we behave as if IDLE had data but
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
-
-                    match self.fetch_new_messages(context, &watch_folder, false).await {
+                    match self
+                        .fetch_new_messages(context, &watch_folder, false, false)
+                        .await
+                    {
                         Ok(res) => {
                             info!(context, "fetch_new_messages returned {:?}", res);
                             if res {

src/imap/scan_folders.rs

@@ -2,15 +2,18 @@ use std::{collections::BTreeMap, time::Instant};
 
 use anyhow::{Context as _, Result};
 
+use crate::config::Config;
 use crate::imap::Imap;
-use crate::{config::Config, log::LogExt};
+use crate::log::LogExt;
 use crate::{context::Context, imap::FolderMeaning};
-use async_std::prelude::*;
+
+use async_std::stream::StreamExt;
 
 use super::{get_folder_meaning, get_folder_meaning_by_name};
 
 impl Imap {
-    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<()> {
+    /// Returns true if folders were scanned, false if scanning was postponed.
+    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<bool> {
         // First of all, debounce to once per minute:
         let mut last_scan = context.last_full_folder_scan.lock().await;
         if let Some(last_scan) = *last_scan {
@@ -20,28 +23,18 @@ impl Imap {
                 .await?;
 
             if elapsed_secs < debounce_secs {
-                return Ok(());
+                return Ok(false);
             }
         }
         info!(context, "Starting full folder scan");
 
         self.prepare(context).await?;
-        let session = self.session.as_mut();
-        let session = session.context("scan_folders(): IMAP No Connection established")?;
-        let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
+        let folders = self.list_folders(context).await?;
         let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
 
         for folder in folders {
-            let folder = match folder {
-                Ok(f) => f,
-                Err(e) => {
-                    warn!(context, "Can't get folder: {}", e);
-                    continue;
-                }
-            };
-
             // Gmail labels are not folders and should be skipped. For example,
             // emails appear in the inbox and under "All Mail" as soon as it is
             // received. The code used to wrongly conclude that the email had
@@ -67,6 +60,9 @@ impl Imap {
             let is_drafts = folder_meaning == FolderMeaning::Drafts
                 || (folder_meaning == FolderMeaning::Unknown
                     && folder_name_meaning == FolderMeaning::Drafts);
+            let is_spam_folder = folder_meaning == FolderMeaning::Spam
+                || (folder_meaning == FolderMeaning::Unknown
+                    && folder_name_meaning == FolderMeaning::Spam);
 
             // Don't scan folders that are watched anyway
             if !watched_folders.contains(&folder.name().to_string()) && !is_drafts {
@@ -74,7 +70,7 @@ impl Imap {
                 self.server_sent_unsolicited_exists(context)?;
 
                 loop {
-                    self.fetch_move_delete(context, folder.name())
+                    self.fetch_move_delete(context, folder.name(), is_spam_folder)
                         .await
                         .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
 
@@ -86,36 +82,51 @@ impl Imap {
             }
         }
 
-        // We iterate over both folder meanings to make sure that if e.g. the "Sent" folder was deleted,
-        // `ConfiguredSentboxFolder` is set to `None`:
-        for config in &[
-            Config::ConfiguredSentboxFolder,
-            Config::ConfiguredSpamFolder,
-        ] {
-            context
-                .set_config(*config, folder_configs.get(config).map(|s| s.as_str()))
-                .await?;
-        }
+        // Set the `ConfiguredSentboxFolder` or set it to `None` if the folder was deleted.
+        context
+            .set_config(
+                Config::ConfiguredSentboxFolder,
+                folder_configs
+                    .get(&Config::ConfiguredSentboxFolder)
+                    .map(|s| s.as_str()),
+            )
+            .await?;
 
         last_scan.replace(Instant::now());
-        Ok(())
+        Ok(true)
     }
+
+    /// Returns the names of all folders on the IMAP server.
+    pub async fn list_folders(
+        self: &mut Imap,
+        context: &Context,
+    ) -> Result<Vec<async_imap::types::Name>> {
+        let session = self.session.as_mut();
+        let session = session.context("No IMAP connection")?;
+        let list = session
+            .list(Some(""), Some("*"))
+            .await?
+            .filter_map(|f| f.ok_or_log_msg(context, "list_folders() can't get folder"));
+        Ok(list.collect().await)
+    }
+}
+
+pub(crate) async fn get_watched_folder_configs(context: &Context) -> Result<Vec<Config>> {
+    let mut res = vec![Config::ConfiguredInboxFolder];
+    if context.get_config_bool(Config::SentboxWatch).await? {
+        res.push(Config::ConfiguredSentboxFolder);
+    }
+    if context.should_watch_mvbox().await? {
+        res.push(Config::ConfiguredMvboxFolder);
+    }
+    Ok(res)
 }
 
 pub(crate) async fn get_watched_folders(context: &Context) -> Result<Vec<String>> {
     let mut res = Vec::new();
-    if let Some(inbox_folder) = context.get_config(Config::ConfiguredInboxFolder).await? {
-        res.push(inbox_folder);
-    }
-    let folder_watched_configured = &[
-        (Config::SentboxWatch, Config::ConfiguredSentboxFolder),
-        (Config::MvboxMove, Config::ConfiguredMvboxFolder),
-    ];
-    for (watched, configured) in folder_watched_configured {
-        if context.get_config_bool(*watched).await? {
-            if let Some(folder) = context.get_config(*configured).await? {
-                res.push(folder);
-            }
+    for folder_config in get_watched_folder_configs(context).await? {
+        if let Some(folder) = context.get_config(folder_config).await? {
+            res.push(folder);
         }
     }
     Ok(res)

src/imex.rs

@@ -16,7 +16,7 @@ use rand::{thread_rng, Rng};
 use crate::blob::BlobObject;
 use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
 use crate::config::Config;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+use crate::contact::ContactId;
 use crate::context::Context;
 use crate::dc_tools::{
     dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
@@ -26,7 +26,7 @@ use crate::e2ee;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::log::LogExt;
-use crate::message::{Message, MsgId};
+use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::pgp;
@@ -165,7 +165,6 @@ pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
 }
 
 async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
-    let mut msg: Message;
     let setup_code = create_setup_code(context);
     /* this may require a keypair to be created. this may take a second ... */
     let setup_file_content = render_setup_file(context, &setup_code).await?;
@@ -177,9 +176,11 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     )
     .await?;
 
-    let chat_id = ChatId::create_for_contact(context, DC_CONTACT_ID_SELF).await?;
-    msg = Message::default();
-    msg.viewtype = Viewtype::File;
+    let chat_id = ChatId::create_for_contact(context, ContactId::SELF).await?;
+    let mut msg = Message {
+        viewtype: Viewtype::File,
+        ..Default::default()
+    };
     msg.param.set(Param::File, setup_file_blob.as_name());
     msg.subject = stock_str::ac_setup_msg_subject(context).await;
     msg.param
@@ -236,7 +237,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
 
     let msg_subj = stock_str::ac_setup_msg_subject(context).await;
     let msg_body = stock_str::ac_setup_msg_body(context).await;
-    let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
+    let msg_body_html = msg_body.replace('\r', "").replace('\n', "<br>");
     Ok(format!(
         concat!(
             "<!DOCTYPE html>\r\n",
@@ -350,9 +351,8 @@ async fn set_self_key(
         }
     };
 
-    let self_addr = context.get_config(Config::ConfiguredAddr).await?;
-    ensure!(self_addr.is_some(), "Missing self addr");
-    let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
+    let self_addr = context.get_primary_self_addr().await?;
+    let addr = EmailAddress::new(&self_addr)?;
     let keypair = pgp::KeyPair {
         addr,
         public: public_key,
@@ -440,13 +440,6 @@ async fn import_backup(
     backup_to_import: &Path,
     passphrase: String,
 ) -> Result<()> {
-    info!(
-        context,
-        "Import \"{}\" to \"{}\".",
-        backup_to_import.display(),
-        context.get_dbfile().display()
-    );
-
     ensure!(
         !context.is_configured().await?,
         "Cannot import backups to accounts in use."
@@ -458,6 +451,16 @@ async fn import_backup(
 
     let backup_file = File::open(backup_to_import).await?;
     let file_size = backup_file.metadata().await?.len();
+    info!(
+        context,
+        "Import \"{}\" ({} bytes) to \"{}\".",
+        backup_to_import.display(),
+        file_size,
+        context.get_dbfile().display()
+    );
+
+    context.sql.config_cache.write().await.clear();
+
     let archive = Archive::new(backup_file);
 
     let mut entries = archive.entries()?;
@@ -899,6 +902,54 @@ mod tests {
         }
     }
 
+    #[async_std::test]
+    async fn test_export_and_import_backup() -> Result<()> {
+        let backup_dir = tempfile::tempdir().unwrap();
+
+        let context1 = TestContext::new_alice().await;
+        assert!(context1.is_configured().await?);
+
+        let context2 = TestContext::new().await;
+        assert!(!context2.is_configured().await?);
+        assert!(has_backup(&context2, backup_dir.path().as_ref())
+            .await
+            .is_err());
+
+        // export from context1
+        assert!(imex(
+            &context1,
+            ImexMode::ExportBackup,
+            backup_dir.path().as_ref(),
+            None,
+        )
+        .await
+        .is_ok());
+        let _event = context1
+            .evtracker
+            .get_matching(|evt| matches!(evt, EventType::ImexProgress(1000)))
+            .await;
+
+        // import to context2
+        let backup = has_backup(&context2, backup_dir.path().as_ref()).await?;
+        assert!(
+            imex(&context2, ImexMode::ImportBackup, backup.as_ref(), None)
+                .await
+                .is_ok()
+        );
+        let _event = context2
+            .evtracker
+            .get_matching(|evt| matches!(evt, EventType::ImexProgress(1000)))
+            .await;
+
+        assert!(context2.is_configured().await?);
+        assert_eq!(
+            context2.get_config(Config::Addr).await?,
+            Some("alice@example.org".to_string())
+        );
+
+        Ok(())
+    }
+
     #[test]
     fn test_normalize_setup_code() {
         let norm = normalize_setup_code("123422343234423452346234723482349234");

src/job.rs

@@ -4,23 +4,21 @@
 //! and job types.
 use std::fmt;
 
-use anyhow::{bail, format_err, Context as _, Error, Result};
+use anyhow::{format_err, Context as _, Result};
 use deltachat_derive::{FromSql, ToSql};
 use rand::{thread_rng, Rng};
 
 use crate::config::Config;
-use crate::contact::{normalize_name, Contact, Modifier, Origin};
+use crate::contact::{normalize_name, Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::events::EventType;
-use crate::imap::{Imap, ImapActionResult};
-use crate::location;
-use crate::log::LogExt;
+use crate::imap::Imap;
 use crate::message::{Message, MsgId};
 use crate::mimefactory::MimeFactory;
 use crate::param::{Param, Params};
 use crate::scheduler::InterruptInfo;
-use crate::smtp::{smtp_send, Smtp};
+use crate::smtp::{smtp_send, SendResult, Smtp};
 use crate::sql;
 
 // results in ~3 weeks for the last backoff timespan
@@ -32,7 +30,6 @@ const JOB_RETRIES: u32 = 17;
 )]
 #[repr(u32)]
 pub(crate) enum Thread {
-    Unknown = 0,
     Imap = 100,
     Smtp = 5000,
 }
@@ -40,7 +37,7 @@ pub(crate) enum Thread {
 /// Job try result.
 #[derive(Debug, Display)]
 pub enum Status {
-    Finished(std::result::Result<(), Error>),
+    Finished(Result<()>),
     RetryNow,
     RetryLater,
 }
@@ -60,12 +57,6 @@ macro_rules! job_try {
     };
 }
 
-impl Default for Thread {
-    fn default() -> Self {
-        Thread::Unknown
-    }
-}
-
 #[derive(
     Debug,
     Display,
@@ -81,12 +72,8 @@ impl Default for Thread {
 )]
 #[repr(u32)]
 pub enum Action {
-    Unknown = 0,
-
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
-    Housekeeping = 105, // low priority ...
     FetchExistingMsgs = 110,
-    MarkseenMsgOnImap = 130,
 
     // this is user initiated so it should have a fairly high priority
     UpdateRecentQuota = 140,
@@ -102,33 +89,19 @@ pub enum Action {
     ResyncFolders = 300,
 
     // Jobs in the SMTP-thread, range from DC_SMTP_THREAD..DC_SMTP_THREAD+999
-    MaybeSendLocations = 5005, // low priority ...
-    MaybeSendLocationsEnded = 5007,
     SendMdn = 5010,
 }
 
-impl Default for Action {
-    fn default() -> Self {
-        Action::Unknown
-    }
-}
-
 impl From<Action> for Thread {
     fn from(action: Action) -> Thread {
         use Action::*;
 
         match action {
-            Unknown => Thread::Unknown,
-
-            Housekeeping => Thread::Imap,
             FetchExistingMsgs => Thread::Imap,
             ResyncFolders => Thread::Imap,
-            MarkseenMsgOnImap => Thread::Imap,
             UpdateRecentQuota => Thread::Imap,
             DownloadMsg => Thread::Imap,
 
-            MaybeSendLocations => Thread::Smtp,
-            MaybeSendLocationsEnded => Thread::Smtp,
             SendMdn => Thread::Smtp,
         }
     }
@@ -197,7 +170,7 @@ impl Job {
                     "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
                     paramsv![
                         self.desired_timestamp,
-                        self.tries as i64,
+                        i64::from(self.tries),
                         self.param.to_string(),
                         self.job_id as i32,
                     ],
@@ -224,7 +197,7 @@ impl Job {
     async fn get_additional_mdn_jobs(
         &self,
         context: &Context,
-        contact_id: u32,
+        contact_id: ContactId,
     ) -> Result<(Vec<u32>, Vec<String>)> {
         // Extract message IDs from job parameters
         let res: Vec<(u32, MsgId)> = context
@@ -271,7 +244,7 @@ impl Job {
             return Status::Finished(Err(format_err!("MDNs are disabled")));
         }
 
-        let contact_id = self.foreign_id;
+        let contact_id = ContactId::new(self.foreign_id);
         let contact = job_try!(Contact::load_from_db(context, contact_id).await);
         if contact.is_blocked() {
             return Status::Finished(Err(format_err!("Contact is blocked")));
@@ -318,13 +291,18 @@ impl Job {
             return Status::RetryLater;
         }
 
-        let status = smtp_send(context, &recipients, &body, smtp, msg_id, 0).await;
-        if matches!(status, Status::Finished(Ok(_))) {
-            // Remove additional SendMdn jobs we have aggregated into this one.
-            job_try!(kill_ids(context, &additional_job_ids).await);
+        match smtp_send(context, &recipients, &body, smtp, msg_id, 0).await {
+            SendResult::Success => {
+                // Remove additional SendMdn jobs we have aggregated into this one.
+                job_try!(kill_ids(context, &additional_job_ids).await);
+                Status::Finished(Ok(()))
+            }
+            SendResult::Retry => {
+                info!(context, "Temporary SMTP failure while sending an MDN");
+                Status::RetryLater
+            }
+            SendResult::Failure(err) => Status::Finished(Err(err)),
         }
-
-        status
     }
 
     /// Read the recipients from old emails sent by the user and add them as contacts.
@@ -352,7 +330,7 @@ impl Job {
                 Config::ConfiguredSentboxFolder,
             ] {
                 if let Some(folder) = job_try!(context.get_config(*config).await) {
-                    if let Err(e) = imap.fetch_new_messages(context, &folder, true).await {
+                    if let Err(e) = imap.fetch_new_messages(context, &folder, false, true).await {
                         // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
                         warn!(context, "Could not fetch messages, retrying: {:#}", e);
                         return Status::RetryLater;
@@ -365,90 +343,38 @@ impl Job {
         Status::Finished(Ok(()))
     }
 
-    /// Synchronizes UIDs for sentbox, inbox and mvbox.
+    /// Synchronizes UIDs for all folders.
     async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.prepare(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
 
-        let sentbox_folder = job_try!(context.get_config(Config::ConfiguredSentboxFolder).await);
-        if let Some(sentbox_folder) = sentbox_folder {
-            job_try!(imap.resync_folder_uids(context, sentbox_folder).await);
-        }
+        let all_folders = match imap.list_folders(context).await {
+            Ok(v) => v,
+            Err(e) => {
+                warn!(context, "Listing folders for resync failed: {:#}", e);
+                return Status::RetryLater;
+            }
+        };
 
-        let inbox_folder = job_try!(context.get_config(Config::ConfiguredInboxFolder).await);
-        if let Some(inbox_folder) = inbox_folder {
-            job_try!(imap.resync_folder_uids(context, inbox_folder).await);
-        }
+        let mut any_failed = false;
 
-        let mvbox_folder = job_try!(context.get_config(Config::ConfiguredMvboxFolder).await);
-        if let Some(mvbox_folder) = mvbox_folder {
-            job_try!(imap.resync_folder_uids(context, mvbox_folder).await);
-        }
-
-        Status::Finished(Ok(()))
-    }
-
-    async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.prepare(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let row = job_try!(
-            context
-                .sql
-                .query_row_optional(
-                    "SELECT uid, folder FROM imap
-                    WHERE rfc724_mid=? AND folder=target
-                    ORDER BY uid ASC
-                    LIMIT 1",
-                    paramsv![msg.rfc724_mid],
-                    |row| {
-                        let uid: u32 = row.get(0)?;
-                        let folder: String = row.get(1)?;
-                        Ok((uid, folder))
-                    }
-                )
+        for folder in all_folders {
+            if let Err(e) = imap
+                .resync_folder_uids(context, folder.name().to_string())
                 .await
-        );
-        if let Some((server_uid, server_folder)) = row {
-            let result = imap.set_seen(context, &server_folder, server_uid).await;
-            match result {
-                ImapActionResult::RetryLater => return Status::RetryLater,
-                ImapActionResult::Success | ImapActionResult::Failed => {}
+            {
+                warn!(context, "{:#}", e);
+                any_failed = true;
             }
+        }
+
+        if any_failed {
+            Status::RetryLater
         } else {
-            info!(
-                context,
-                "Can't mark the message {} as seen on IMAP because there is no known UID",
-                msg.rfc724_mid
-            );
+            Status::Finished(Ok(()))
         }
-
-        // XXX we send MDN even in case of failure to mark the messages as seen, e.g. if it was
-        // already deleted on the server by another device. The job will not be retried so locally
-        // there is no risk of double-sending MDNs.
-        //
-        // Read receipts for system messages are never sent. These messages have no place to
-        // display received read receipt anyway.  And since their text is locally generated,
-        // quoting them is dangerous as it may contain contact names. E.g., for original message
-        // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
-        // be a display name stored in address book rather than the name sent in the From field by
-        // the user.
-        if msg.param.get_bool(Param::WantsMdn).unwrap_or_default() && !msg.is_system_message() {
-            let mdns_enabled = job_try!(context.get_config_bool(Config::MdnsEnabled).await);
-            if mdns_enabled {
-                if let Err(err) = send_mdn(context, &msg).await {
-                    warn!(context, "could not send out mdn for {}: {}", msg.id, err);
-                    return Status::Finished(Err(err));
-                }
-            }
-        }
-
-        Status::Finished(Ok(()))
     }
 }
 
@@ -463,9 +389,12 @@ pub async fn kill_action(context: &Context, action: Action) -> Result<()> {
 
 /// Remove jobs with specified IDs.
 async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
+    if job_ids.is_empty() {
+        return Ok(());
+    }
     let q = format!(
         "DELETE FROM jobs WHERE id IN({})",
-        job_ids.iter().map(|_| "?").collect::<Vec<&str>>().join(",")
+        sql::repeat_vars(job_ids.len())
     );
     context
         .sql
@@ -636,19 +565,9 @@ async fn perform_job_action(
     );
 
     let try_res = match job.action {
-        Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))),
         Action::SendMdn => job.send_mdn(context, connection.smtp()).await,
-        Action::MaybeSendLocations => location::job_maybe_send_locations(context, job).await,
-        Action::MaybeSendLocationsEnded => {
-            location::job_maybe_send_locations_ended(context, job).await
-        }
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
-        Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
-        Action::Housekeeping => {
-            sql::housekeeping(context).await.ok_or_log(context);
-            Status::Finished(Ok(()))
-        }
         Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
             Ok(status) => status,
             Err(err) => Status::Finished(Err(err)),
@@ -676,16 +595,20 @@ fn get_backoff_time_offset(tries: u32, action: Action) -> i64 {
             if seconds < 1 {
                 seconds = 1;
             }
-            seconds as i64
+            i64::from(seconds)
         }
     }
 }
 
-async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
+pub(crate) async fn send_mdn(context: &Context, msg_id: MsgId, from_id: ContactId) -> Result<()> {
     let mut param = Params::new();
-    param.set(Param::MsgId, msg.id.to_u32().to_string());
+    param.set(Param::MsgId, msg_id.to_u32().to_string());
 
-    add(context, Job::new(Action::SendMdn, msg.from_id, param, 0)).await?;
+    add(
+        context,
+        Job::new(Action::SendMdn, from_id.to_u32(), param, 0),
+    )
+    .await?;
 
     Ok(())
 }
@@ -708,17 +631,14 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
 
     if delay_seconds == 0 {
         match action {
-            Action::Unknown => unreachable!(),
-            Action::Housekeeping
-            | Action::ResyncFolders
-            | Action::MarkseenMsgOnImap
+            Action::ResyncFolders
             | Action::FetchExistingMsgs
             | Action::UpdateRecentQuota
             | Action::DownloadMsg => {
                 info!(context, "interrupt: imap");
                 context.interrupt_inbox(InterruptInfo::new(false)).await;
             }
-            Action::MaybeSendLocations | Action::MaybeSendLocationsEnded | Action::SendMdn => {
+            Action::SendMdn => {
                 info!(context, "interrupt: smtp");
                 context.interrupt_smtp(InterruptInfo::new(false)).await;
             }
@@ -727,18 +647,6 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
     Ok(())
 }
 
-async fn load_housekeeping_job(context: &Context) -> Result<Option<Job>> {
-    let last_time = context.get_config_i64(Config::LastHousekeeping).await?;
-
-    let next_time = last_time + (60 * 60 * 24);
-    if next_time <= time() {
-        kill_action(context, Action::Housekeeping).await?;
-        Ok(Some(Job::new(Action::Housekeeping, 0, Params::new(), 0)))
-    } else {
-        Ok(None)
-    }
-}
-
 /// Load jobs from the database.
 ///
 /// Load jobs for this "[Thread]", i.e. either load SMTP jobs or load
@@ -782,7 +690,7 @@ LIMIT 1;
         params = paramsv![thread_i];
     };
 
-    let job = loop {
+    loop {
         let job_res = context
             .sql
             .query_row_optional(query, params.clone(), |row| {
@@ -801,7 +709,7 @@ LIMIT 1;
             .await;
 
         match job_res {
-            Ok(job) => break job,
+            Ok(job) => return Ok(job),
             Err(err) => {
                 // Remove invalid job from the DB
                 info!(context, "cleaning up job, because of {}", err);
@@ -819,20 +727,6 @@ LIMIT 1;
                     .with_context(|| format!("Failed to delete invalid job {}", id))?;
             }
         }
-    };
-
-    match thread {
-        Thread::Unknown => {
-            bail!("unknown thread for job")
-        }
-        Thread::Imap => {
-            if let Some(job) = job {
-                Ok(Some(job))
-            } else {
-                Ok(load_housekeeping_job(context).await?)
-            }
-        }
-        Thread::Smtp => Ok(job),
     }
 }
 
@@ -880,8 +774,7 @@ mod tests {
             &InterruptInfo::new(false),
         )
         .await?;
-        // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
-        assert_eq!(jobs.unwrap().action, Action::Housekeeping);
+        assert!(jobs.is_none());
 
         insert_job(&t, 1, true).await;
         let jobs = load_next(

src/key.rs

@@ -91,16 +91,17 @@ impl DcKey for SignedPublicKey {
     type KeyType = SignedPublicKey;
 
     async fn load_self(context: &Context) -> Result<Self::KeyType> {
+        let addr = context.get_primary_self_addr().await?;
         match context
             .sql
             .query_row_optional(
                 r#"
             SELECT public_key
               FROM keypairs
-             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
+             WHERE addr=?
                AND is_default=1;
             "#,
-                paramsv![],
+                paramsv![addr],
                 |row| {
                     let bytes: Vec<u8> = row.get(0)?;
                     Ok(bytes)
@@ -198,10 +199,7 @@ impl DcSecretKey for SignedSecretKey {
 }
 
 async fn generate_keypair(context: &Context) -> Result<KeyPair> {
-    let addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await?
-        .context("no address configured")?;
+    let addr = context.get_primary_self_addr().await?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -297,8 +295,8 @@ pub async fn store_self_keypair(
             .context("failed to clear default")?;
     }
     let is_default = match default {
-        KeyPairUse::Default => true as i32,
-        KeyPairUse::ReadOnly => false as i32,
+        KeyPairUse::Default => i32::from(true),
+        KeyPairUse::ReadOnly => i32::from(false),
     };
 
     let addr = keypair.addr.to_string();
@@ -318,7 +316,7 @@ pub async fn store_self_keypair(
 }
 
 /// A key fingerprint
-#[derive(Clone, Eq, PartialEq, Hash)]
+#[derive(Clone, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)]
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {

src/lib.rs

@@ -7,7 +7,8 @@
     clippy::all,
     clippy::indexing_slicing,
     clippy::wildcard_imports,
-    clippy::needless_borrow
+    clippy::needless_borrow,
+    clippy::cast_lossless
 )]
 #![allow(
     clippy::match_bool,

src/location.rs

@@ -1,20 +1,20 @@
 //! Location handling.
 use std::convert::TryFrom;
 
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Context as _, Result};
+use async_std::channel::Receiver;
+use async_std::future::timeout;
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+use std::time::Duration;
 
 use crate::chat::{self, ChatId};
-use crate::config::Config;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+use crate::contact::ContactId;
 use crate::context::Context;
-use crate::dc_tools::time;
+use crate::dc_tools::{duration_to_str, time};
 use crate::events::EventType;
-use crate::job::{self, Job};
-use crate::message::{Message, MsgId};
+use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
-use crate::param::Params;
 use crate::stock_str;
 
 /// Location record
@@ -25,7 +25,7 @@ pub struct Location {
     pub longitude: f64,
     pub accuracy: f64,
     pub timestamp: i64,
-    pub contact_id: u32,
+    pub contact_id: ContactId,
     pub msg_id: u32,
     pub chat_id: ChatId,
     pub marker: Option<String>,
@@ -101,10 +101,10 @@ impl Kml {
             let val = event.unescape_and_decode(reader).unwrap_or_default();
 
             let val = val
-                .replace("\n", "")
-                .replace("\r", "")
-                .replace("\t", "")
-                .replace(" ", "");
+                .replace('\n', "")
+                .replace('\r', "")
+                .replace('\t', "")
+                .replace(' ', "");
 
             if self.tag.contains(KmlTag::WHEN) && val.len() >= 19 {
                 // YYYY-MM-DDTHH:MM:SSZ
@@ -227,32 +227,11 @@ pub async fn send_locations_to_chat(
     }
     context.emit_event(EventType::ChatModified(chat_id));
     if 0 != seconds {
-        schedule_maybe_send_locations(context, false).await?;
-        job::add(
-            context,
-            job::Job::new(
-                job::Action::MaybeSendLocationsEnded,
-                chat_id.to_u32(),
-                Params::new(),
-                seconds + 1,
-            ),
-        )
-        .await?;
+        context.interrupt_location().await;
     }
     Ok(())
 }
 
-async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) -> Result<()> {
-    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await? {
-        job::add(
-            context,
-            job::Job::new(job::Action::MaybeSendLocations, 0, Params::new(), 60),
-        )
-        .await?;
-    };
-    Ok(())
-}
-
 /// Returns whether `chat_id` or any chat is sending locations.
 ///
 /// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
@@ -314,18 +293,18 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                         accuracy,
                         time(),
                         chat_id,
-                        DC_CONTACT_ID_SELF,
+                        ContactId::SELF,
                     ]
             ).await {
                 warn!(context, "failed to store location {:?}", err);
             } else {
+                info!(context, "stored location for chat {}", chat_id);
                 continue_streaming = true;
             }
         }
         if continue_streaming {
-            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+            context.emit_event(EventType::LocationChanged(Some(ContactId::SELF)));
         };
-        schedule_maybe_send_locations(context, false).await.ok();
     }
 
     continue_streaming
@@ -424,10 +403,7 @@ pub async fn delete_all(context: &Context) -> Result<()> {
 pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)> {
     let mut last_added_location_id = 0;
 
-    let self_addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await?
-        .unwrap_or_default();
+    let self_addr = context.get_primary_self_addr().await?;
 
     let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
         "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
@@ -462,10 +438,10 @@ pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)
              GROUP BY timestamp \
              ORDER BY timestamp;",
                 paramsv![
-                    DC_CONTACT_ID_SELF,
+                    ContactId::SELF,
                     locations_send_begin,
                     locations_last_sent,
-                    DC_CONTACT_ID_SELF
+                    ContactId::SELF
                 ],
                 |row| {
                     let location_id: i32 = row.get(0)?;
@@ -558,7 +534,7 @@ pub async fn set_msg_location_id(context: &Context, msg_id: MsgId, location_id:
 pub(crate) async fn save(
     context: &Context,
     chat_id: ChatId,
-    contact_id: u32,
+    contact_id: ContactId,
     locations: &[Location],
     independent: bool,
 ) -> Result<Option<u32>> {
@@ -585,12 +561,12 @@ pub(crate) async fn save(
             conn.prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?;
         let mut stmt_insert = conn.prepare_cached(stmt_insert)?;
 
-        let exists = stmt_test.exists(paramsv![timestamp, contact_id as i32])?;
+        let exists = stmt_test.exists(paramsv![timestamp, contact_id])?;
 
         if independent || !exists {
             stmt_insert.execute(paramsv![
                 timestamp,
-                contact_id as i32,
+                contact_id,
                 chat_id,
                 latitude,
                 longitude,
@@ -611,147 +587,140 @@ pub(crate) async fn save(
     Ok(newest_location_id)
 }
 
-pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> job::Status {
-    let now = time();
-    let mut continue_streaming = false;
-    info!(
-        context,
-        " ----------------- MAYBE_SEND_LOCATIONS -------------- ",
-    );
+pub(crate) async fn location_loop(context: &Context, interrupt_receiver: Receiver<()>) {
+    loop {
+        let next_event = match maybe_send_locations(context).await {
+            Err(err) => {
+                warn!(context, "maybe_send_locations failed: {}", err);
+                Some(60) // Retry one minute later.
+            }
+            Ok(next_event) => next_event,
+        };
 
+        let duration = if let Some(next_event) = next_event {
+            Duration::from_secs(next_event)
+        } else {
+            Duration::from_secs(86400)
+        };
+
+        info!(
+            context,
+            "Location loop is waiting for {} or interrupt",
+            duration_to_str(duration)
+        );
+        timeout(duration, interrupt_receiver.recv()).await.ok();
+    }
+}
+
+/// Returns number of seconds until the next time location streaming for some chat ends
+/// automatically.
+async fn maybe_send_locations(context: &Context) -> Result<Option<u64>> {
+    let mut next_event: Option<u64> = None;
+
+    let now = time();
     let rows = context
         .sql
         .query_map(
-            "SELECT id, locations_send_begin, locations_last_sent \
-         FROM chats \
-         WHERE locations_send_until>?;",
-            paramsv![now],
+            "SELECT id, locations_send_begin, locations_send_until, locations_last_sent
+             FROM chats
+             WHERE locations_send_until>0",
+            [],
             |row| {
                 let chat_id: ChatId = row.get(0)?;
                 let locations_send_begin: i64 = row.get(1)?;
-                let locations_last_sent: i64 = row.get(2)?;
-                continue_streaming = true;
-
-                // be a bit tolerant as the timer may not align exactly with time(NULL)
-                if now - locations_last_sent < (60 - 3) {
-                    Ok(None)
-                } else {
-                    Ok(Some((chat_id, locations_send_begin, locations_last_sent)))
-                }
+                let locations_send_until: i64 = row.get(2)?;
+                let locations_last_sent: i64 = row.get(3)?;
+                Ok((
+                    chat_id,
+                    locations_send_begin,
+                    locations_send_until,
+                    locations_last_sent,
+                ))
             },
             |rows| {
-                rows.filter_map(|v| v.transpose())
-                    .collect::<std::result::Result<Vec<_>, _>>()
+                rows.collect::<std::result::Result<Vec<_>, _>>()
                     .map_err(Into::into)
             },
         )
-        .await;
+        .await
+        .context("failed to query location streaming chats")?;
 
-    if let Ok(rows) = rows {
-        let mut msgs = Vec::new();
-
-        {
-            let conn = job_try!(context.sql.get_conn().await);
-
-            let mut stmt_locations = job_try!(conn.prepare_cached(
-                "SELECT id \
+    for (chat_id, locations_send_begin, locations_send_until, locations_last_sent) in rows {
+        if locations_send_begin > 0 && locations_send_until > now {
+            let can_send = now > locations_last_sent + 60;
+            let has_locations = context
+                .sql
+                .exists(
+                    "SELECT COUNT(id) \
      FROM locations \
      WHERE from_id=? \
      AND timestamp>=? \
      AND timestamp>? \
-     AND independent=0 \
-             ORDER BY timestamp;",
-            ));
+     AND independent=0",
+                    paramsv![ContactId::SELF, locations_send_begin, locations_last_sent,],
+                )
+                .await?;
 
-            for (chat_id, locations_send_begin, locations_last_sent) in &rows {
-                if !stmt_locations
-                    .exists(paramsv![
-                        DC_CONTACT_ID_SELF,
-                        *locations_send_begin,
-                        *locations_last_sent,
-                    ])
-                    .unwrap_or_default()
-                {
-                    // if there is no new location, there's nothing to send.
-                    // however, maybe we want to bypass this test eg. 15 minutes
-                } else {
-                    // pending locations are attached automatically to every message,
+            next_event = next_event
+                .into_iter()
+                .chain(u64::try_from(locations_send_until - now).into_iter())
+                .min();
+
+            if has_locations {
+                if can_send {
+                    // Send location-only message.
+                    // Pending locations are attached automatically to every message,
                     // so also to this empty text message.
-                    // DC_CMD_LOCATION is only needed to create a nicer subject.
-                    //
-                    // for optimisation and to avoid flooding the sending queue,
-                    // we could sending these messages only if we're really online.
-                    // the easiest way to determine this, is to check for an empty message queue.
-                    // (might not be 100%, however, as positions are sent combined later
-                    // and dc_set_location() is typically called periodically, this is ok)
+                    info!(
+                        context,
+                        "Chat {} has pending locations, sending them.", chat_id
+                    );
                     let mut msg = Message::new(Viewtype::Text);
                     msg.hidden = true;
                     msg.param.set_cmd(SystemMessage::LocationOnly);
-                    msgs.push((*chat_id, msg));
+                    chat::send_msg(context, chat_id, &mut msg).await?;
+                } else {
+                    // Wait until pending locations can be sent.
+                    info!(
+                        context,
+                        "Chat {} has pending locations, but they can't be sent yet.", chat_id
+                    );
+                    next_event = next_event
+                        .into_iter()
+                        .chain(u64::try_from(locations_last_sent + 61 - now).into_iter())
+                        .min();
                 }
+            } else {
+                info!(
+                    context,
+                    "Chat {} has location streaming enabled, but no pending locations.", chat_id
+                );
             }
-        }
-
-        for (chat_id, mut msg) in msgs.into_iter() {
-            // TODO: better error handling
-            chat::send_msg(context, chat_id, &mut msg)
-                .await
-                .unwrap_or_default();
-        }
-    }
-
-    if continue_streaming {
-        job_try!(schedule_maybe_send_locations(context, true).await);
-    }
-    job::Status::Finished(Ok(()))
-}
-
-pub(crate) async fn job_maybe_send_locations_ended(
-    context: &Context,
-    job: &mut Job,
-) -> job::Status {
-    // this function is called when location-streaming _might_ have ended for a chat.
-    // the function checks, if location-streaming is really ended;
-    // if so, a device-message is added if not yet done.
-
-    let chat_id = ChatId::new(job.foreign_id);
-
-    let (send_begin, send_until) = job_try!(
-        context
-            .sql
-            .query_row(
-                "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
-                paramsv![chat_id],
-                |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
-            )
-            .await
-    );
-
-    let now = time();
-    if !(send_begin != 0 && now <= send_until) {
-        // still streaming -
-        // may happen as several calls to dc_send_locations_to_chat()
-        // do not un-schedule pending DC_MAYBE_SEND_LOC_ENDED jobs
-        if !(send_begin == 0 && send_until == 0) {
-            // not streaming, device-message already sent
-            job_try!(
-                context
-                    .sql
-                    .execute(
-                        "UPDATE chats \
-                             SET locations_send_begin=0, locations_send_until=0 \
- WHERE id=?",
-                        paramsv![chat_id],
-                    )
-                    .await
+        } else {
+            // Location streaming was either explicitly disabled (locations_send_begin = 0) or
+            // locations_send_until is in the past.
+            info!(
+                context,
+                "Disabling location streaming for chat {}.", chat_id
             );
+            context
+                .sql
+                .execute(
+                    "UPDATE chats \
+                         SET locations_send_begin=0, locations_send_until=0 \
+                         WHERE id=?",
+                    paramsv![chat_id],
+                )
+                .await
+                .context("failed to disable location streaming")?;
 
             let stock_str = stock_str::msg_location_disabled(context).await;
-            job_try!(chat::add_info_msg(context, chat_id, &stock_str, now).await);
+            chat::add_info_msg(context, chat_id, &stock_str, now).await?;
             context.emit_event(EventType::ChatModified(chat_id));
         }
     }
-    job::Status::Finished(Ok(()))
+
+    Ok(next_event)
 }
 
 #[cfg(test)]
@@ -759,6 +728,7 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
+    use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils::TestContext;
 
     #[async_std::test]
@@ -819,4 +789,68 @@ mod tests {
         assert!(!is_marker(" "));
         assert!(!is_marker("\t"));
     }
+
+    /// Tests that location.kml is hidden.
+    #[async_std::test]
+    async fn receive_location_kml() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        dc_receive_imf(
+            &alice,
+            br#"Subject: Hello
+Message-ID: hello@example.net
+To: Alice <alice@example.org>
+From: Bob <bob@example.net>
+Date: Mon, 20 Dec 2021 00:00:00 +0000
+Chat-Version: 1.0
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+Text message."#,
+            false,
+        )
+        .await?;
+        let received_msg = alice.get_last_msg().await;
+        assert_eq!(received_msg.text.unwrap(), "Text message.");
+
+        dc_receive_imf(
+            &alice,
+            br#"Subject: locations
+MIME-Version: 1.0
+To: <alice@example.org>
+From: <bob@example.net>
+Date: Tue, 21 Dec 2021 00:00:00 +0000
+Chat-Version: 1.0
+Message-ID: <foobar@example.net>
+Content-Type: multipart/mixed; boundary="U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF"
+
+
+--U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+
+
+--U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
+Content-Type: application/vnd.google-earth.kml+xml
+Content-Disposition: attachment; filename="location.kml"
+
+<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2">
+<Document addr="bob@example.net">
+<Placemark><Timestamp><when>2021-11-21T00:00:00Z</when></Timestamp><Point><coordinates accuracy="1.0000000000000000">10.00000000000000,20.00000000000000</coordinates></Point></Placemark>
+</Document>
+</kml>
+
+--U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF--"#,
+            false,
+        )
+        .await?;
+
+        // Received location message is not visible, last message stays the same.
+        let received_msg2 = alice.get_last_msg().await;
+        assert_eq!(received_msg2.id, received_msg.id);
+
+        let locations = get_range(&alice, None, None, 0, 0).await?;
+        assert_eq!(locations.len(), 1);
+        Ok(())
+    }
 }

src/login_param.rs

@@ -139,8 +139,18 @@ pub struct LoginParam {
 }
 
 impl LoginParam {
+    /// Load entered (candidate) account settings
+    pub async fn load_candidate_params(context: &Context) -> Result<Self> {
+        LoginParam::from_database(context, "").await
+    }
+
+    /// Load configured (working) account settings
+    pub async fn load_configured_params(context: &Context) -> Result<Self> {
+        LoginParam::from_database(context, "configured_").await
+    }
+
     /// Read the login parameters from the database.
-    pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
+    async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
@@ -242,18 +252,18 @@ impl LoginParam {
     }
 
     /// Save this loginparam to the database.
-    pub async fn save_to_database(&self, context: &Context, prefix: impl AsRef<str>) -> Result<()> {
-        let prefix = prefix.as_ref();
+    pub async fn save_as_configured_params(&self, context: &Context) -> Result<()> {
+        let prefix = "configured_";
         let sql = &context.sql;
 
-        let key = format!("{}addr", prefix);
-        sql.set_raw_config(key, Some(&self.addr)).await?;
+        context.set_primary_self_addr(&self.addr).await?;
 
         let key = format!("{}mail_server", prefix);
         sql.set_raw_config(key, Some(&self.imap.server)).await?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_raw_config_int(key, self.imap.port as i32).await?;
+        sql.set_raw_config_int(key, i32::from(self.imap.port))
+            .await?;
 
         let key = format!("{}mail_user", prefix);
         sql.set_raw_config(key, Some(&self.imap.user)).await?;
@@ -273,7 +283,8 @@ impl LoginParam {
         sql.set_raw_config(key, Some(&self.smtp.server)).await?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_raw_config_int(key, self.smtp.port as i32).await?;
+        sql.set_raw_config_int(key, i32::from(self.smtp.port))
+            .await?;
 
         let key = format!("{}send_user", prefix);
         sql.set_raw_config(key, Some(&self.smtp.user)).await?;
@@ -436,8 +447,8 @@ mod tests {
             socks5_config: None,
         };
 
-        param.save_to_database(&t, "foobar_").await?;
-        let loaded = LoginParam::from_database(&t, "foobar_").await?;
+        param.save_as_configured_params(&t).await?;
+        let loaded = LoginParam::load_configured_params(&t).await?;
 
         assert_eq!(param, loaded);
         Ok(())

src/message.rs

@@ -1,7 +1,6 @@
 //! # Messages and their identifiers.
 
 use std::collections::BTreeSet;
-use std::convert::TryInto;
 
 use anyhow::{ensure, format_err, Context as _, Result};
 use async_std::path::{Path, PathBuf};
@@ -10,25 +9,27 @@ use rusqlite::types::ValueRef;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
+use crate::config::Config;
 use crate::constants::{
-    Blocked, Chattype, VideochatType, Viewtype, DC_CHAT_ID_TRASH, DC_CONTACT_ID_INFO,
-    DC_CONTACT_ID_SELF, DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
+    Blocked, Chattype, VideochatType, DC_CHAT_ID_TRASH, DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
 };
-use crate::contact::{Contact, Origin};
+use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::dc_tools::{
     dc_create_smeared_timestamp, dc_get_filebytes, dc_get_filemeta, dc_gm2local_offset,
     dc_read_file, dc_timestamp_to_str, dc_truncate, time,
 };
 use crate::download::DownloadState;
-use crate::ephemeral::Timer as EphemeralTimer;
+use crate::ephemeral::{start_ephemeral_timers_msgids, Timer as EphemeralTimer};
 use crate::events::EventType;
-use crate::job::{self, Action};
+use crate::imap::markseen_on_imap_table;
+use crate::job;
 use crate::log::LogExt;
 use crate::mimeparser::{parse_message_id, FailureReport, SystemMessage};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
 use crate::scheduler::InterruptInfo;
+use crate::sql;
 use crate::stock_str;
 use crate::summary::Summary;
 
@@ -183,7 +184,7 @@ impl rusqlite::types::ToSql for MsgId {
                 format_err!("Invalid MsgId {}", self.0).into(),
             ));
         }
-        let val = rusqlite::types::Value::Integer(self.0 as i64);
+        let val = rusqlite::types::Value::Integer(i64::from(self.0));
         let out = rusqlite::types::ToSqlOutput::Owned(val);
         Ok(out)
     }
@@ -194,7 +195,7 @@ impl rusqlite::types::FromSql for MsgId {
     fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
         // Would be nice if we could use match here, but alas.
         i64::column_result(value).and_then(|val| {
-            if 0 <= val && val <= std::u32::MAX as i64 {
+            if 0 <= val && val <= i64::from(std::u32::MAX) {
                 Ok(MsgId::new(val as u32))
             } else {
                 Err(rusqlite::types::FromSqlError::OutOfRange(val))
@@ -240,8 +241,8 @@ impl Default for MessengerMessage {
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct Message {
     pub(crate) id: MsgId,
-    pub(crate) from_id: u32,
-    pub(crate) to_id: u32,
+    pub(crate) from_id: ContactId,
+    pub(crate) to_id: ContactId,
     pub(crate) chat_id: ChatId,
     pub(crate) viewtype: Viewtype,
     pub(crate) state: MessageState,
@@ -387,7 +388,7 @@ impl Message {
     }
 
     pub async fn try_calc_and_set_dimensions(&mut self, context: &Context) -> Result<()> {
-        if chat::msgtype_has_file(self.viewtype) {
+        if self.viewtype.has_file() {
             let file_param = self.param.get_path(Param::File, context)?;
             if let Some(path_and_filename) = file_param {
                 if (self.viewtype == Viewtype::Image || self.viewtype == Viewtype::Gif)
@@ -431,7 +432,7 @@ impl Message {
     /// this is done by dc_set_location() and dc_send_locations_to_chat().
     ///
     /// Typically results in the event #DC_EVENT_LOCATION_CHANGED with
-    /// contact_id set to DC_CONTACT_ID_SELF.
+    /// contact_id set to ContactId::SELF.
     ///
     /// @param latitude North-south position of the location.
     /// @param longitude East-west position of the location.
@@ -456,7 +457,7 @@ impl Message {
         self.id
     }
 
-    pub fn get_from_id(&self) -> u32 {
+    pub fn get_from_id(&self) -> ContactId {
         self.from_id
     }
 
@@ -544,7 +545,7 @@ impl Message {
             &chat_loaded
         };
 
-        let contact = if self.from_id != DC_CONTACT_ID_SELF {
+        let contact = if self.from_id != ContactId::SELF {
             match chat.typ {
                 Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
                     Some(Contact::get_by_id(context, self.from_id).await?)
@@ -597,8 +598,8 @@ impl Message {
 
     pub fn is_info(&self) -> bool {
         let cmd = self.param.get_cmd();
-        self.from_id == DC_CONTACT_ID_INFO
-            || self.to_id == DC_CONTACT_ID_INFO
+        self.from_id == ContactId::INFO
+            || self.to_id == ContactId::INFO
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
@@ -620,7 +621,7 @@ impl Message {
     /// copied to the blobdir.  Thus those attachments need to be
     /// created immediately in the blobdir with a valid filename.
     pub fn is_increation(&self) -> bool {
-        chat::msgtype_has_file(self.viewtype) && self.state == MessageState::OutPreparing
+        self.viewtype.has_file() && self.state == MessageState::OutPreparing
     }
 
     pub fn is_setupmessage(&self) -> bool {
@@ -1017,7 +1018,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
     ret += &format!(" by {}", name);
     ret += "\n";
 
-    if msg.from_id != DC_CONTACT_ID_SELF {
+    if msg.from_id != ContactId::SELF {
         let s = dc_timestamp_to_str(if 0 != msg.timestamp_rcvd {
             msg.timestamp_rcvd
         } else {
@@ -1038,7 +1039,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
         );
     }
 
-    if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
+    if msg.from_id == ContactId::INFO || msg.to_id == ContactId::INFO {
         // device-internal message, no further details needed
         return Ok(ret);
     }
@@ -1049,7 +1050,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
             "SELECT contact_id, timestamp_sent FROM msgs_mdns WHERE msg_id=?;",
             paramsv![msg_id],
             |row| {
-                let contact_id: i32 = row.get(0)?;
+                let contact_id: ContactId = row.get(0)?;
                 let ts: i64 = row.get(1)?;
                 Ok((contact_id, ts))
             },
@@ -1061,7 +1062,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
             let fts = dc_timestamp_to_str(ts);
             ret += &format!("Read: {}", fts);
 
-            let name = Contact::load_from_db(context, contact_id.try_into()?)
+            let name = Contact::load_from_db(context, contact_id)
                 .await
                 .map(|contact| contact.get_name_n_addr())
                 .unwrap_or_default();
@@ -1237,7 +1238,7 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     for msg_id in msg_ids.iter() {
         let msg = Message::load_from_db(context, *msg_id).await?;
         if msg.location_id > 0 {
-            delete_poi_location(context, msg.location_id).await;
+            delete_poi_location(context, msg.location_id).await?;
         }
         msg_id
             .trash(context)
@@ -1253,32 +1254,26 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     }
 
     if !msg_ids.is_empty() {
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
-        job::kill_action(context, Action::Housekeeping).await?;
-        job::add(
-            context,
-            job::Job::new(Action::Housekeeping, 0, Params::new(), 10),
-        )
-        .await?;
+        context.emit_msgs_changed_without_ids();
+
+        // Run housekeeping to delete unused blobs.
+        context.set_config(Config::LastHousekeeping, None).await?;
     }
 
-    // Interrupt Inbox loop to start message deletion.
+    // Interrupt Inbox loop to start message deletion and run housekeeping.
     context.interrupt_inbox(InterruptInfo::new(false)).await;
     Ok(())
 }
 
-async fn delete_poi_location(context: &Context, location_id: u32) -> bool {
+async fn delete_poi_location(context: &Context, location_id: u32) -> Result<()> {
     context
         .sql
         .execute(
             "DELETE FROM locations WHERE independent = 1 AND id=?;",
             paramsv![location_id as i32],
         )
-        .await
-        .is_ok()
+        .await?;
+    Ok(())
 }
 
 pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()> {
@@ -1286,61 +1281,94 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
         return Ok(());
     }
 
-    let conn = context.sql.get_conn().await?;
-    let msgs = async_std::task::spawn_blocking(move || -> Result<_> {
-        let mut stmt = conn.prepare_cached(concat!(
-            "SELECT",
-            "    m.chat_id AS chat_id,",
-            "    m.state AS state,",
-            "    c.blocked AS blocked",
-            " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
-            " WHERE m.id=? AND m.chat_id>9"
-        ))?;
-
-        let mut msgs = Vec::with_capacity(msg_ids.len());
-        for id in msg_ids.into_iter() {
-            let query_res = stmt.query_row(paramsv![id], |row| {
+    let msgs = context
+        .sql
+        .query_map(
+            format!(
+                "SELECT
+                    m.id AS id,
+                    m.chat_id AS chat_id,
+                    m.state AS state,
+                    m.ephemeral_timer AS ephemeral_timer,
+                    m.param AS param,
+                    m.from_id AS from_id,
+                    m.rfc724_mid AS rfc724_mid,
+                    c.blocked AS blocked
+                 FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id
+                 WHERE m.id IN ({}) AND m.chat_id>9",
+                sql::repeat_vars(msg_ids.len())
+            ),
+            rusqlite::params_from_iter(&msg_ids),
+            |row| {
+                let id: MsgId = row.get("id")?;
+                let chat_id: ChatId = row.get("chat_id")?;
+                let state: MessageState = row.get("state")?;
+                let param: Params = row.get::<_, String>("param")?.parse().unwrap_or_default();
+                let from_id: ContactId = row.get("from_id")?;
+                let rfc724_mid: String = row.get("rfc724_mid")?;
+                let blocked: Option<Blocked> = row.get("blocked")?;
+                let ephemeral_timer: EphemeralTimer = row.get("ephemeral_timer")?;
                 Ok((
-                    row.get::<_, ChatId>("chat_id")?,
-                    row.get::<_, MessageState>("state")?,
-                    row.get::<_, Option<Blocked>>("blocked")?
-                        .unwrap_or_default(),
+                    id,
+                    chat_id,
+                    state,
+                    param,
+                    from_id,
+                    rfc724_mid,
+                    blocked.unwrap_or_default(),
+                    ephemeral_timer,
                 ))
-            });
-            if let Err(rusqlite::Error::QueryReturnedNoRows) = query_res {
-                continue;
-            }
-            let (chat_id, state, blocked) = query_res.map_err(Into::<anyhow::Error>::into)?;
-            msgs.push((id, chat_id, state, blocked));
-        }
-        drop(stmt);
-        drop(conn);
-        Ok(msgs)
-    })
-    .await?;
+            },
+            |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+        )
+        .await?;
+
+    if msgs.iter().any(
+        |(_id, _chat_id, _state, _param, _from_id, _rfc724_mid, _blocked, ephemeral_timer)| {
+            *ephemeral_timer != EphemeralTimer::Disabled
+        },
+    ) {
+        start_ephemeral_timers_msgids(context, &msg_ids)
+            .await
+            .context("failed to start ephemeral timers")?;
+    }
 
     let mut updated_chat_ids = BTreeSet::new();
-
-    for (id, curr_chat_id, curr_state, curr_blocked) in msgs.into_iter() {
-        if let Err(err) = id.start_ephemeral_timer(context).await {
-            error!(
-                context,
-                "Failed to start ephemeral timer for message {}: {}", id, err
-            );
-            continue;
-        }
-
+    for (
+        id,
+        curr_chat_id,
+        curr_state,
+        curr_param,
+        curr_from_id,
+        curr_rfc724_mid,
+        curr_blocked,
+        _curr_ephemeral_timer,
+    ) in msgs.into_iter()
+    {
         if curr_blocked == Blocked::Not
             && (curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed)
         {
             update_msg_state(context, id, MessageState::InSeen).await?;
             info!(context, "Seen message {}.", id);
 
-            job::add(
-                context,
-                job::Job::new(Action::MarkseenMsgOnImap, id.to_u32(), Params::new(), 0),
-            )
-            .await?;
+            markseen_on_imap_table(context, &curr_rfc724_mid).await?;
+
+            // Read receipts for system messages are never sent. These messages have no place to
+            // display received read receipt anyway.  And since their text is locally generated,
+            // quoting them is dangerous as it may contain contact names. E.g., for original message
+            // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
+            // be a display name stored in address book rather than the name sent in the From field by
+            // the user.
+            if curr_param.get_bool(Param::WantsMdn).unwrap_or_default()
+                && curr_param.get_cmd() == SystemMessage::Unknown
+            {
+                let mdns_enabled = context.get_config_bool(Config::MdnsEnabled).await?;
+                if mdns_enabled {
+                    if let Err(err) = job::send_mdn(context, id, curr_from_id).await {
+                        warn!(context, "could not send out mdn for {}: {}", id, err);
+                    }
+                }
+            }
             updated_chat_ids.insert(curr_chat_id);
         }
     }
@@ -1426,11 +1454,11 @@ pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl
 /// returns Some if an event should be send
 pub async fn handle_mdn(
     context: &Context,
-    from_id: u32,
+    from_id: ContactId,
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Result<Option<(ChatId, MsgId)>> {
-    if from_id == DC_CONTACT_ID_SELF {
+    if from_id == ContactId::SELF {
         warn!(
             context,
             "ignoring MDN sent to self, this is a bug on the sender device"
@@ -1479,7 +1507,7 @@ pub async fn handle_mdn(
         .sql
         .exists(
             "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=? AND contact_id=?;",
-            paramsv![msg_id, from_id as i32,],
+            paramsv![msg_id, from_id],
         )
         .await?
     {
@@ -1487,7 +1515,7 @@ pub async fn handle_mdn(
             .sql
             .execute(
                 "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?);",
-                paramsv![msg_id, from_id as i32, timestamp_sent],
+                paramsv![msg_id, from_id, timestamp_sent],
             )
             .await?;
     }
@@ -1635,7 +1663,7 @@ pub async fn estimate_deletion_cnt(
     from_server: bool,
     seconds: i64,
 ) -> Result<usize> {
-    let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+    let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
         .await?
         .unwrap_or_default();
     let threshold_timestamp = time() - seconds;
@@ -1701,16 +1729,114 @@ pub(crate) async fn rfc724_mid_exists(
     Ok(res)
 }
 
+/// How a message is primarily displayed.
+#[derive(
+    Debug,
+    Display,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    FromPrimitive,
+    ToPrimitive,
+    FromSql,
+    ToSql,
+    Serialize,
+    Deserialize,
+)]
+#[repr(u32)]
+pub enum Viewtype {
+    Unknown = 0,
+
+    /// Text message.
+    /// The text of the message is set using dc_msg_set_text()
+    /// and retrieved with dc_msg_get_text().
+    Text = 10,
+
+    /// Image message.
+    /// If the image is an animated GIF, the type DC_MSG_GIF should be used.
+    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension
+    /// and retrieved via dc_msg_set_file(), dc_msg_set_dimension().
+    Image = 20,
+
+    /// Animated GIF message.
+    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension()
+    /// and retrieved via dc_msg_get_file(), dc_msg_get_width(), dc_msg_get_height().
+    Gif = 21,
+
+    /// Message containing a sticker, similar to image.
+    /// If possible, the ui should display the image without borders in a transparent way.
+    /// A click on a sticker will offer to install the sticker set in some future.
+    Sticker = 23,
+
+    /// Message containing an Audio file.
+    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
+    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration().
+    Audio = 40,
+
+    /// A voice message that was directly recorded by the user.
+    /// For all other audio messages, the type #DC_MSG_AUDIO should be used.
+    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
+    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration()
+    Voice = 41,
+
+    /// Video messages.
+    /// File, width, height and durarion
+    /// are set via dc_msg_set_file(), dc_msg_set_dimension(), dc_msg_set_duration()
+    /// and retrieved via
+    /// dc_msg_get_file(), dc_msg_get_width(),
+    /// dc_msg_get_height(), dc_msg_get_duration().
+    Video = 50,
+
+    /// Message containing any file, eg. a PDF.
+    /// The file is set via dc_msg_set_file()
+    /// and retrieved via dc_msg_get_file().
+    File = 60,
+
+    /// Message is an invitation to a videochat.
+    VideochatInvitation = 70,
+
+    /// Message is an webxdc instance.
+    Webxdc = 80,
+}
+
+impl Default for Viewtype {
+    fn default() -> Self {
+        Viewtype::Unknown
+    }
+}
+
+impl Viewtype {
+    /// Whether a message with this [`Viewtype`] should have a file attachment.
+    pub fn has_file(&self) -> bool {
+        match self {
+            Viewtype::Unknown => false,
+            Viewtype::Text => false,
+            Viewtype::Image => true,
+            Viewtype::Gif => true,
+            Viewtype::Sticker => true,
+            Viewtype::Audio => true,
+            Viewtype::Voice => true,
+            Viewtype::Video => true,
+            Viewtype::File => true,
+            Viewtype::VideochatInvitation => false,
+            Viewtype::Webxdc => true,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use num_traits::FromPrimitive;
+
     use crate::chat::{marknoticed_chat, ChatItem};
     use crate::chatlist::Chatlist;
-    use crate::constants::DC_CONTACT_ID_DEVICE;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils as test;
     use crate::test_utils::TestContext;
 
+    use super::*;
+
     #[test]
     fn test_guess_msgtype_from_suffix() {
         assert_eq!(
@@ -1844,7 +1970,7 @@ mod tests {
         // test that get_width() and get_height() are returning some dimensions for images;
         // (as the device-chat contains a welcome-images, we check that)
         t.update_device_chats().await.ok();
-        let device_chat_id = ChatId::get_for_contact(&t, DC_CONTACT_ID_DEVICE)
+        let device_chat_id = ChatId::get_for_contact(&t, ContactId::DEVICE)
             .await
             .unwrap();
 
@@ -1918,7 +2044,6 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello\n",
-            "INBOX",
             false,
         )
         .await
@@ -2127,7 +2252,6 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -2145,7 +2269,6 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello again\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -2155,4 +2278,29 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn test_viewtype_derive_display_works_as_expected() {
+        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
+    }
+
+    #[test]
+    fn test_viewtype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Viewtype::Unknown, Viewtype::default());
+        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
+        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
+        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
+        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
+        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
+        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
+        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
+        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
+        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
+        assert_eq!(
+            Viewtype::VideochatInvitation,
+            Viewtype::from_i32(70).unwrap()
+        );
+        assert_eq!(Viewtype::Webxdc, Viewtype::from_i32(80).unwrap());
+    }
 }

src/mimefactory.rs

@@ -7,9 +7,9 @@ use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
 
 use crate::blob::BlobObject;
-use crate::chat::{self, Chat};
+use crate::chat::Chat;
 use crate::config::Config;
-use crate::constants::{Chattype, Viewtype, DC_FROM_HANDSHAKE};
+use crate::constants::{Chattype, DC_FROM_HANDSHAKE};
 use crate::contact::Contact;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::IsNoneOrEmpty;
@@ -22,7 +22,7 @@ use crate::ephemeral::Timer as EphemeralTimer;
 use crate::format_flowed::{format_flowed, format_flowed_quote};
 use crate::html::new_html_mimepart;
 use crate::location;
-use crate::message::{self, Message, MsgId};
+use crate::message::{self, Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
@@ -133,11 +133,7 @@ impl<'a> MimeFactory<'a> {
     ) -> Result<MimeFactory<'a>> {
         let chat = Chat::load_from_db(context, msg.chat_id).await?;
 
-        let from_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
-
+        let from_addr = context.get_primary_self_addr().await?;
         let config_displayname = context
             .get_config(Config::Displayname)
             .await?
@@ -237,10 +233,7 @@ impl<'a> MimeFactory<'a> {
         ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
         let contact = Contact::load_from_db(context, msg.from_id).await?;
-        let from_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
+        let from_addr = context.get_primary_self_addr().await?;
         let from_displayname = context
             .get_config(Config::Displayname)
             .await?
@@ -278,10 +271,7 @@ impl<'a> MimeFactory<'a> {
         &self,
         context: &Context,
     ) -> Result<Vec<(Option<Peerstate>, &str)>> {
-        let self_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .context("not configured")?;
+        let self_addr = context.get_primary_self_addr().await?;
 
         let mut res = Vec::new();
         for (_, addr) in self
@@ -503,33 +493,80 @@ impl<'a> MimeFactory<'a> {
             }
         }
 
+        // Start with Internet Message Format headers in the order of the standard example
+        // <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.1.1>.
         headers
             .unprotected
-            .push(Header::new("MIME-Version".into(), "1.0".into()));
+            .push(Header::new_with_value("From".into(), vec![from]).unwrap());
+        if let Some(sender_displayname) = &self.sender_displayname {
+            let sender =
+                Address::new_mailbox_with_name(sender_displayname.clone(), self.from_addr.clone());
+            headers
+                .unprotected
+                .push(Header::new_with_value("Sender".into(), vec![sender]).unwrap());
+        }
+        headers
+            .unprotected
+            .push(Header::new_with_value("To".into(), to).unwrap());
 
+        let subject_str = self.subject_str(context).await?;
+        let encoded_subject = if subject_str
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
+        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
+        // but we do not want to encode all subjects just because they contain a space.
+        {
+            subject_str.clone()
+        } else {
+            encode_words(&subject_str)
+        };
+        headers
+            .protected
+            .push(Header::new("Subject".into(), encoded_subject));
+
+        let date = chrono::Utc
+            .from_local_datetime(&chrono::NaiveDateTime::from_timestamp(self.timestamp, 0))
+            .unwrap()
+            .to_rfc2822();
+        headers.unprotected.push(Header::new("Date".into(), date));
+
+        let rfc724_mid = match self.loaded {
+            Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
+            Loaded::Mdn { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
+        };
+        let rfc724_mid_headervalue = render_rfc724_mid(&rfc724_mid);
+
+        // Amazon's SMTP servers change the `Message-ID`, just as Outlook's SMTP servers do.
+        // Outlook's servers add an `X-Microsoft-Original-Message-ID` header with the original `Message-ID`,
+        // and when downloading messages we look for this header in order to correctly identify
+        // messages.
+        // Amazon's servers do not add such a header, so we just add it ourselves.
+        if let Some(server) = context.get_config(Config::ConfiguredSendServer).await? {
+            if server.ends_with(".amazonaws.com") {
+                headers.unprotected.push(Header::new(
+                    "X-Microsoft-Original-Message-ID".into(),
+                    rfc724_mid_headervalue.clone(),
+                ))
+            }
+        }
+
+        headers
+            .unprotected
+            .push(Header::new("Message-ID".into(), rfc724_mid_headervalue));
+
+        // Reply headers as in <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.2>.
+        if !self.in_reply_to.is_empty() {
+            headers
+                .unprotected
+                .push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
+        }
         if !self.references.is_empty() {
             headers
                 .unprotected
                 .push(Header::new("References".into(), self.references.clone()));
         }
 
-        if !self.in_reply_to.is_empty() {
-            headers
-                .unprotected
-                .push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
-        }
-
-        let date = chrono::Utc
-            .from_local_datetime(&chrono::NaiveDateTime::from_timestamp(self.timestamp, 0))
-            .unwrap()
-            .to_rfc2822();
-
-        headers.unprotected.push(Header::new("Date".into(), date));
-
-        headers
-            .unprotected
-            .push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
-
+        // Automatic Response headers <https://www.rfc-editor.org/rfc/rfc3834>
         if let Loaded::Mdn { .. } = self.loaded {
             headers.unprotected.push(Header::new(
                 "Auto-Submitted".to_string(),
@@ -542,6 +579,11 @@ impl<'a> MimeFactory<'a> {
             ));
         }
 
+        // Non-standard headers.
+        headers
+            .unprotected
+            .push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
+
         if self.req_mdn {
             // we use "Chat-Disposition-Notification-To"
             // because replies to "Disposition-Notification-To" are weird in many cases
@@ -556,21 +598,9 @@ impl<'a> MimeFactory<'a> {
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
         let skip_autocrypt = self.should_skip_autocrypt();
-        let subject_str = self.subject_str(context).await?;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let encrypt_helper = EncryptHelper::new(context).await?;
 
-        let encoded_subject = if subject_str
-            .chars()
-            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
-        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
-        // but we do not want to encode all subjects just because they contain a space.
-        {
-            subject_str.clone()
-        } else {
-            encode_words(&subject_str)
-        };
-
         if !skip_autocrypt {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
@@ -579,15 +609,6 @@ impl<'a> MimeFactory<'a> {
                 .push(Header::new("Autocrypt".into(), aheader));
         }
 
-        headers
-            .protected
-            .push(Header::new("Subject".into(), encoded_subject));
-
-        let rfc724_mid = match self.loaded {
-            Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
-            Loaded::Mdn { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
-        };
-
         let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(context).await?;
         if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
             headers.protected.push(Header::new(
@@ -596,25 +617,11 @@ impl<'a> MimeFactory<'a> {
             ));
         }
 
-        headers.unprotected.push(Header::new(
-            "Message-ID".into(),
-            render_rfc724_mid(&rfc724_mid),
-        ));
-
+        // MIME header <https://datatracker.ietf.org/doc/html/rfc2045>.
+        // Content-Type
         headers
             .unprotected
-            .push(Header::new_with_value("To".into(), to).unwrap());
-
-        headers
-            .unprotected
-            .push(Header::new_with_value("From".into(), vec![from]).unwrap());
-        if let Some(sender_displayname) = &self.sender_displayname {
-            let sender =
-                Address::new_mailbox_with_name(sender_displayname.clone(), self.from_addr.clone());
-            headers
-                .unprotected
-                .push(Header::new_with_value("Sender".into(), vec![sender]).unwrap());
-        }
+            .push(Header::new("MIME-Version".into(), "1.0".into()));
 
         let mut is_gossiped = false;
 
@@ -843,9 +850,12 @@ impl<'a> MimeFactory<'a> {
         }
 
         if chat.typ == Chattype::Group {
-            headers
-                .protected
-                .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
+            // Send group ID unless it is an ad hoc group that has no ID.
+            if !chat.grpid.is_empty() {
+                headers
+                    .protected
+                    .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
+            }
 
             let encoded = encode_words(&chat.name);
             headers
@@ -1123,7 +1133,7 @@ impl<'a> MimeFactory<'a> {
         }
 
         // add attachment part
-        if chat::msgtype_has_file(self.msg.viewtype) {
+        if self.msg.viewtype.has_file() {
             if !is_file_size_okay(context, self.msg).await? {
                 bail!(
                     "Message exceeds the recommended {} MB.",
@@ -1435,12 +1445,13 @@ fn maybe_encode_words(words: &str) -> String {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use async_std::fs::File;
     use async_std::prelude::*;
+    use mailparse::{addrparse_header, MailHeaderMap};
 
     use crate::chat::ChatId;
     use crate::chat::{
-        add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg,
+        self, add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg,
         ProtectionStatus,
     };
     use crate::chatlist::Chatlist;
@@ -1449,9 +1460,7 @@ mod tests {
     use crate::mimeparser::MimeMessage;
     use crate::test_utils::{get_chat_msg, TestContext};
 
-    use async_std::fs::File;
-    use mailparse::{addrparse_header, MailHeaderMap};
-
+    use super::*;
     #[test]
     fn test_render_email_address() {
         let display_name = "ä space";
@@ -1656,7 +1665,6 @@ mod tests {
             Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
             \n\
             hello\n",
-            "INBOX",
             false,
         )
         .await
@@ -1749,7 +1757,6 @@ mod tests {
                 t.get_last_msg().await.rfc724_mid
             )
             .as_bytes(),
-            "INBOX",
             false,
         )
         .await?;
@@ -1860,7 +1867,6 @@ mod tests {
                     Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                     \n\
                     Some other, completely unrelated content\n",
-                "INBOX",
                 false,
             )
             .await
@@ -1885,9 +1891,7 @@ mod tests {
             .await
             .unwrap();
 
-        dc_receive_imf(context, imf_raw, "INBOX", false)
-            .await
-            .unwrap();
+        dc_receive_imf(context, imf_raw, false).await.unwrap();
 
         let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
 
@@ -2084,4 +2088,27 @@ mod tests {
 
         Ok(())
     }
+
+    /// Tests that standard IMF header "From:" comes before non-standard "Autocrypt:" header.
+    #[async_std::test]
+    async fn test_from_before_autocrypt() -> Result<()> {
+        // create chat with bob
+        let t = TestContext::new_alice().await;
+        let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
+
+        // send message to bob: that should get multipart/mixed because of the avatar moved to inner header;
+        // make sure, `Subject:` stays in the outer header (imf header)
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("this is the text!".to_string()));
+
+        let sent_msg = t.send_msg(chat.id, &mut msg).await;
+        let payload = sent_msg.payload();
+
+        assert_eq!(payload.match_indices("Autocrypt:").count(), 1);
+        assert_eq!(payload.match_indices("From:").count(), 1);
+
+        assert!(payload.match_indices("From:").next() < payload.match_indices("Autocrypt:").next());
+
+        Ok(())
+    }
 }

src/mimeparser.rs

@@ -2,6 +2,7 @@
 
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
+use std::io::Cursor;
 use std::pin::Pin;
 
 use anyhow::{bail, Result};
@@ -12,8 +13,8 @@ use once_cell::sync::Lazy;
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
-use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
-use crate::contact::addr_normalize;
+use crate::constants::{DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
+use crate::contact::{addr_normalize, ContactId};
 use crate::context::Context;
 use crate::dc_tools::{dc_get_filemeta, dc_truncate, parse_receive_headers};
 use crate::dehtml::dehtml;
@@ -23,7 +24,7 @@ use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
 use crate::location;
-use crate::message;
+use crate::message::{self, Viewtype};
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::simplify::simplify;
@@ -368,6 +369,16 @@ impl MimeMessage {
             } else if value == "protection-disabled" {
                 self.is_system_message = SystemMessage::ChatProtectionDisabled;
             }
+        } else if self.get_header(HeaderDef::ChatGroupMemberRemoved).is_some() {
+            self.is_system_message = SystemMessage::MemberRemovedFromGroup;
+        } else if self.get_header(HeaderDef::ChatGroupMemberAdded).is_some() {
+            self.is_system_message = SystemMessage::MemberAddedToGroup;
+        } else if self.get_header(HeaderDef::ChatGroupNameChanged).is_some() {
+            self.is_system_message = SystemMessage::GroupNameChanged;
+        } else if let Some(value) = self.get_header(HeaderDef::ChatContent) {
+            if value == "group-avatar-changed" {
+                self.is_system_message = SystemMessage::GroupImageChanged;
+            }
         }
     }
 
@@ -1020,8 +1031,9 @@ impl MimeMessage {
         if decoded_data.is_empty() {
             return;
         }
+        let reader = Cursor::new(decoded_data);
         let msg_type = if context
-            .is_webxdc_file(filename, decoded_data)
+            .is_webxdc_file(filename, reader)
             .await
             .unwrap_or(false)
         {
@@ -1138,7 +1150,7 @@ impl MimeMessage {
         }
     }
 
-    pub fn get_rfc724_mid(&self) -> Option<String> {
+    pub(crate) fn get_rfc724_mid(&self) -> Option<String> {
         self.get_header(HeaderDef::XMicrosoftOriginalMessageId)
             .or_else(|| self.get_header(HeaderDef::MessageId))
             .and_then(|msgid| parse_message_id(msgid).ok())
@@ -1203,6 +1215,9 @@ impl MimeMessage {
         if let Some(_disposition) = report_fields.get_header_value(HeaderDef::Disposition) {
             let original_message_id = report_fields
                 .get_header_value(HeaderDef::OriginalMessageId)
+                // MS Exchange doesn't add an Original-Message-Id header. Instead, they put
+                // the original message id into the In-Reply-To header:
+                .or_else(|| report.headers.get_header_value(HeaderDef::InReplyTo))
                 .and_then(|v| parse_message_id(&v).ok());
             let additional_message_ids = report_fields
                 .get_header_value(HeaderDef::AdditionalMessageIds)
@@ -1366,7 +1381,7 @@ impl MimeMessage {
     pub async fn handle_reports(
         &self,
         context: &Context,
-        from_id: u32,
+        from_id: ContactId,
         sent_timestamp: i64,
         parts: &[Part],
     ) {
@@ -1480,8 +1495,8 @@ async fn update_gossip_peerstates(
 pub(crate) struct Report {
     /// Original-Message-ID header
     ///
-    /// It MUST be present if the original message has a Message-ID according to RFC 8098, but MS
-    /// Exchange does not add it nevertheless, in which case it is `None`.
+    /// It MUST be present if the original message has a Message-ID according to RFC 8098.
+    /// In case we can't find it (shouldn't happen), this is None.
     original_message_id: Option<String>,
     /// Additional-Message-IDs
     additional_message_ids: Vec<String>,
@@ -2900,7 +2915,6 @@ On 2020-10-25, Bob wrote:
         dc_receive_imf(
             &t.ctx,
             include_bytes!("../test-data/message/subj_with_multimedia_msg.eml"),
-            "INBOX",
             false,
         )
         .await
@@ -3049,7 +3063,7 @@ Subject: ...
 
 Some quote.
 "###;
-        dc_receive_imf(&t, raw, "INBOX", false).await?;
+        dc_receive_imf(&t, raw, false).await?;
 
         // Delta Chat generates In-Reply-To with a starting tab when Message-ID is too long.
         let raw = br###"In-Reply-To:
@@ -3066,7 +3080,7 @@ Subject: ...
 Some reply
 "###;
 
-        dc_receive_imf(&t, raw, "INBOX", false).await?;
+        dc_receive_imf(&t, raw, false).await?;
 
         let msg = t.get_last_msg().await;
         assert_eq!(msg.get_text().unwrap(), "Some reply");
@@ -3094,13 +3108,13 @@ Message.
 "###;
 
         // Bob receives message.
-        dc_receive_imf(&bob, raw, "INBOX", false).await?;
+        dc_receive_imf(&bob, raw, false).await?;
         let msg = bob.get_last_msg().await;
         // Message is incoming.
         assert!(msg.param.get_bool(Param::WantsMdn).unwrap());
 
         // Alice receives copy-to-self.
-        dc_receive_imf(&alice, raw, "INBOX", false).await?;
+        dc_receive_imf(&alice, raw, false).await?;
         let msg = alice.get_last_msg().await;
         // Message is outgoing, don't send read receipt to self.
         assert!(msg.param.get_bool(Param::WantsMdn).is_none());
@@ -3126,7 +3140,6 @@ Message.
                  \n\
                  hello\n"
                 .as_bytes(),
-            "INBOX",
             false,
         )
         .await?;
@@ -3164,7 +3177,6 @@ Message.
                  \n\
                  --SNIPP--"
             .as_bytes(),
-            "INBOX",
             false,
         )
         .await?;
@@ -3183,10 +3195,32 @@ Message.
     #[async_std::test]
     async fn test_ms_exchange_mdn() -> Result<()> {
         let t = TestContext::new_alice().await;
-        let raw =
+        t.set_config(Config::ShowEmails, Some("2")).await?;
+
+        let original =
+            include_bytes!("../test-data/message/ms_exchange_report_original_message.eml");
+        dc_receive_imf(&t, original, false).await?;
+        let original_msg_id = t.get_last_msg().await.id;
+
+        // 1. Test mimeparser directly
+        let mdn =
             include_bytes!("../test-data/message/ms_exchange_report_disposition_notification.eml");
-        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await?;
-        assert!(!mimeparser.mdn_reports.is_empty());
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn).await?;
+        assert_eq!(mimeparser.mdn_reports.len(), 1);
+        assert_eq!(
+            mimeparser.mdn_reports[0].original_message_id.as_deref(),
+            Some("d5904dc344eeb5deaf9bb44603f0c716@posteo.de")
+        );
+        assert!(mimeparser.mdn_reports[0].additional_message_ids.is_empty());
+
+        // 2. Test that marking the original msg as read works
+        dc_receive_imf(&t, mdn, false).await?;
+
+        assert_eq!(
+            original_msg_id.get_state(&t).await?,
+            MessageState::OutMdnRcvd
+        );
+
         Ok(())
     }
 }

src/peerstate.rs

@@ -4,11 +4,13 @@ use std::collections::HashSet;
 use std::fmt;
 
 use crate::aheader::{Aheader, EncryptPreference};
-use crate::chat::{self, ChatIdBlocked};
-use crate::constants::Blocked;
+use crate::chat::{self};
+use crate::chatlist::Chatlist;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
+use crate::message::Message;
+use crate::mimeparser::SystemMessage;
 use crate::sql::Sql;
 use crate::stock_str;
 use anyhow::{bail, Result};
@@ -271,14 +273,34 @@ impl Peerstate {
                 .query_get_value("SELECT id FROM contacts WHERE addr=?;", paramsv![self.addr])
                 .await?
             {
-                let chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Request)
-                    .await?
-                    .id;
-
+                let chats = Chatlist::try_load(context, 0, None, contact_id).await?;
                 let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
-
-                chat::add_info_msg(context, chat_id, &msg, timestamp).await?;
-                context.emit_event(EventType::ChatModified(chat_id));
+                for (chat_id, msg_id) in chats.iter() {
+                    let timestamp_sort = if let Some(msg_id) = msg_id {
+                        let lastmsg = Message::load_from_db(context, *msg_id).await?;
+                        lastmsg.timestamp_sort
+                    } else {
+                        context
+                            .sql
+                            .query_get_value(
+                                "SELECT created_timestamp FROM chats WHERE id=?;",
+                                paramsv![chat_id],
+                            )
+                            .await?
+                            .unwrap_or(0)
+                    };
+                    chat::add_info_msg_with_cmd(
+                        context,
+                        *chat_id,
+                        &msg,
+                        SystemMessage::Unknown,
+                        timestamp_sort,
+                        Some(timestamp),
+                        None,
+                    )
+                    .await?;
+                    context.emit_event(EventType::ChatModified(*chat_id));
+                }
             } else {
                 bail!("contact with peerstate.addr {:?} not found", &self.addr);
             }
@@ -379,10 +401,9 @@ impl Peerstate {
     pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&SignedPublicKey> {
         match min_verified {
             PeerstateVerifiedStatus::BidirectVerified => self.verified_key.as_ref(),
-            PeerstateVerifiedStatus::Unverified => self
-                .public_key
-                .as_ref()
-                .or_else(|| self.gossip_key.as_ref()),
+            PeerstateVerifiedStatus::Unverified => {
+                self.public_key.as_ref().or(self.gossip_key.as_ref())
+            }
         }
     }
 

src/plaintext.rs

@@ -41,7 +41,7 @@ impl PlainText {
             // as <http://example.org> cannot be handled correctly
             // (they would become &lt;http://example.org&gt; where the trailing &gt; would become a valid url part).
             // to avoid double encoding, we escape our html-entities by \r that must not be used in the string elsewhere.
-            let line = line.to_string().replace("\r", "");
+            let line = line.to_string().replace('\r', "");
 
             let mut line = LINKIFY_MAIL_RE
                 .replace_all(&*line, "\rLTa href=\rQUOTmailto:$1\rQUOT\rGT$1\rLT/a\rGT")

src/provider.rs

@@ -113,7 +113,7 @@ pub async fn get_provider_info(
     domain: &str,
     skip_mx: bool,
 ) -> Option<&'static Provider> {
-    let domain = domain.rsplitn(2, '@').next()?;
+    let domain = domain.rsplit('@').next()?;
 
     if let Some(provider) = get_provider_by_domain(domain) {
         return Some(provider);

src/provider/data.rs

@@ -366,7 +366,7 @@ static P_EXAMPLE_COM: Lazy<Provider> = Lazy::new(|| {
 }
 });
 
-// fastmail.md: fastmail.com
+// fastmail.md: 123mail.org, 150mail.com, 150ml.com, 16mail.com, 2-mail.com, 4email.net, 50mail.com, airpost.net, allmail.net, bestmail.us, cluemail.com, elitemail.org, emailcorner.net, emailengine.net, emailengine.org, emailgroups.net, emailplus.org, emailuser.net, eml.cc, f-m.fm, fast-email.com, fast-mail.org, fastem.com, fastemail.us, fastemailer.com, fastest.cc, fastimap.com, fastmail.cn, fastmail.co.uk, fastmail.com, fastmail.com.au, fastmail.de, fastmail.es, fastmail.fm, fastmail.fr, fastmail.im, fastmail.in, fastmail.jp, fastmail.mx, fastmail.net, fastmail.nl, fastmail.org, fastmail.se, fastmail.to, fastmail.tw, fastmail.uk, fastmail.us, fastmailbox.net, fastmessaging.com, fea.st, fmail.co.uk, fmailbox.com, fmgirl.com, fmguy.com, ftml.net, h-mail.us, hailmail.net, imap-mail.com, imap.cc, imapmail.org, inoutbox.com, internet-e-mail.com, internet-mail.org, internetemails.net, internetmailing.net, jetemail.net, justemail.net, letterboxes.org, mail-central.com, mail-page.com, mailandftp.com, mailas.com, mailbolt.com, mailc.net, mailcan.com, mailforce.net, mailftp.com, mailhaven.com, mailingaddress.org, mailite.com, mailmight.com, mailnew.com, mailsent.net, mailservice.ms, mailup.net, mailworks.org, ml1.net, mm.st, myfastmail.com, mymacmail.com, nospammail.net, ownmail.net, petml.com, postinbox.com, postpro.net, proinbox.com, promessage.com, realemail.net, reallyfast.biz, reallyfast.info, rushpost.com, sent.as, sent.at, sent.com, speedpost.net, speedymail.org, ssl-mail.com, swift-mail.com, the-fastest.net, the-quickest.com, theinternetemail.com, veryfast.biz, veryspeedy.net, warpmail.net, xsmail.com, yepmail.net, your-mail.com
 static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
     id: "fastmail",
     status: Status::Preparation,
@@ -389,13 +389,6 @@ static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
             port: 465,
             username_pattern: Email,
         },
-        Server {
-            protocol: Smtp,
-            socket: Starttls,
-            hostname: "smtp.fastmail.com",
-            port: 587,
-            username_pattern: Email,
-        },
     ],
     config_defaults: None,
     strict_tls: true,
@@ -716,8 +709,8 @@ static P_MAIL_DE: Lazy<Provider> = Lazy::new(|| Provider {
 static P_MAIL_RU: Lazy<Provider> = Lazy::new(|| {
     Provider {
     id: "mail.ru",
-    status: Status::Ok,
-    before_login_hint: "Не рекомендуется использовать mail.ru, потому что он разряжает вашу батарею быстрее, чем другие провайдеры.",
+    status: Status::Preparation,
+    before_login_hint: "Вам необходимо сгенерировать \"пароль для внешнего приложения\" в веб-интерфейсе mail.ru, чтобы mail.ru работал с Delta Chat.",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/mail-ru",
     server: vec![
@@ -905,7 +898,7 @@ static P_NAVER: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
+// outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com, outlook.de
 static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| Provider {
     id: "outlook.com",
     status: Status::Ok,
@@ -1495,7 +1488,123 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("example.net", &*P_EXAMPLE_COM),
+        ("123mail.org", &*P_FASTMAIL),
+        ("150mail.com", &*P_FASTMAIL),
+        ("150ml.com", &*P_FASTMAIL),
+        ("16mail.com", &*P_FASTMAIL),
+        ("2-mail.com", &*P_FASTMAIL),
+        ("4email.net", &*P_FASTMAIL),
+        ("50mail.com", &*P_FASTMAIL),
+        ("airpost.net", &*P_FASTMAIL),
+        ("allmail.net", &*P_FASTMAIL),
+        ("bestmail.us", &*P_FASTMAIL),
+        ("cluemail.com", &*P_FASTMAIL),
+        ("elitemail.org", &*P_FASTMAIL),
+        ("emailcorner.net", &*P_FASTMAIL),
+        ("emailengine.net", &*P_FASTMAIL),
+        ("emailengine.org", &*P_FASTMAIL),
+        ("emailgroups.net", &*P_FASTMAIL),
+        ("emailplus.org", &*P_FASTMAIL),
+        ("emailuser.net", &*P_FASTMAIL),
+        ("eml.cc", &*P_FASTMAIL),
+        ("f-m.fm", &*P_FASTMAIL),
+        ("fast-email.com", &*P_FASTMAIL),
+        ("fast-mail.org", &*P_FASTMAIL),
+        ("fastem.com", &*P_FASTMAIL),
+        ("fastemail.us", &*P_FASTMAIL),
+        ("fastemailer.com", &*P_FASTMAIL),
+        ("fastest.cc", &*P_FASTMAIL),
+        ("fastimap.com", &*P_FASTMAIL),
+        ("fastmail.cn", &*P_FASTMAIL),
+        ("fastmail.co.uk", &*P_FASTMAIL),
         ("fastmail.com", &*P_FASTMAIL),
+        ("fastmail.com.au", &*P_FASTMAIL),
+        ("fastmail.de", &*P_FASTMAIL),
+        ("fastmail.es", &*P_FASTMAIL),
+        ("fastmail.fm", &*P_FASTMAIL),
+        ("fastmail.fr", &*P_FASTMAIL),
+        ("fastmail.im", &*P_FASTMAIL),
+        ("fastmail.in", &*P_FASTMAIL),
+        ("fastmail.jp", &*P_FASTMAIL),
+        ("fastmail.mx", &*P_FASTMAIL),
+        ("fastmail.net", &*P_FASTMAIL),
+        ("fastmail.nl", &*P_FASTMAIL),
+        ("fastmail.org", &*P_FASTMAIL),
+        ("fastmail.se", &*P_FASTMAIL),
+        ("fastmail.to", &*P_FASTMAIL),
+        ("fastmail.tw", &*P_FASTMAIL),
+        ("fastmail.uk", &*P_FASTMAIL),
+        ("fastmail.us", &*P_FASTMAIL),
+        ("fastmailbox.net", &*P_FASTMAIL),
+        ("fastmessaging.com", &*P_FASTMAIL),
+        ("fea.st", &*P_FASTMAIL),
+        ("fmail.co.uk", &*P_FASTMAIL),
+        ("fmailbox.com", &*P_FASTMAIL),
+        ("fmgirl.com", &*P_FASTMAIL),
+        ("fmguy.com", &*P_FASTMAIL),
+        ("ftml.net", &*P_FASTMAIL),
+        ("h-mail.us", &*P_FASTMAIL),
+        ("hailmail.net", &*P_FASTMAIL),
+        ("imap-mail.com", &*P_FASTMAIL),
+        ("imap.cc", &*P_FASTMAIL),
+        ("imapmail.org", &*P_FASTMAIL),
+        ("inoutbox.com", &*P_FASTMAIL),
+        ("internet-e-mail.com", &*P_FASTMAIL),
+        ("internet-mail.org", &*P_FASTMAIL),
+        ("internetemails.net", &*P_FASTMAIL),
+        ("internetmailing.net", &*P_FASTMAIL),
+        ("jetemail.net", &*P_FASTMAIL),
+        ("justemail.net", &*P_FASTMAIL),
+        ("letterboxes.org", &*P_FASTMAIL),
+        ("mail-central.com", &*P_FASTMAIL),
+        ("mail-page.com", &*P_FASTMAIL),
+        ("mailandftp.com", &*P_FASTMAIL),
+        ("mailas.com", &*P_FASTMAIL),
+        ("mailbolt.com", &*P_FASTMAIL),
+        ("mailc.net", &*P_FASTMAIL),
+        ("mailcan.com", &*P_FASTMAIL),
+        ("mailforce.net", &*P_FASTMAIL),
+        ("mailftp.com", &*P_FASTMAIL),
+        ("mailhaven.com", &*P_FASTMAIL),
+        ("mailingaddress.org", &*P_FASTMAIL),
+        ("mailite.com", &*P_FASTMAIL),
+        ("mailmight.com", &*P_FASTMAIL),
+        ("mailnew.com", &*P_FASTMAIL),
+        ("mailsent.net", &*P_FASTMAIL),
+        ("mailservice.ms", &*P_FASTMAIL),
+        ("mailup.net", &*P_FASTMAIL),
+        ("mailworks.org", &*P_FASTMAIL),
+        ("ml1.net", &*P_FASTMAIL),
+        ("mm.st", &*P_FASTMAIL),
+        ("myfastmail.com", &*P_FASTMAIL),
+        ("mymacmail.com", &*P_FASTMAIL),
+        ("nospammail.net", &*P_FASTMAIL),
+        ("ownmail.net", &*P_FASTMAIL),
+        ("petml.com", &*P_FASTMAIL),
+        ("postinbox.com", &*P_FASTMAIL),
+        ("postpro.net", &*P_FASTMAIL),
+        ("proinbox.com", &*P_FASTMAIL),
+        ("promessage.com", &*P_FASTMAIL),
+        ("realemail.net", &*P_FASTMAIL),
+        ("reallyfast.biz", &*P_FASTMAIL),
+        ("reallyfast.info", &*P_FASTMAIL),
+        ("rushpost.com", &*P_FASTMAIL),
+        ("sent.as", &*P_FASTMAIL),
+        ("sent.at", &*P_FASTMAIL),
+        ("sent.com", &*P_FASTMAIL),
+        ("speedpost.net", &*P_FASTMAIL),
+        ("speedymail.org", &*P_FASTMAIL),
+        ("ssl-mail.com", &*P_FASTMAIL),
+        ("swift-mail.com", &*P_FASTMAIL),
+        ("the-fastest.net", &*P_FASTMAIL),
+        ("the-quickest.com", &*P_FASTMAIL),
+        ("theinternetemail.com", &*P_FASTMAIL),
+        ("veryfast.biz", &*P_FASTMAIL),
+        ("veryspeedy.net", &*P_FASTMAIL),
+        ("warpmail.net", &*P_FASTMAIL),
+        ("xsmail.com", &*P_FASTMAIL),
+        ("yepmail.net", &*P_FASTMAIL),
+        ("your-mail.com", &*P_FASTMAIL),
         ("firemail.at", &*P_FIREMAIL_DE),
         ("firemail.de", &*P_FIREMAIL_DE),
         ("five.chat", &*P_FIVE_CHAT),
@@ -1539,6 +1648,7 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("office365.com", &*P_OUTLOOK_COM),
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
+        ("outlook.de", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
         ("posteo.af", &*P_POSTEO),
         ("posteo.at", &*P_POSTEO),
@@ -1743,4 +1853,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd(2022, 1, 11));
+    Lazy::new(|| chrono::NaiveDate::from_ymd(2022, 1, 31));

src/qr.rs

@@ -9,7 +9,7 @@ use std::collections::BTreeMap;
 use crate::chat::{self, get_chat_id_by_grpid, ChatIdBlocked};
 use crate::config::Config;
 use crate::constants::Blocked;
-use crate::contact::{addr_normalize, may_be_valid_addr, Contact, Origin};
+use crate::contact::{addr_normalize, may_be_valid_addr, Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::key::Fingerprint;
@@ -30,7 +30,7 @@ const HTTPS_SCHEME: &str = "https://";
 #[derive(Debug, Clone, PartialEq)]
 pub enum Qr {
     AskVerifyContact {
-        contact_id: u32,
+        contact_id: ContactId,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
@@ -38,16 +38,16 @@ pub enum Qr {
     AskVerifyGroup {
         grpname: String,
         grpid: String,
-        contact_id: u32,
+        contact_id: ContactId,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
     },
     FprOk {
-        contact_id: u32,
+        contact_id: ContactId,
     },
     FprMismatch {
-        contact_id: Option<u32>,
+        contact_id: Option<ContactId>,
     },
     FprWithoutAddr {
         fingerprint: String,
@@ -60,7 +60,7 @@ pub enum Qr {
         instance_pattern: String,
     },
     Addr {
-        contact_id: u32,
+        contact_id: ContactId,
     },
     Url {
         url: String,
@@ -69,7 +69,7 @@ pub enum Qr {
         text: String,
     },
     WithdrawVerifyContact {
-        contact_id: u32,
+        contact_id: ContactId,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
@@ -77,13 +77,13 @@ pub enum Qr {
     WithdrawVerifyGroup {
         grpname: String,
         grpid: String,
-        contact_id: u32,
+        contact_id: ContactId,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
     },
     ReviveVerifyContact {
-        contact_id: u32,
+        contact_id: ContactId,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
@@ -91,7 +91,7 @@ pub enum Qr {
     ReviveVerifyGroup {
         grpname: String,
         grpid: String,
-        contact_id: u32,
+        contact_id: ContactId,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
@@ -173,7 +173,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
     };
 
     let name = if let Some(encoded_name) = param.get("n") {
-        let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
+        let encoded_name = encoded_name.replace('+', "%20"); // sometimes spaces are encoded as `+`
         match percent_decode_str(&encoded_name).decode_utf8() {
             Ok(name) => name.to_string(),
             Err(err) => bail!("Invalid name: {}", err),
@@ -188,7 +188,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
 
     let grpname = if grpid.is_some() {
         if let Some(encoded_name) = param.get("g") {
-            let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
+            let encoded_name = encoded_name.replace('+', "%20"); // sometimes spaces are encoded as `+`
             match percent_decode_str(&encoded_name).decode_utf8() {
                 Ok(name) => Some(name.to_string()),
                 Err(err) => bail!("Invalid group name: {}", err),
@@ -711,7 +711,7 @@ mod tests {
             ..
         } = qr
         {
-            assert_ne!(contact_id, 0);
+            assert_ne!(contact_id, ContactId::UNDEFINED);
             assert_eq!(grpname, "test ? test !");
         } else {
             bail!("Wrong QR code type");
@@ -729,7 +729,7 @@ mod tests {
             ..
         } = qr
         {
-            assert_ne!(contact_id, 0);
+            assert_ne!(contact_id, ContactId::UNDEFINED);
             assert_eq!(grpname, "test ? test !");
 
             let contact = Contact::get_by_id(&ctx.ctx, contact_id).await?;
@@ -751,7 +751,7 @@ mod tests {
         ).await?;
 
         if let Qr::AskVerifyContact { contact_id, .. } = qr {
-            assert_ne!(contact_id, 0);
+            assert_ne!(contact_id, ContactId::UNDEFINED);
         } else {
             bail!("Wrong QR code type");
         }

src/qr_code_generator.rs

@@ -6,8 +6,7 @@ use crate::{
     chat::{Chat, ChatId},
     color::color_int_to_hex_string,
     config::Config,
-    constants::DC_CONTACT_ID_SELF,
-    contact::Contact,
+    contact::{Contact, ContactId},
     context::Context,
     securejoin, stock_str,
 };
@@ -41,7 +40,7 @@ async fn generate_join_group_qr_code(context: &Context, chat_id: ChatId) -> Resu
 }
 
 async fn generate_verification_qr(context: &Context) -> Result<String> {
-    let contact = Contact::get_by_id(context, DC_CONTACT_ID_SELF).await?;
+    let contact = Contact::get_by_id(context, ContactId::SELF).await?;
 
     let avatar = match contact.get_profile_image(context).await? {
         Some(path) => {

src/quota.rs

@@ -6,13 +6,12 @@ use std::collections::BTreeMap;
 
 use crate::chat::add_device_msg_with_importance;
 use crate::config::Config;
-use crate::constants::Viewtype;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::imap::scan_folders::get_watched_folders;
 use crate::imap::Imap;
 use crate::job::{Action, Status};
-use crate::message::Message;
+use crate::message::{Message, Viewtype};
 use crate::param::Params;
 use crate::{job, stock_str, EventType};
 

src/scheduler.rs

@@ -8,17 +8,19 @@ use async_std::{
 use crate::config::Config;
 use crate::context::Context;
 use crate::dc_tools::maybe_add_time_based_warnings;
-use crate::ephemeral::delete_expired_imap_messages;
+use crate::dc_tools::time;
+use crate::ephemeral::{self, delete_expired_imap_messages};
 use crate::imap::Imap;
 use crate::job::{self, Thread};
+use crate::location;
+use crate::log::LogExt;
 use crate::smtp::{send_smtp_messages, Smtp};
+use crate::sql;
 
 use self::connectivity::ConnectivityStore;
 
 pub(crate) mod connectivity;
 
-pub(crate) struct StopToken;
-
 /// Job and connection scheduler.
 #[derive(Debug)]
 #[allow(clippy::large_enum_variant)]
@@ -33,6 +35,10 @@ pub(crate) enum Scheduler {
         sentbox_handle: Option<task::JoinHandle<()>>,
         smtp: SmtpConnectionState,
         smtp_handle: Option<task::JoinHandle<()>>,
+        ephemeral_handle: Option<task::JoinHandle<()>>,
+        ephemeral_interrupt_send: Sender<()>,
+        location_handle: Option<task::JoinHandle<()>>,
+        location_interrupt_send: Sender<()>,
     },
 }
 
@@ -58,6 +64,14 @@ impl Context {
     pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
         self.scheduler.read().await.interrupt_smtp(info).await;
     }
+
+    pub(crate) async fn interrupt_ephemeral_task(&self) {
+        self.scheduler.read().await.interrupt_ephemeral_task().await;
+    }
+
+    pub(crate) async fn interrupt_location(&self) {
+        self.scheduler.read().await.interrupt_location().await;
+    }
 }
 
 async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConnectionHandlers) {
@@ -67,19 +81,16 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
     let ImapConnectionHandlers {
         mut connection,
         stop_receiver,
-        shutdown_sender,
     } = inbox_handlers;
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started
-            .send(())
-            .await
-            .expect("inbox loop, missing started receiver");
         let ctx = ctx1;
+        if let Err(err) = started.send(()).await {
+            warn!(ctx, "inbox loop, missing started receiver: {}", err);
+            return;
+        };
 
-        // track number of continously executed jobs
-        let mut jobs_loaded = 0;
         let mut info = InterruptInfo::default();
         loop {
             let job = match job::load_next(&ctx, Thread::Imap, &info).await {
@@ -91,22 +102,26 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
             };
 
             match job {
-                Some(job) if jobs_loaded <= 20 => {
-                    jobs_loaded += 1;
+                Some(job) => {
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
                     info = Default::default();
                 }
-                Some(job) => {
-                    // Let the fetch run, but return back to the job afterwards.
-                    jobs_loaded = 0;
-                    info!(ctx, "postponing imap-job {} to run fetch...", job);
-                    fetch(&ctx, &mut connection).await;
-                }
                 None => {
-                    jobs_loaded = 0;
-
                     maybe_add_time_based_warnings(&ctx).await;
 
+                    match ctx.get_config_i64(Config::LastHousekeeping).await {
+                        Ok(last_housekeeping_time) => {
+                            let next_housekeeping_time =
+                                last_housekeeping_time.saturating_add(60 * 60 * 24);
+                            if next_housekeeping_time <= time() {
+                                sql::housekeeping(&ctx).await.ok_or_log(&ctx);
+                            }
+                        }
+                        Err(err) => {
+                            warn!(ctx, "Failed to get last housekeeping time: {}", err);
+                        }
+                    };
+
                     info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await;
                 }
             }
@@ -120,36 +135,6 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         })
         .race(fut)
         .await;
-    shutdown_sender
-        .send(())
-        .await
-        .expect("inbox loop, missing shutdown receiver");
-}
-
-async fn fetch(ctx: &Context, connection: &mut Imap) {
-    match ctx.get_config(Config::ConfiguredInboxFolder).await {
-        Ok(Some(watch_folder)) => {
-            if let Err(err) = connection.prepare(ctx).await {
-                warn!(ctx, "Could not connect: {}", err);
-                return;
-            }
-
-            // fetch
-            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
-                connection.trigger_reconnect(ctx).await;
-                warn!(ctx, "{:#}", err);
-            }
-        }
-        Ok(None) => {
-            info!(ctx, "Can not fetch inbox folder, not set");
-        }
-        Err(err) => {
-            warn!(
-                ctx,
-                "Can not fetch inbox folder, failed to get config: {:?}", err
-            );
-        }
-    }
 }
 
 async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
@@ -161,7 +146,30 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 return connection.fake_idle(ctx, Some(watch_folder)).await;
             }
 
-            // Mark expired messages for deletion.
+            if folder == Config::ConfiguredInboxFolder {
+                if let Err(err) = connection
+                    .store_seen_flags_on_imap(ctx)
+                    .await
+                    .context("store_seen_flags_on_imap failed")
+                {
+                    warn!(ctx, "{:#}", err);
+                }
+            }
+
+            // Fetch the watched folder.
+            if let Err(err) = connection
+                .fetch_move_delete(ctx, &watch_folder, false)
+                .await
+            {
+                connection.trigger_reconnect(ctx).await;
+                warn!(ctx, "{:#}", err);
+                return InterruptInfo::new(false);
+            }
+
+            // Mark expired messages for deletion. Marked messages will be deleted from the server
+            // on the next iteration of `fetch_move_delete`. `delete_expired_imap_messages` is not
+            // called right before `fetch_move_delete` because it is not well optimized and would
+            // otherwise slow down message fetching.
             if let Err(err) = delete_expired_imap_messages(ctx)
                 .await
                 .context("delete_expired_imap_messages failed")
@@ -169,24 +177,44 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 warn!(ctx, "{:#}", err);
             }
 
-            // Scan other folders before fetching from watched folder. This may result in the
-            // messages being moved into the watched folder, for example from the Spam folder to
-            // the Inbox folder.
+            // Scan additional folders only after finishing fetching the watched folder.
+            //
+            // On iOS the application has strictly limited time to work in background, so we may not
+            // be able to scan all folders before time is up if there are many of them.
             if folder == Config::ConfiguredInboxFolder {
                 // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
-                if let Err(err) = connection.scan_folders(ctx).await {
-                    // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
-                    // but maybe just one folder can't be selected or something
-                    warn!(ctx, "{}", err);
+                match connection.scan_folders(ctx).await {
+                    Err(err) => {
+                        // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
+                        // but maybe just one folder can't be selected or something
+                        warn!(ctx, "{}", err);
+                    }
+                    Ok(true) => {
+                        // Fetch the watched folder again in case scanning other folder moved messages
+                        // there.
+                        //
+                        // In most cases this will select the watched folder and return because there are
+                        // no new messages. We want to select the watched folder anyway before going IDLE
+                        // there, so this does not take additional protocol round-trip.
+                        if let Err(err) = connection
+                            .fetch_move_delete(ctx, &watch_folder, false)
+                            .await
+                        {
+                            connection.trigger_reconnect(ctx).await;
+                            warn!(ctx, "{:#}", err);
+                            return InterruptInfo::new(false);
+                        }
+                    }
+                    Ok(false) => {}
                 }
             }
 
-            // fetch
-            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
-                connection.trigger_reconnect(ctx).await;
-                warn!(ctx, "{:#}", err);
-                return InterruptInfo::new(false);
-            }
+            // Synchronize Seen flags.
+            connection
+                .sync_seen_flags(ctx, &watch_folder)
+                .await
+                .context("sync_seen_flags")
+                .ok_or_log(ctx);
 
             connection.connectivity.set_connected(ctx).await;
 
@@ -231,17 +259,16 @@ async fn simple_imap_loop(
     let ImapConnectionHandlers {
         mut connection,
         stop_receiver,
-        shutdown_sender,
     } = inbox_handlers;
 
     let ctx1 = ctx.clone();
 
     let fut = async move {
-        started
-            .send(())
-            .await
-            .expect("simple imap loop, missing started receive");
         let ctx = ctx1;
+        if let Err(err) = started.send(()).await {
+            warn!(&ctx, "simple imap loop, missing started receiver: {}", err);
+            return;
+        }
 
         loop {
             fetch_idle(&ctx, &mut connection, folder).await;
@@ -255,10 +282,6 @@ async fn simple_imap_loop(
         })
         .race(fut)
         .await;
-    shutdown_sender
-        .send(())
-        .await
-        .expect("simple imap loop, missing shutdown receiver");
 }
 
 async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnectionHandlers) {
@@ -268,18 +291,18 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
     let SmtpConnectionHandlers {
         mut connection,
         stop_receiver,
-        shutdown_sender,
         idle_interrupt_receiver,
     } = smtp_handlers;
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started
-            .send(())
-            .await
-            .expect("smtp loop, missing started receiver");
         let ctx = ctx1;
+        if let Err(err) = started.send(()).await {
+            warn!(&ctx, "smtp loop, missing started receiver: {}", err);
+            return;
+        }
 
+        let mut timeout = None;
         let mut interrupt_info = Default::default();
         loop {
             let job = match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
@@ -297,9 +320,16 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
                     interrupt_info = Default::default();
                 }
                 None => {
-                    if let Err(err) = send_smtp_messages(&ctx, &mut connection).await {
+                    let res = send_smtp_messages(&ctx, &mut connection).await;
+                    if let Err(err) = &res {
                         warn!(ctx, "send_smtp_messages failed: {:#}", err);
                     }
+                    let success = res.unwrap_or(false);
+                    timeout = if success {
+                        None
+                    } else {
+                        Some(timeout.map_or(30, |timeout: u64| timeout.saturating_mul(3)))
+                    };
 
                     // Fake Idle
                     info!(ctx, "smtp fake idle - started");
@@ -308,7 +338,27 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
                         Some(err) => connection.connectivity.set_err(&ctx, err).await,
                     }
 
-                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    // If send_smtp_messages() failed, we set a timeout for the fake-idle so that
+                    // sending is retried (at the latest) after the timeout. If sending fails
+                    // again, we increase the timeout exponentially, in order not to do lots of
+                    // unnecessary retries.
+                    if let Some(timeout) = timeout {
+                        info!(
+                            ctx,
+                            "smtp has messages to retry, planning to retry {} seconds later",
+                            timeout
+                        );
+                        let duration = std::time::Duration::from_secs(timeout);
+                        interrupt_info = async_std::future::timeout(duration, async {
+                            idle_interrupt_receiver.recv().await.unwrap_or_default()
+                        })
+                        .await
+                        .unwrap_or_default();
+                    } else {
+                        info!(ctx, "smtp has no messages to retry, waiting for interrupt");
+                        interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    };
+
                     info!(ctx, "smtp fake idle - interrupted")
                 }
             }
@@ -322,15 +372,15 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
         })
         .race(fut)
         .await;
-    shutdown_sender
-        .send(())
-        .await
-        .expect("smtp loop, missing shutdown receiver");
 }
 
 impl Scheduler {
-    /// Start the scheduler, panics if it is already running.
+    /// Start the scheduler, returns error if it is already running.
     pub async fn start(&mut self, ctx: Context) -> Result<()> {
+        if self.is_running() {
+            bail!("scheduler is already stopped");
+        }
+
         let (mvbox, mvbox_handlers) = ImapConnectionState::new(&ctx).await?;
         let (sentbox, sentbox_handlers) = ImapConnectionState::new(&ctx).await?;
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
@@ -342,6 +392,8 @@ impl Scheduler {
         let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1);
         let mut sentbox_handle = None;
         let (smtp_start_send, smtp_start_recv) = channel::bounded(1);
+        let (ephemeral_interrupt_send, ephemeral_interrupt_recv) = channel::bounded(1);
+        let (location_interrupt_send, location_interrupt_recv) = channel::bounded(1);
 
         let inbox_handle = {
             let ctx = ctx.clone();
@@ -350,7 +402,7 @@ impl Scheduler {
             }))
         };
 
-        if ctx.get_config_bool(Config::MvboxMove).await? {
+        if ctx.should_watch_mvbox().await? {
             let ctx = ctx.clone();
             mvbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
@@ -365,7 +417,7 @@ impl Scheduler {
             mvbox_start_send
                 .send(())
                 .await
-                .expect("mvbox start send, missing receiver");
+                .context("mvbox start send, missing receiver")?;
             mvbox_handlers
                 .connection
                 .connectivity
@@ -388,7 +440,7 @@ impl Scheduler {
             sentbox_start_send
                 .send(())
                 .await
-                .expect("sentbox start send, missing receiver");
+                .context("sentbox start send, missing receiver")?;
             sentbox_handlers
                 .connection
                 .connectivity
@@ -403,6 +455,20 @@ impl Scheduler {
             }))
         };
 
+        let ephemeral_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                ephemeral::ephemeral_loop(&ctx, ephemeral_interrupt_recv).await;
+            }))
+        };
+
+        let location_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                location::location_loop(&ctx, location_interrupt_recv).await;
+            }))
+        };
+
         *self = Scheduler::Running {
             inbox,
             mvbox,
@@ -412,6 +478,10 @@ impl Scheduler {
             mvbox_handle,
             sentbox_handle,
             smtp_handle,
+            ephemeral_handle,
+            ephemeral_interrupt_send,
+            location_handle,
+            location_interrupt_send,
         };
 
         // wait for all loops to be started
@@ -477,11 +547,31 @@ impl Scheduler {
         }
     }
 
-    /// Halts the scheduler, must be called first, and then `stop`.
-    pub(crate) async fn pre_stop(&self) -> StopToken {
+    async fn interrupt_ephemeral_task(&self) {
+        if let Scheduler::Running {
+            ref ephemeral_interrupt_send,
+            ..
+        } = self
+        {
+            ephemeral_interrupt_send.try_send(()).ok();
+        }
+    }
+
+    async fn interrupt_location(&self) {
+        if let Scheduler::Running {
+            ref location_interrupt_send,
+            ..
+        } = self
+        {
+            location_interrupt_send.try_send(()).ok();
+        }
+    }
+
+    /// Halt the scheduler.
+    pub(crate) async fn stop(&mut self) -> Result<()> {
         match self {
             Scheduler::Stopped => {
-                panic!("WARN: already stopped");
+                bail!("scheduler is already stopped");
             }
             Scheduler::Running {
                 inbox,
@@ -492,39 +582,23 @@ impl Scheduler {
                 sentbox_handle,
                 smtp,
                 smtp_handle,
+                ephemeral_handle,
+                location_handle,
                 ..
             } => {
                 if inbox_handle.is_some() {
-                    inbox.stop().await;
+                    inbox.stop().await?;
                 }
                 if mvbox_handle.is_some() {
-                    mvbox.stop().await;
+                    mvbox.stop().await?;
                 }
                 if sentbox_handle.is_some() {
-                    sentbox.stop().await;
+                    sentbox.stop().await?;
                 }
                 if smtp_handle.is_some() {
-                    smtp.stop().await;
+                    smtp.stop().await?;
                 }
 
-                StopToken
-            }
-        }
-    }
-
-    /// Halt the scheduler, must only be called after pre_stop.
-    pub(crate) async fn stop(&mut self, _t: StopToken) {
-        match self {
-            Scheduler::Stopped => {
-                panic!("WARN: already stopped");
-            }
-            Scheduler::Running {
-                inbox_handle,
-                mvbox_handle,
-                sentbox_handle,
-                smtp_handle,
-                ..
-            } => {
                 if let Some(handle) = inbox_handle.take() {
                     handle.await;
                 }
@@ -537,8 +611,15 @@ impl Scheduler {
                 if let Some(handle) = smtp_handle.take() {
                     handle.await;
                 }
+                if let Some(handle) = ephemeral_handle.take() {
+                    handle.cancel().await;
+                }
+                if let Some(handle) = location_handle.take() {
+                    handle.cancel().await;
+                }
 
                 *self = Scheduler::Stopped;
+                Ok(())
             }
         }
     }
@@ -552,8 +633,6 @@ impl Scheduler {
 /// Connection state logic shared between imap and smtp connections.
 #[derive(Debug)]
 struct ConnectionState {
-    /// Channel to notify that shutdown has completed.
-    shutdown_receiver: Receiver<()>,
     /// Channel to interrupt the whole connection.
     stop_sender: Sender<()>,
     /// Channel to interrupt idle.
@@ -564,14 +643,13 @@ struct ConnectionState {
 
 impl ConnectionState {
     /// Shutdown this connection completely.
-    async fn stop(&self) {
+    async fn stop(&self) -> Result<()> {
         // Trigger shutdown of the run loop.
         self.stop_sender
             .send(())
             .await
-            .expect("stop, missing receiver");
-        // Wait for a notification that the run loop has been shutdown.
-        self.shutdown_receiver.recv().await.ok();
+            .context("failed to stop, missing receiver")?;
+        Ok(())
     }
 
     async fn interrupt(&self, info: InterruptInfo) {
@@ -588,18 +666,15 @@ pub(crate) struct SmtpConnectionState {
 impl SmtpConnectionState {
     fn new() -> (Self, SmtpConnectionHandlers) {
         let (stop_sender, stop_receiver) = channel::bounded(1);
-        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
         let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = SmtpConnectionHandlers {
             connection: Smtp::new(),
             stop_receiver,
-            shutdown_sender,
             idle_interrupt_receiver,
         };
 
         let state = ConnectionState {
-            shutdown_receiver,
             stop_sender,
             idle_interrupt_sender,
             connectivity: handlers.connection.connectivity.clone(),
@@ -616,15 +691,15 @@ impl SmtpConnectionState {
     }
 
     /// Shutdown this connection completely.
-    async fn stop(&self) {
-        self.state.stop().await;
+    async fn stop(&self) -> Result<()> {
+        self.state.stop().await?;
+        Ok(())
     }
 }
 
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,
-    shutdown_sender: Sender<()>,
     idle_interrupt_receiver: Receiver<InterruptInfo>,
 }
 
@@ -637,17 +712,14 @@ impl ImapConnectionState {
     /// Construct a new connection.
     async fn new(context: &Context) -> Result<(Self, ImapConnectionHandlers)> {
         let (stop_sender, stop_receiver) = channel::bounded(1);
-        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
         let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = ImapConnectionHandlers {
             connection: Imap::new_configured(context, idle_interrupt_receiver).await?,
             stop_receiver,
-            shutdown_sender,
         };
 
         let state = ConnectionState {
-            shutdown_receiver,
             stop_sender,
             idle_interrupt_sender,
             connectivity: handlers.connection.connectivity.clone(),
@@ -664,8 +736,9 @@ impl ImapConnectionState {
     }
 
     /// Shutdown this connection completely.
-    async fn stop(&self) {
-        self.state.stop().await;
+    async fn stop(&self) -> Result<()> {
+        self.state.stop().await?;
+        Ok(())
     }
 }
 
@@ -673,7 +746,6 @@ impl ImapConnectionState {
 struct ImapConnectionHandlers {
     connection: Imap,
     stop_receiver: Receiver<()>,
-    shutdown_sender: Sender<()>,
 }
 
 #[derive(Default, Debug)]

src/scheduler/connectivity.rs

@@ -5,6 +5,7 @@ use async_std::sync::{Mutex, RwLockReadGuard};
 
 use crate::dc_tools::time;
 use crate::events::EventType;
+use crate::imap::scan_folders::get_watched_folder_configs;
 use crate::quota::{
     QUOTA_ERROR_THRESHOLD_PERCENTAGE, QUOTA_MAX_AGE_SECONDS, QUOTA_WARN_THRESHOLD_PERCENTAGE,
 };
@@ -303,7 +304,7 @@ impl Context {
             <html>
             <head>
                 <meta charset="UTF-8" />
-                <meta name="viewport" content="initial-scale=1.0" />
+                <meta name="viewport" content="initial-scale=1.0; user-scalable=no" />
                 <style>
                     ul {
                         list-style-type: none;
@@ -362,17 +363,14 @@ impl Context {
                 [
                     (
                         Config::ConfiguredInboxFolder,
-                        None,
                         inbox.state.connectivity.clone(),
                     ),
                     (
                         Config::ConfiguredMvboxFolder,
-                        Some(Config::MvboxMove),
                         mvbox.state.connectivity.clone(),
                     ),
                     (
                         Config::ConfiguredSentboxFolder,
-                        Some(Config::SentboxWatch),
                         sentbox.state.connectivity.clone(),
                     ),
                 ],
@@ -391,20 +389,12 @@ impl Context {
         //                               - "Sent": Connected
         // =============================================================================================
 
+        let watched_folders = get_watched_folder_configs(self).await?;
         ret += &format!("<h3>{}</h3><ul>", stock_str::incoming_messages(self).await);
-        for (folder, watch, state) in &folders_states {
-            let w = if let Some(watch_config) = *watch {
-                self.get_config(watch_config)
-                    .await
-                    .ok_or_log(self)
-                    .flatten()
-                    == Some("1".to_string())
-            } else {
-                true
-            };
-
+        for (folder, state) in &folders_states {
             let mut folder_added = false;
-            if w {
+
+            if watched_folders.contains(folder) {
                 let f = self.get_config(*folder).await.ok_or_log(self).flatten();
 
                 if let Some(foldername) = f {
@@ -459,13 +449,7 @@ impl Context {
         //                                [======67%=====       ]
         // =============================================================================================
 
-        let domain = dc_tools::EmailAddress::new(
-            &self
-                .get_config(Config::ConfiguredAddr)
-                .await?
-                .unwrap_or_default(),
-        )?
-        .domain;
+        let domain = dc_tools::EmailAddress::new(&self.get_primary_self_addr().await?)?.domain;
         ret += &format!(
             "<h3>{}</h3><ul>",
             stock_str::storage_on_domain(self, domain).await
@@ -554,7 +538,7 @@ impl Context {
                 self.schedule_quota_update().await?;
             }
         } else {
-            ret += &format!("<li>{}</li>", stock_str::one_moment(self).await);
+            ret += &format!("<li>{}</li>", stock_str::not_connected(self).await);
             self.schedule_quota_update().await?;
         }
         ret += "</ul>";

src/securejoin.rs

@@ -3,21 +3,20 @@
 use std::convert::TryFrom;
 
 use anyhow::{bail, Context as _, Error, Result};
-use async_std::sync::Mutex;
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 
 use crate::aheader::EncryptPreference;
-use crate::chat::{self, is_contact_in_chat, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
+use crate::chat::{self, Chat, ChatId, ChatIdBlocked};
 use crate::config::Config;
-use crate::constants::{Blocked, Chattype, Viewtype, DC_CONTACT_ID_LAST_SPECIAL};
-use crate::contact::{Contact, Origin, VerifiedStatus};
+use crate::constants::Blocked;
+use crate::contact::{Contact, ContactId, Origin, VerifiedStatus};
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::e2ee::ensure_secret_key_exists;
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
-use crate::message::Message;
+use crate::message::{Message, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
 use crate::peerstate::{Peerstate, PeerstateKeyType, PeerstateVerifiedStatus, ToSave};
@@ -25,28 +24,16 @@ use crate::qr::check_qr;
 use crate::stock_str;
 use crate::token;
 
+mod bob;
 mod bobstate;
 mod qrinvite;
 
 use crate::token::Namespace;
-use bobstate::{BobHandshakeStage, BobState, BobStateHandle};
+use bobstate::BobState;
 use qrinvite::QrInvite;
 
 pub const NON_ALPHANUMERIC_WITHOUT_DOT: &AsciiSet = &NON_ALPHANUMERIC.remove(b'.');
 
-macro_rules! joiner_progress {
-    ($context:tt, $contact_id:expr, $progress:expr) => {
-        assert!(
-            $progress >= 0 && $progress <= 1000,
-            "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
-        );
-        $context.emit_event($crate::events::EventType::SecurejoinJoinerProgress {
-            contact_id: $contact_id,
-            progress: $progress,
-        });
-    };
-}
-
 macro_rules! inviter_progress {
     ($context:tt, $contact_id:expr, $progress:expr) => {
         assert!(
@@ -60,100 +47,6 @@ macro_rules! inviter_progress {
     };
 }
 
-/// State for setup-contact/secure-join protocol joiner's side, aka Bob's side.
-///
-/// The setup-contact protocol needs to carry state for both the inviter (Alice) and the
-/// joiner/invitee (Bob).  For Alice this state is minimal and in the `tokens` table in the
-/// database.  For Bob this state is only carried live on the [`Context`] in this struct.
-#[derive(Debug, Default)]
-pub(crate) struct Bob {
-    inner: Mutex<Option<BobState>>,
-}
-
-/// Return value for [`Bob::start_protocol`].
-///
-/// This indicates which protocol variant was started and provides the required information
-/// about it.
-enum StartedProtocolVariant {
-    /// The setup-contact protocol, to verify a contact.
-    SetupContact,
-    /// The secure-join protocol, to join a group.
-    SecureJoin {
-        group_id: String,
-        group_name: String,
-    },
-}
-
-impl Bob {
-    /// Starts the securejoin protocol with the QR `invite`.
-    ///
-    /// This will try to start the securejoin protocol for the given QR `invite`.  If it
-    /// succeeded the protocol state will be tracked in `self`.
-    ///
-    /// This function takes care of starting the "ongoing" mechanism if required and
-    /// handling errors while starting the protocol.
-    ///
-    /// # Returns
-    ///
-    /// If the started protocol is joining a group the returned struct contains information
-    /// about the group and ongoing process.
-    async fn start_protocol(
-        &self,
-        context: &Context,
-        invite: QrInvite,
-    ) -> Result<StartedProtocolVariant, JoinError> {
-        let mut guard = self.inner.lock().await;
-        if guard.is_some() {
-            warn!(context, "The new securejoin will replace the ongoing one.");
-            *guard = None;
-        }
-        let variant = match invite {
-            QrInvite::Group {
-                ref grpid,
-                ref name,
-                ..
-            } => StartedProtocolVariant::SecureJoin {
-                group_id: grpid.clone(),
-                group_name: name.clone(),
-            },
-            _ => StartedProtocolVariant::SetupContact,
-        };
-        match BobState::start_protocol(context, invite).await {
-            Ok((state, stage)) => {
-                if matches!(stage, BobHandshakeStage::RequestWithAuthSent) {
-                    joiner_progress!(context, state.invite().contact_id(), 400);
-                }
-                *guard = Some(state);
-                Ok(variant)
-            }
-            Err(err) => {
-                if let StartedProtocolVariant::SecureJoin { .. } = variant {
-                    context.free_ongoing().await;
-                }
-                Err(err)
-            }
-        }
-    }
-
-    /// Returns a handle to the [`BobState`] of the handshake.
-    ///
-    /// If there currently isn't a handshake running this will return `None`.  Otherwise
-    /// this will return a handle to the current [`BobState`].  This handle allows
-    /// processing an incoming message and allows terminating the handshake.
-    ///
-    /// The handle contains an exclusive lock, which is held until the handle is dropped.
-    /// This guarantees all state and state changes are correct and allows safely
-    /// terminating the handshake without worrying about concurrency.
-    async fn state(&self, context: &Context) -> Option<BobStateHandle<'_>> {
-        let guard = self.inner.lock().await;
-        let ret = BobStateHandle::from_guard(guard);
-        if ret.is_none() {
-            info!(context, "No active BobState found for securejoin handshake");
-        }
-        ret
-    }
-}
-
 /// Generates a Secure Join QR code.
 ///
 /// With `group` set to `None` this generates a setup-contact QR code, with `group` set to a
@@ -173,19 +66,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group: Option<ChatId>) -> R
         .is_none();
     let invitenumber = token::lookup_or_new(context, Namespace::InviteNumber, group).await;
     let auth = token::lookup_or_new(context, Namespace::Auth, group).await;
-    let self_addr = match context.get_config(Config::ConfiguredAddr).await {
-        Ok(Some(addr)) => addr,
-        Ok(None) => {
-            bail!("Not configured, cannot generate QR code.");
-        }
-        Err(err) => {
-            bail!(
-                "Unable to retrieve configuration, cannot generate QR code: {:?}",
-                err
-            );
-        }
-    };
-
+    let self_addr = context.get_primary_self_addr().await?;
     let self_name = context
         .get_config(Config::Displayname)
         .await?
@@ -206,6 +87,12 @@ pub async fn dc_get_securejoin_qr(context: &Context, group: Option<ChatId>) -> R
     let qr = if let Some(group) = group {
         // parameters used: a=g=x=i=s=
         let chat = Chat::load_from_db(context, group).await?;
+        if chat.grpid.is_empty() {
+            bail!(
+                "can't generate securejoin QR code for ad-hoc group {}",
+                group
+            );
+        }
         let group_name = chat.get_name();
         let group_name_urlencoded = utf8_percent_encode(group_name, NON_ALPHANUMERIC).to_string();
         if sync_token {
@@ -280,7 +167,7 @@ pub enum JoinError {
 pub async fn dc_join_securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
     securejoin(context, qr).await.map_err(|err| {
         warn!(context, "Fatal joiner error: {:#}", err);
-        // This is a modal operation, the user has context on what failed.
+        // The user just scanned this QR code so has context on what failed.
         error!(context, "QR process failed");
         err
     })
@@ -297,47 +184,7 @@ async fn securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
 
     let invite = QrInvite::try_from(qr_scan)?;
 
-    match context.bob.start_protocol(context, invite.clone()).await? {
-        StartedProtocolVariant::SetupContact => {
-            // for a one-to-one-chat, the chat is already known, return the chat-id,
-            // the verification runs in background
-            let chat_id = ChatId::create_for_contact(context, invite.contact_id())
-                .await
-                .map_err(JoinError::UnknownContact)?;
-            Ok(chat_id)
-        }
-        StartedProtocolVariant::SecureJoin {
-            group_id,
-            group_name,
-        } => {
-            // for a group-join, also create the chat soon and let the verification run in background.
-            // however, the group will become usable only when the protocol has finished.
-            let contact_id = invite.contact_id();
-            let chat_id = if let Some((chat_id, _protected, _blocked)) =
-                chat::get_chat_id_by_grpid(context, &group_id).await?
-            {
-                chat_id.unblock(context).await?;
-                chat_id
-            } else {
-                ChatId::create_multiuser_record(
-                    context,
-                    Chattype::Group,
-                    &group_id,
-                    &group_name,
-                    Blocked::Not,
-                    ProtectionStatus::Unprotected, // protection is added later as needed
-                    None,
-                )
-                .await?
-            };
-            if !is_contact_in_chat(context, chat_id, contact_id).await? {
-                chat::add_to_chat_contacts_table(context, chat_id, contact_id).await?;
-            }
-            let msg = stock_str::secure_join_started(context, contact_id).await;
-            chat::add_info_msg(context, chat_id, &msg, time()).await?;
-            Ok(chat_id)
-        }
-    }
+    bob::start_protocol(context, invite).await
 }
 
 /// Error when failing to send a protocol handshake message.
@@ -352,7 +199,7 @@ pub struct SendMsgError(#[from] anyhow::Error);
 /// Bob's handshake messages are sent in `BobState::send_handshake_message()`.
 async fn send_alice_handshake_msg(
     context: &Context,
-    contact_id: u32,
+    contact_id: ContactId,
     step: &str,
     fingerprint: Option<Fingerprint>,
 ) -> Result<(), SendMsgError> {
@@ -380,7 +227,7 @@ async fn send_alice_handshake_msg(
 }
 
 /// Get an unblocked chat that can be used for info messages.
-async fn info_chat_id(context: &Context, contact_id: u32) -> Result<ChatId> {
+async fn info_chat_id(context: &Context, contact_id: ContactId) -> Result<ChatId> {
     let chat_id_blocked = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Not).await?;
     Ok(chat_id_blocked.id)
 }
@@ -388,7 +235,7 @@ async fn info_chat_id(context: &Context, contact_id: u32) -> Result<ChatId> {
 async fn fingerprint_equals_sender(
     context: &Context,
     fingerprint: &Fingerprint,
-    contact_id: u32,
+    contact_id: ContactId,
 ) -> Result<bool, Error> {
     let contact = Contact::load_from_db(context, contact_id).await?;
     let peerstate = match Peerstate::from_addr(context, contact.get_addr()).await {
@@ -442,9 +289,8 @@ pub(crate) enum HandshakeMessage {
 
 /// Handle incoming secure-join handshake.
 ///
-/// This function will update the securejoin state in [`InnerContext::bob`] and also
-/// terminate the ongoing process using [`Context::stop_ongoing`] as required by the
-/// protocol.
+/// This function will update the securejoin state in the database as the protocol
+/// progresses.
 ///
 /// A message which results in [`Err`] will be hidden from the user but not deleted, it may
 /// be a valid message for something else we are not aware off.  E.g. it could be part of a
@@ -452,15 +298,13 @@ pub(crate) enum HandshakeMessage {
 ///
 /// When `handle_securejoin_handshake()` is called, the message is not yet filed in the
 /// database; this is done by `receive_imf()` later on as needed.
-///
-/// [`InnerContext::bob`]: crate::context::InnerContext::bob
 #[allow(clippy::indexing_slicing)]
 pub(crate) async fn handle_securejoin_handshake(
     context: &Context,
     mime_message: &MimeMessage,
-    contact_id: u32,
+    contact_id: ContactId,
 ) -> Result<HandshakeMessage> {
-    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+    if contact_id.is_special() {
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
@@ -521,38 +365,7 @@ pub(crate) async fn handle_securejoin_handshake(
             ====             Bob - the joiner's side             =====
             ====   Step 4 in "Setup verified contact" protocol   =====
             ========================================================*/
-            match context.bob.state(context).await {
-                Some(mut bobstate) => match bobstate.handle_message(context, mime_message).await {
-                    Some(BobHandshakeStage::Terminated(why)) => {
-                        could_not_establish_secure_connection(
-                            context,
-                            contact_id,
-                            bobstate.chat_id(context).await?,
-                            why,
-                        )
-                        .await?;
-                        Ok(HandshakeMessage::Done)
-                    }
-                    Some(_stage) => {
-                        if join_vg {
-                            // the message reads "Alice replied, waiting for being added to the group…";
-                            // show it only on secure-join and not on setup-contact therefore.
-                            let msg = stock_str::secure_join_replies(context, contact_id).await;
-                            chat::add_info_msg(
-                                context,
-                                bobstate.chat_id(context).await?,
-                                &msg,
-                                time(),
-                            )
-                            .await?;
-                        }
-                        joiner_progress!(context, bobstate.invite().contact_id(), 400);
-                        Ok(HandshakeMessage::Done)
-                    }
-                    None => Ok(HandshakeMessage::Ignore),
-                },
-                None => Ok(HandshakeMessage::Ignore),
-            }
+            bob::handle_auth_required(context, mime_message).await
         }
         "vg-request-with-auth" | "vc-request-with-auth" => {
             /*==========================================================
@@ -683,44 +496,14 @@ pub(crate) async fn handle_securejoin_handshake(
             ====             Bob - the joiner's side             ====
             ====   Step 7 in "Setup verified contact" protocol   ====
             =======================================================*/
-            info!(context, "matched vc-contact-confirm step");
-            let retval = if join_vg {
-                HandshakeMessage::Propagate
-            } else {
-                HandshakeMessage::Ignore
-            };
-            match context.bob.state(context).await {
-                Some(mut bobstate) => match bobstate.handle_message(context, mime_message).await {
-                    Some(BobHandshakeStage::Terminated(why)) => {
-                        could_not_establish_secure_connection(
-                            context,
-                            contact_id,
-                            bobstate.chat_id(context).await?,
-                            why,
-                        )
-                        .await?;
-                        Ok(HandshakeMessage::Done)
-                    }
-                    Some(BobHandshakeStage::Completed) => {
-                        // Can only be BobHandshakeStage::Completed
-                        secure_connection_established(
-                            context,
-                            contact_id,
-                            bobstate.chat_id(context).await?,
-                        )
-                        .await?;
-                        Ok(retval)
-                    }
-                    Some(_) => {
-                        warn!(
-                            context,
-                            "Impossible state returned from handling handshake message"
-                        );
-                        Ok(retval)
-                    }
-                    None => Ok(retval),
+            match BobState::from_db(&context.sql).await? {
+                Some(bobstate) => {
+                    bob::handle_contact_confirm(context, bobstate, mime_message).await
+                }
+                None => match join_vg {
+                    true => Ok(HandshakeMessage::Propagate),
+                    false => Ok(HandshakeMessage::Ignore),
                 },
-                None => Ok(retval),
             }
         }
         "vg-member-added-received" | "vc-contact-confirm-received" => {
@@ -782,9 +565,9 @@ pub(crate) async fn handle_securejoin_handshake(
 pub(crate) async fn observe_securejoin_on_other_device(
     context: &Context,
     mime_message: &MimeMessage,
-    contact_id: u32,
+    contact_id: ContactId,
 ) -> Result<HandshakeMessage> {
-    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+    if contact_id.is_special() {
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
@@ -847,11 +630,11 @@ pub(crate) async fn observe_securejoin_on_other_device(
 
 async fn secure_connection_established(
     context: &Context,
-    contact_id: u32,
+    contact_id: ContactId,
     chat_id: ChatId,
 ) -> Result<(), Error> {
     let contact = Contact::get_by_id(context, contact_id).await?;
-    let msg = stock_str::contact_verified(context, contact.get_name_n_addr()).await;
+    let msg = stock_str::contact_verified(context, &contact).await;
     chat::add_info_msg(context, chat_id, &msg, time()).await?;
     context.emit_event(EventType::ChatModified(chat_id));
     Ok(())
@@ -859,20 +642,12 @@ async fn secure_connection_established(
 
 async fn could_not_establish_secure_connection(
     context: &Context,
-    contact_id: u32,
+    contact_id: ContactId,
     chat_id: ChatId,
     details: &str,
 ) -> Result<(), Error> {
-    let contact = Contact::get_by_id(context, contact_id).await;
-    let msg = stock_str::contact_not_verified(
-        context,
-        if let Ok(ref contact) = contact {
-            contact.get_addr()
-        } else {
-            "?"
-        },
-    )
-    .await;
+    let contact = Contact::get_by_id(context, contact_id).await?;
+    let msg = stock_str::contact_not_verified(context, &contact).await;
     chat::add_info_msg(context, chat_id, &msg, time()).await?;
     warn!(
         context,
@@ -941,31 +716,36 @@ mod tests {
     use crate::chat::ProtectionStatus;
     use crate::chatlist::Chatlist;
     use crate::constants::Chattype;
+    use crate::dc_receive_imf::dc_receive_imf;
     use crate::peerstate::Peerstate;
-    use crate::test_utils::{LogSink, TestContext};
+    use crate::test_utils::{TestContext, TestContextManager};
 
     #[async_std::test]
-    async fn test_setup_contact() -> Result<()> {
-        let (log_tx, _log_sink) = LogSink::create();
-        let alice = TestContext::builder()
-            .configure_alice()
-            .with_log_sink(log_tx.clone())
-            .build()
-            .await;
-        let bob = TestContext::builder()
-            .configure_bob()
-            .with_log_sink(log_tx)
-            .build()
-            .await;
-        assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 0);
-        assert_eq!(Chatlist::try_load(&bob, 0, None, None).await?.len(), 0);
+    async fn test_setup_contact() {
+        let mut tcm = TestContextManager::new().await;
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+        assert_eq!(
+            Chatlist::try_load(&alice, 0, None, None)
+                .await
+                .unwrap()
+                .len(),
+            0
+        );
+        assert_eq!(
+            Chatlist::try_load(&bob, 0, None, None).await.unwrap().len(),
+            0
+        );
 
         // Step 1: Generate QR-code, ChatId(0) indicates setup-contact
-        let qr = dc_get_securejoin_qr(&alice.ctx, None).await?;
+        let qr = dc_get_securejoin_qr(&alice.ctx, None).await.unwrap();
 
         // Step 2: Bob scans QR-code, sends vc-request
-        dc_join_securejoin(&bob.ctx, &qr).await?;
-        assert_eq!(Chatlist::try_load(&bob, 0, None, None).await?.len(), 1);
+        dc_join_securejoin(&bob.ctx, &qr).await.unwrap();
+        assert_eq!(
+            Chatlist::try_load(&bob, 0, None, None).await.unwrap().len(),
+            1
+        );
 
         let sent = bob.pop_sent_msg().await;
         assert!(!bob.ctx.has_ongoing().await);
@@ -977,7 +757,13 @@ mod tests {
 
         // Step 3: Alice receives vc-request, sends vc-auth-required
         alice.recv_msg(&sent).await;
-        assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 1);
+        assert_eq!(
+            Chatlist::try_load(&alice, 0, None, None)
+                .await
+                .unwrap()
+                .len(),
+            1
+        );
 
         let sent = alice.pop_sent_msg().await;
         let msg = bob.parse_msg(&sent).await;
@@ -1039,21 +825,30 @@ mod tests {
             .await
             .unwrap();
         assert_eq!(
-            contact_bob.is_verified(&alice.ctx).await?,
+            contact_bob.is_verified(&alice.ctx).await.unwrap(),
             VerifiedStatus::Unverified
         );
 
         // Step 5+6: Alice receives vc-request-with-auth, sends vc-contact-confirm
         alice.recv_msg(&sent).await;
         assert_eq!(
-            contact_bob.is_verified(&alice.ctx).await?,
+            contact_bob.is_verified(&alice.ctx).await.unwrap(),
             VerifiedStatus::BidirectVerified
         );
 
         // exactly one one-to-one chat should be visible for both now
         // (check this before calling alice.create_chat() explicitly below)
-        assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 1);
-        assert_eq!(Chatlist::try_load(&bob, 0, None, None).await?.len(), 1);
+        assert_eq!(
+            Chatlist::try_load(&alice, 0, None, None)
+                .await
+                .unwrap()
+                .len(),
+            1
+        );
+        assert_eq!(
+            Chatlist::try_load(&bob, 0, None, None).await.unwrap().len(),
+            1
+        );
 
         // Check Alice got the verified message in her 1:1 chat.
         {
@@ -1093,14 +888,14 @@ mod tests {
             .await
             .unwrap();
         assert_eq!(
-            contact_bob.is_verified(&bob.ctx).await?,
+            contact_bob.is_verified(&bob.ctx).await.unwrap(),
             VerifiedStatus::Unverified
         );
 
         // Step 7: Bob receives vc-contact-confirm, sends vc-contact-confirm-received
         bob.recv_msg(&sent).await;
         assert_eq!(
-            contact_alice.is_verified(&bob.ctx).await?,
+            contact_alice.is_verified(&bob.ctx).await.unwrap(),
             VerifiedStatus::BidirectVerified
         );
 
@@ -1131,7 +926,6 @@ mod tests {
             msg.get_header(HeaderDef::SecureJoin).unwrap(),
             "vc-contact-confirm-received"
         );
-        Ok(())
     }
 
     #[async_std::test]
@@ -1143,17 +937,9 @@ mod tests {
 
     #[async_std::test]
     async fn test_setup_contact_bob_knows_alice() -> Result<()> {
-        let (log_tx, _log_sink) = LogSink::create();
-        let alice = TestContext::builder()
-            .configure_alice()
-            .with_log_sink(log_tx.clone())
-            .build()
-            .await;
-        let bob = TestContext::builder()
-            .configure_bob()
-            .with_log_sink(log_tx)
-            .build()
-            .await;
+        let mut tcm = TestContextManager::new().await;
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
 
         // Ensure Bob knows Alice_FP
         let alice_pubkey = SignedPublicKey::load_self(&alice.ctx).await?;
@@ -1276,17 +1062,9 @@ mod tests {
 
     #[async_std::test]
     async fn test_setup_contact_concurrent_calls() -> Result<()> {
-        let (log_tx, _log_sink) = LogSink::create();
-        let alice = TestContext::builder()
-            .configure_alice()
-            .with_log_sink(log_tx.clone())
-            .build()
-            .await;
-        let bob = TestContext::builder()
-            .configure_bob()
-            .with_log_sink(log_tx)
-            .build()
-            .await;
+        let mut tcm = TestContextManager::new().await;
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
 
         // do a scan that is not working as claire is never responding
         let qr_stale = "OPENPGP4FPR:1234567890123456789012345678901234567890#a=claire%40foo.de&n=&i=12345678901&s=23456789012";
@@ -1315,25 +1093,19 @@ mod tests {
 
     #[async_std::test]
     async fn test_secure_join() -> Result<()> {
-        let (log_tx, _log_sink) = LogSink::create();
-        let alice = TestContext::builder()
-            .configure_alice()
-            .with_log_sink(log_tx.clone())
-            .build()
-            .await;
-        let bob = TestContext::builder()
-            .configure_bob()
-            .with_log_sink(log_tx)
-            .build()
-            .await;
+        let mut tcm = TestContextManager::new().await;
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+
+        // We start with empty chatlists.
         assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 0);
         assert_eq!(Chatlist::try_load(&bob, 0, None, None).await?.len(), 0);
 
-        let chatid =
+        let alice_chatid =
             chat::create_group_chat(&alice.ctx, ProtectionStatus::Protected, "the chat").await?;
 
         // Step 1: Generate QR-code, secure-join implied by chatid
-        let qr = dc_get_securejoin_qr(&alice.ctx, Some(chatid))
+        let qr = dc_get_securejoin_qr(&alice.ctx, Some(alice_chatid))
             .await
             .unwrap();
 
@@ -1424,6 +1196,35 @@ mod tests {
             "vg-member-added"
         );
 
+        {
+            // Now Alice's chat with Bob should still be hidden, the verified message should
+            // appear in the group chat.
+
+            let chat = alice
+                .get_chat(&bob)
+                .await
+                .expect("Alice has no 1:1 chat with bob");
+            assert_eq!(
+                chat.blocked,
+                Blocked::Yes,
+                "Alice's 1:1 chat with Bob is not hidden"
+            );
+            let msg_id = chat::get_chat_msgs(&alice.ctx, alice_chatid, 0x1, None)
+                .await
+                .unwrap()
+                .into_iter()
+                .filter_map(|item| match item {
+                    chat::ChatItem::Message { msg_id } => Some(msg_id),
+                    _ => None,
+                })
+                .min()
+                .expect("No messages in Alice's group chat");
+            let msg = Message::load_from_db(&alice.ctx, msg_id).await.unwrap();
+            assert!(msg.is_info());
+            let text = msg.get_text().unwrap();
+            assert!(text.contains("bob@example.net verified"));
+        }
+
         // Bob should not yet have Alice verified
         let contact_alice_id =
             Contact::lookup_id_by_addr(&bob.ctx, "alice@example.org", Origin::Unknown)
@@ -1438,10 +1239,53 @@ mod tests {
 
         // Step 7: Bob receives vg-member-added, sends vg-member-added-received
         bob.recv_msg(&sent).await;
-        assert_eq!(
-            contact_alice.is_verified(&bob.ctx).await?,
-            VerifiedStatus::BidirectVerified
-        );
+        {
+            // Bob has Alice verified, message shows up in the group chat.
+            assert_eq!(
+                contact_alice.is_verified(&bob.ctx).await?,
+                VerifiedStatus::BidirectVerified
+            );
+            let chat = bob
+                .get_chat(&alice)
+                .await
+                .expect("Bob has no 1:1 chat with Alice");
+            assert_eq!(
+                chat.blocked,
+                Blocked::Yes,
+                "Bob's 1:1 chat with Alice is not hidden"
+            );
+            for item in chat::get_chat_msgs(&bob.ctx, bob_chatid, 0x1, None)
+                .await
+                .unwrap()
+            {
+                if let chat::ChatItem::Message { msg_id } = item {
+                    let msg = Message::load_from_db(&bob.ctx, msg_id).await.unwrap();
+                    let text = msg.get_text().unwrap();
+                    println!("msg {} text: {}", msg_id, text);
+                }
+            }
+            let mut msg_iter = chat::get_chat_msgs(&bob.ctx, bob_chatid, 0x1, None)
+                .await
+                .unwrap()
+                .into_iter();
+            loop {
+                match msg_iter.next() {
+                    Some(chat::ChatItem::Message { msg_id }) => {
+                        let msg = Message::load_from_db(&bob.ctx, msg_id).await.unwrap();
+                        let text = msg.get_text().unwrap();
+                        match text.contains("alice@example.org verified") {
+                            true => {
+                                assert!(msg.is_info());
+                                break;
+                            }
+                            false => continue,
+                        }
+                    }
+                    Some(_) => continue,
+                    None => panic!("Verified message not found in Bob's group chat"),
+                }
+            }
+        }
 
         let sent = bob.pop_sent_msg().await;
         let msg = alice.parse_msg(&sent).await;
@@ -1471,4 +1315,25 @@ mod tests {
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_adhoc_group_no_qr() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("2")).await?;
+
+        let mime = br#"Subject: First thread
+Message-ID: first@example.org
+To: Alice <alice@example.org>, Bob <bob@example.net>
+From: Claire <claire@example.org>
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+First thread."#;
+
+        dc_receive_imf(&alice, mime, false).await?;
+        let msg = alice.get_last_msg().await;
+        let chat_id = msg.chat_id;
+
+        assert!(dc_get_securejoin_qr(&alice, Some(chat_id)).await.is_err());
+        Ok(())
+    }
 }

src/securejoin/bob.rs

@@ -0,0 +1,257 @@
+//! Bob's side of SecureJoin handling.
+//!
+//! This are some helper functions around [`BobState`] which augment the state changes with
+//! the required user interactions.
+
+use anyhow::Result;
+
+use crate::chat::{is_contact_in_chat, ChatId, ProtectionStatus};
+use crate::constants::{Blocked, Chattype};
+use crate::contact::Contact;
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::events::EventType;
+use crate::mimeparser::MimeMessage;
+use crate::{chat, stock_str};
+
+use super::bobstate::{BobHandshakeStage, BobState};
+use super::qrinvite::QrInvite;
+use super::{HandshakeMessage, JoinError};
+
+/// Starts the securejoin protocol with the QR `invite`.
+///
+/// This will try to start the securejoin protocol for the given QR `invite`.  If it
+/// succeeded the protocol state will be tracked in `self`.
+///
+/// This function takes care of handling multiple concurrent joins and handling errors while
+/// starting the protocol.
+///
+/// # Returns
+///
+/// The [`ChatId`] of the created chat is returned, for a SetupContact QR this is the 1:1
+/// chat with Alice, for a SecureJoin QR this is the group chat.
+pub(super) async fn start_protocol(
+    context: &Context,
+    invite: QrInvite,
+) -> Result<ChatId, JoinError> {
+    // A 1:1 chat is needed to send messages to Alice.  When joining a group this chat is
+    // hidden, if a user starts sending messages in it it will be unhidden in
+    // dc_receive_imf.
+    let hidden = match invite {
+        QrInvite::Contact { .. } => Blocked::Not,
+        QrInvite::Group { .. } => Blocked::Yes,
+    };
+    let chat_id = ChatId::create_for_contact_with_blocked(context, invite.contact_id(), hidden)
+        .await
+        .map_err(JoinError::UnknownContact)?;
+
+    // Now start the protocol and initialise the state
+    let (state, stage, aborted_states) =
+        BobState::start_protocol(context, invite.clone(), chat_id).await?;
+    for state in aborted_states {
+        error!(context, "Aborting previously unfinished QR Join process.");
+        state.notify_aborted(context, "new QR scanned").await?;
+        state.emit_progress(context, JoinerProgress::Error);
+    }
+    if matches!(stage, BobHandshakeStage::RequestWithAuthSent) {
+        state.emit_progress(context, JoinerProgress::RequestWithAuthSent);
+    }
+    match invite {
+        QrInvite::Group { .. } => {
+            // For a secure-join we need to create the group and add the contact.  The group will
+            // only become usable once the protocol is finished.
+            // TODO: how does this group become usable?
+            let group_chat_id = state.joining_chat_id(context).await?;
+            if !is_contact_in_chat(context, group_chat_id, invite.contact_id()).await? {
+                chat::add_to_chat_contacts_table(context, group_chat_id, invite.contact_id())
+                    .await?;
+            }
+            let msg = stock_str::secure_join_started(context, invite.contact_id()).await;
+            chat::add_info_msg(context, group_chat_id, &msg, time()).await?;
+            Ok(group_chat_id)
+        }
+        QrInvite::Contact { .. } => {
+            // For setup-contact the BobState already ensured the 1:1 chat exists because it
+            // uses it to send the handshake messages.
+            Ok(state.alice_chat())
+        }
+    }
+}
+
+/// Handles `vc-auth-required` and `vg-auth-required` handshake messages.
+///
+/// # Bob - the joiner's side
+/// ## Step 4 in the "Setup Contact protocol"
+pub(super) async fn handle_auth_required(
+    context: &Context,
+    message: &MimeMessage,
+) -> Result<HandshakeMessage> {
+    match BobState::from_db(&context.sql).await? {
+        Some(mut bobstate) => match bobstate.handle_message(context, message).await? {
+            Some(BobHandshakeStage::Terminated(why)) => {
+                bobstate.notify_aborted(context, why).await?;
+                Ok(HandshakeMessage::Done)
+            }
+            Some(_stage) => {
+                if bobstate.is_join_group() {
+                    // The message reads "Alice replied, waiting to be added to the group…",
+                    // so only show it on secure-join and not on setup-contact.
+                    let contact_id = bobstate.invite().contact_id();
+                    let msg = stock_str::secure_join_replies(context, contact_id).await;
+                    let chat_id = bobstate.joining_chat_id(context).await?;
+                    chat::add_info_msg(context, chat_id, &msg, time()).await?;
+                }
+                bobstate.emit_progress(context, JoinerProgress::RequestWithAuthSent);
+                Ok(HandshakeMessage::Done)
+            }
+            None => Ok(HandshakeMessage::Ignore),
+        },
+        None => Ok(HandshakeMessage::Ignore),
+    }
+}
+
+/// Handles `vc-contact-confirm` and `vg-member-added` handshake messages.
+///
+/// # Bob - the joiner's side
+/// ## Step 4 in the "Setup Contact protocol"
+pub(super) async fn handle_contact_confirm(
+    context: &Context,
+    mut bobstate: BobState,
+    message: &MimeMessage,
+) -> Result<HandshakeMessage> {
+    let retval = if bobstate.is_join_group() {
+        HandshakeMessage::Propagate
+    } else {
+        HandshakeMessage::Ignore
+    };
+    match bobstate.handle_message(context, message).await? {
+        Some(BobHandshakeStage::Terminated(why)) => {
+            bobstate.notify_aborted(context, why).await?;
+            Ok(HandshakeMessage::Done)
+        }
+        Some(BobHandshakeStage::Completed) => {
+            // Note this goes to the 1:1 chat, as when joining a group we implicitly also
+            // verify both contacts (this could be a bug/security issue, see
+            // e.g. https://github.com/deltachat/deltachat-core-rust/issues/1177).
+            bobstate.notify_peer_verified(context).await?;
+            Ok(retval)
+        }
+        Some(_) => {
+            warn!(
+                context,
+                "Impossible state returned from handling handshake message"
+            );
+            Ok(retval)
+        }
+        None => Ok(retval),
+    }
+}
+
+/// Private implementations for user interactions about this [`BobState`].
+impl BobState {
+    fn is_join_group(&self) -> bool {
+        match self.invite() {
+            QrInvite::Contact { .. } => false,
+            QrInvite::Group { .. } => true,
+        }
+    }
+
+    fn emit_progress(&self, context: &Context, progress: JoinerProgress) {
+        let contact_id = self.invite().contact_id();
+        context.emit_event(EventType::SecurejoinJoinerProgress {
+            contact_id,
+            progress: progress.into(),
+        });
+    }
+
+    /// Returns the [`ChatId`] of the chat being joined.
+    ///
+    /// This is the chat in which you want to notify the user as well.
+    ///
+    /// When joining a group this is the [`ChatId`] of the group chat, when verifying a
+    /// contact this is the [`ChatId`] of the 1:1 chat.  The 1:1 chat is assumed to exist
+    /// because a [`BobState`] can not exist without, the group chat will be created if it
+    /// does not yet exist.
+    async fn joining_chat_id(&self, context: &Context) -> Result<ChatId> {
+        match self.invite() {
+            QrInvite::Contact { .. } => Ok(self.alice_chat()),
+            QrInvite::Group {
+                ref grpid,
+                ref name,
+                ..
+            } => {
+                let group_chat_id = match chat::get_chat_id_by_grpid(context, grpid).await? {
+                    Some((chat_id, _protected, _blocked)) => {
+                        chat_id.unblock(context).await?;
+                        chat_id
+                    }
+                    None => {
+                        ChatId::create_multiuser_record(
+                            context,
+                            Chattype::Group,
+                            grpid,
+                            name,
+                            Blocked::Not,
+                            ProtectionStatus::Unprotected, // protection is added later as needed
+                            None,
+                        )
+                        .await?
+                    }
+                };
+                Ok(group_chat_id)
+            }
+        }
+    }
+
+    /// Notifies the user that the SecureJoin was aborted.
+    ///
+    /// This creates an info message in the chat being joined.
+    async fn notify_aborted(&self, context: &Context, why: &str) -> Result<()> {
+        let contact = Contact::get_by_id(context, self.invite().contact_id()).await?;
+        let msg = stock_str::contact_not_verified(context, &contact).await;
+        let chat_id = self.joining_chat_id(context).await?;
+        chat::add_info_msg(context, chat_id, &msg, time()).await?;
+        warn!(
+            context,
+            "StockMessage::ContactNotVerified posted to joining chat ({})", why
+        );
+        Ok(())
+    }
+
+    /// Notifies the user that the SecureJoin peer is verified.
+    ///
+    /// This creates an info message in the chat being joined.
+    async fn notify_peer_verified(&self, context: &Context) -> Result<()> {
+        let contact = Contact::get_by_id(context, self.invite().contact_id()).await?;
+        let msg = stock_str::contact_verified(context, &contact).await;
+        let chat_id = self.joining_chat_id(context).await?;
+        chat::add_info_msg(context, chat_id, &msg, time()).await?;
+        context.emit_event(EventType::ChatModified(chat_id));
+        Ok(())
+    }
+}
+
+/// Progress updates for [`EventType::SecurejoinJoinerProgress`].
+///
+/// This has an `From<JoinerProgress> for usize` impl yielding numbers between 0 and a 1000
+/// which can be shown as a progress bar.
+enum JoinerProgress {
+    /// An error occurred.
+    Error,
+    /// vg-vc-request-with-auth sent.
+    ///
+    /// Typically shows as "alice@addr verified, introducing myself."
+    RequestWithAuthSent,
+    // /// Completed securejoin.
+    // Succeeded,
+}
+
+impl From<JoinerProgress> for usize {
+    fn from(progress: JoinerProgress) -> Self {
+        match progress {
+            JoinerProgress::Error => 0,
+            JoinerProgress::RequestWithAuthSent => 400,
+            // JoinerProgress::Succeeded => 1000,
+        }
+    }
+}

src/securejoin/bobstate.rs

@@ -5,22 +5,21 @@
 //! provides all the information to its driver so it can perform the correct interactions.
 //!
 //! The [`BobState`] is only directly used to initially create it when starting the
-//! protocol.  Afterwards it must be stored in a mutex and the [`BobStateHandle`] should be
-//! used to work with the state.
+//! protocol.
 
-use anyhow::{bail, Error, Result};
-use async_std::sync::MutexGuard;
+use anyhow::{Error, Result};
+use rusqlite::Connection;
 
 use crate::chat::{self, ChatId};
-use crate::constants::{Blocked, Viewtype};
 use crate::contact::{Contact, Origin};
 use crate::context::Context;
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, SignedPublicKey};
-use crate::message::Message;
+use crate::message::{Message, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
+use crate::sql::Sql;
 
 use super::qrinvite::QrInvite;
 use super::{
@@ -46,123 +45,14 @@ pub enum BobHandshakeStage {
     Terminated(&'static str),
 }
 
-/// A handle to work with the [`BobState`] of Bob's securejoin protocol.
+/// The securejoin state kept while Bob is joining.
 ///
-/// This handle can only be created for when an underlying [`BobState`] exists.  It keeps
-/// open a lock which guarantees unique access to the state and this struct must be dropped
-/// to return the lock.
-pub struct BobStateHandle<'a> {
-    guard: MutexGuard<'a, Option<BobState>>,
-    bobstate: BobState,
-    clear_state_on_drop: bool,
-}
-
-impl<'a> BobStateHandle<'a> {
-    /// Creates a new instance, upholding the guarantee that [`BobState`] must exist.
-    pub fn from_guard(mut guard: MutexGuard<'a, Option<BobState>>) -> Option<Self> {
-        guard.take().map(|bobstate| Self {
-            guard,
-            bobstate,
-            clear_state_on_drop: false,
-        })
-    }
-
-    /// Returns the [`ChatId`] of the group chat to join or the 1:1 chat with Alice.
-    pub async fn chat_id(&self, context: &Context) -> Result<ChatId> {
-        match self.bobstate.invite {
-            QrInvite::Group { ref grpid, .. } => {
-                if let Some((chat_id, _, _)) = chat::get_chat_id_by_grpid(context, grpid).await? {
-                    Ok(chat_id)
-                } else {
-                    bail!("chat not found")
-                }
-            }
-            QrInvite::Contact { .. } => Ok(self.bobstate.chat_id),
-        }
-    }
-
-    /// Returns a reference to the [`QrInvite`] of the joiner process.
-    pub fn invite(&self) -> &QrInvite {
-        &self.bobstate.invite
-    }
-
-    /// Handles the given message for the securejoin handshake for Bob.
-    ///
-    /// This proxies to [`BobState::handle_message`] and makes sure to clear the state when
-    /// the protocol state is terminal.  It returns `Some` if the message successfully
-    /// advanced the state of the protocol state machine, `None` otherwise.
-    pub async fn handle_message(
-        &mut self,
-        context: &Context,
-        mime_message: &MimeMessage,
-    ) -> Option<BobHandshakeStage> {
-        info!(context, "Handling securejoin message for BobStateHandle");
-        match self.bobstate.handle_message(context, mime_message).await {
-            Ok(Some(stage)) => {
-                if matches!(
-                    stage,
-                    BobHandshakeStage::Completed | BobHandshakeStage::Terminated(_)
-                ) {
-                    self.finish_protocol(context).await;
-                }
-                Some(stage)
-            }
-            Ok(None) => None,
-            Err(err) => {
-                warn!(
-                    context,
-                    "Error handling handshake message, aborting handshake: {}", err
-                );
-                self.finish_protocol(context).await;
-                None
-            }
-        }
-    }
-
-    /// Marks the bob handshake as finished.
-    ///
-    /// This will clear the state on [`InnerContext::bob`] once this handle is dropped,
-    /// allowing a new handshake to be started from [`Bob`].
-    ///
-    /// Note that the state is only cleared on Drop since otherwise the invariant that the
-    /// state is always consistent is violated.  However the "ongoing" process is released
-    /// here a little bit earlier as this requires access to the Context, which we do not
-    /// have on Drop (Drop can not run asynchronous code).  Stopping the "ongoing" process
-    /// will release [`securejoin`](super::securejoin) which in turn will finally free the
-    /// ongoing process using [`Context::free_ongoing`].
-    ///
-    /// [`InnerContext::bob`]: crate::context::InnerContext::bob
-    /// [`Bob`]: super::Bob
-    async fn finish_protocol(&mut self, context: &Context) {
-        info!(context, "Finishing securejoin handshake protocol for Bob");
-        self.clear_state_on_drop = true;
-        if let QrInvite::Group { .. } = self.bobstate.invite {
-            context.stop_ongoing().await;
-        }
-    }
-}
-
-impl<'a> Drop for BobStateHandle<'a> {
-    fn drop(&mut self) {
-        if self.clear_state_on_drop {
-            // The Option should already be empty because we take it out in the ctor,
-            // however the typesystem doesn't guarantee this so do it again anyway.
-            self.guard.take();
-        } else {
-            // Make sure to put back the BobState into the Option of the Mutex, it was taken
-            // out by the constructor.
-            self.guard.replace(self.bobstate.clone());
-        }
-    }
-}
-
-/// The securejoin state kept in-memory while Bob is joining.
+/// This is stored in the database and loaded from there using [`BobState::from_db`].  To
+/// create a new one use [`BobState::start_protocol`].
 ///
-/// This is currently stored in [`Bob`] which is stored on the [`Context`], thus Bob can
-/// only run one securejoin joiner protocol at a time.
-///
-/// This purposefully has nothing optional, the state is always fully valid.  See
-/// [`Bob::state`] to get access to this state.
+/// This purposefully has nothing optional, the state is always fully valid.  However once a
+/// terminal state is reached in [`BobState::next`] the entry in the database will already
+/// have been deleted.
 ///
 /// # Conducting the securejoin handshake
 ///
@@ -177,6 +67,8 @@ impl<'a> Drop for BobStateHandle<'a> {
 /// [`Bob::state`]: super::Bob::state
 #[derive(Debug, Clone)]
 pub struct BobState {
+    /// Database primary key.
+    id: i64,
     /// The QR Invite code.
     invite: QrInvite,
     /// The next expected message from Alice.
@@ -188,39 +80,120 @@ pub struct BobState {
 impl BobState {
     /// Starts the securejoin protocol and creates a new [`BobState`].
     ///
+    /// The `chat_id` needs to be the ID of the 1:1 chat with Alice, this chat will be used
+    /// to exchange the SecureJoin handshake messages as well as for showing error messages.
+    ///
     /// # Bob - the joiner's side
     /// ## Step 2 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    ///
+    /// This currently aborts any other securejoin process if any did not yet complete.  The
+    /// ChatIds of the relevant 1:1 chat of any aborted handshakes are returned so that you
+    /// can report the aboreted handshake in the chat.  (Yes, there can only ever be one
+    /// ChatId in that Vec, the database doesn't care though.)
     pub async fn start_protocol(
         context: &Context,
         invite: QrInvite,
-    ) -> Result<(Self, BobHandshakeStage), JoinError> {
-        let chat_id =
-            ChatId::create_for_contact_with_blocked(context, invite.contact_id(), Blocked::Yes)
-                .await
-                .map_err(JoinError::UnknownContact)?;
-        if fingerprint_equals_sender(context, invite.fingerprint(), invite.contact_id()).await? {
-            // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
-            info!(context, "Taking securejoin protocol shortcut");
-            let state = Self {
-                invite,
-                next: SecureJoinStep::ContactConfirm,
-                chat_id,
+        chat_id: ChatId,
+    ) -> Result<(Self, BobHandshakeStage, Vec<Self>), JoinError> {
+        let (stage, next) =
+            if fingerprint_equals_sender(context, invite.fingerprint(), invite.contact_id()).await?
+            {
+                // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
+                info!(context, "Taking securejoin protocol shortcut");
+                send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::RequestWithAuth)
+                    .await?;
+                (
+                    BobHandshakeStage::RequestWithAuthSent,
+                    SecureJoinStep::ContactConfirm,
+                )
+            } else {
+                send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::Request).await?;
+                (BobHandshakeStage::RequestSent, SecureJoinStep::AuthRequired)
             };
-            state
-                .send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
-                .await?;
-            Ok((state, BobHandshakeStage::RequestWithAuthSent))
-        } else {
-            let state = Self {
-                invite,
-                next: SecureJoinStep::AuthRequired,
-                chat_id,
-            };
-            state
-                .send_handshake_message(context, BobHandshakeMsg::Request)
-                .await?;
-            Ok((state, BobHandshakeStage::RequestSent))
-        }
+        let (id, aborted_states) =
+            Self::insert_new_db_entry(context, next, invite.clone(), chat_id).await?;
+        let state = Self {
+            id,
+            invite,
+            next,
+            chat_id,
+        };
+        Ok((state, stage, aborted_states))
+    }
+
+    /// Inserts a new entry in the bobstate table, deleting all previous entries.
+    ///
+    /// Returns the ID of the newly inserted entry and all the aborted states.
+    async fn insert_new_db_entry(
+        context: &Context,
+        next: SecureJoinStep,
+        invite: QrInvite,
+        chat_id: ChatId,
+    ) -> Result<(i64, Vec<Self>)> {
+        context
+            .sql
+            .transaction(move |transaction| {
+                // We need to start a write transaction right away, so that we have the
+                // database locked and no one else can write to this table while we read the
+                // rows that we will delete.  So start with a dummy UPDATE.
+                transaction.execute(
+                    r#"UPDATE bobstate SET next_step=?;"#,
+                    params![SecureJoinStep::Terminated],
+                )?;
+                let mut stmt = transaction.prepare("SELECT id FROM bobstate;")?;
+                let mut aborted = Vec::new();
+                for id in stmt.query_map(params![], |row| row.get::<_, i64>(0))? {
+                    let id = id?;
+                    let state = BobState::from_db_id(transaction, id)?;
+                    aborted.push(state);
+                }
+
+                // Finally delete everything and insert new row.
+                transaction.execute("DELETE FROM bobstate;", params![])?;
+                transaction.execute(
+                    "INSERT INTO bobstate (invite, next_step, chat_id) VALUES (?, ?, ?);",
+                    params![invite, next, chat_id],
+                )?;
+                let id = transaction.last_insert_rowid();
+                Ok((id, aborted))
+            })
+            .await
+    }
+
+    /// Load [`BobState`] from the database.
+    pub async fn from_db(sql: &Sql) -> Result<Option<Self>> {
+        // Because of how Self::start_protocol() updates the database we are currently
+        // guaranteed to only have one row.
+        sql.query_row_optional(
+            "SELECT id, invite, next_step, chat_id FROM bobstate;",
+            paramsv![],
+            |row| {
+                let s = BobState {
+                    id: row.get(0)?,
+                    invite: row.get(1)?,
+                    next: row.get(2)?,
+                    chat_id: row.get(3)?,
+                };
+                Ok(s)
+            },
+        )
+        .await
+    }
+
+    fn from_db_id(connection: &Connection, id: i64) -> rusqlite::Result<Self> {
+        connection.query_row(
+            "SELECT invite, next_step, chat_id FROM bobstate WHERE id=?;",
+            params![id],
+            |row| {
+                let s = BobState {
+                    id,
+                    invite: row.get(0)?,
+                    next: row.get(1)?,
+                    chat_id: row.get(2)?,
+                };
+                Ok(s)
+            },
+        )
     }
 
     /// Returns the [`QrInvite`] used to create this [`BobState`].
@@ -228,20 +201,45 @@ impl BobState {
         &self.invite
     }
 
+    /// Returns the [`ChatId`] of the 1:1 chat with the inviter (Alice).
+    pub fn alice_chat(&self) -> ChatId {
+        self.chat_id
+    }
+
+    /// Updates the [`BobState::next`] field in memory and the database.
+    ///
+    /// If the next state is a terminal state it will remove this [`BobState`] from the
+    /// database.
+    ///
+    /// If a user scanned a new QR code after this [`BobState`] was loaded this update will
+    /// fail currently because starting a new joiner process currently kills any previously
+    /// running processes.  This is a limitation which will go away in the future.
+    async fn update_next(&mut self, sql: &Sql, next: SecureJoinStep) -> Result<()> {
+        // TODO: write test verifying how this would fail.
+        match next {
+            SecureJoinStep::AuthRequired | SecureJoinStep::ContactConfirm => {
+                sql.execute(
+                    "UPDATE bobstate SET next_step=? WHERE id=?;",
+                    paramsv![next, self.id],
+                )
+                .await?;
+            }
+            SecureJoinStep::Terminated | SecureJoinStep::Completed => {
+                sql.execute("DELETE FROM bobstate WHERE id=?;", paramsv!(self.id))
+                    .await?;
+            }
+        }
+        self.next = next;
+        Ok(())
+    }
+
     /// Handles the given message for the securejoin handshake for Bob.
     ///
     /// If the message was not used for this handshake `None` is returned, otherwise the new
     /// stage is returned.  Once [`BobHandshakeStage::Completed`] or
     /// [`BobHandshakeStage::Terminated`] are reached this [`BobState`] should be destroyed,
     /// further calling it will just result in the messages being unused by this handshake.
-    ///
-    /// # Errors
-    ///
-    /// Under normal operation this should never return an error, regardless of what kind of
-    /// message it is called with.  Any errors therefore should be treated as fatal internal
-    /// errors and this entire [`BobState`] should be thrown away as the state machine can
-    /// no longer be considered consistent.
-    async fn handle_message(
+    pub async fn handle_message(
         &mut self,
         context: &Context,
         mime_message: &MimeMessage,
@@ -304,17 +302,20 @@ impl BobState {
             } else {
                 "Required encryption missing"
             };
-            self.next = SecureJoinStep::Terminated;
+            self.update_next(&context.sql, SecureJoinStep::Terminated)
+                .await?;
             return Ok(Some(BobHandshakeStage::Terminated(reason)));
         }
         if !fingerprint_equals_sender(context, self.invite.fingerprint(), self.invite.contact_id())
             .await?
         {
-            self.next = SecureJoinStep::Terminated;
+            self.update_next(&context.sql, SecureJoinStep::Terminated)
+                .await?;
             return Ok(Some(BobHandshakeStage::Terminated("Fingerprint mismatch")));
         }
         info!(context, "Fingerprint verified.",);
-        self.next = SecureJoinStep::ContactConfirm;
+        self.update_next(&context.sql, SecureJoinStep::ContactConfirm)
+            .await?;
         self.send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
             .await?;
         Ok(Some(BobHandshakeStage::RequestWithAuthSent))
@@ -362,7 +363,8 @@ impl BobState {
         if vg_expect_encrypted
             && !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint()))
         {
-            self.next = SecureJoinStep::Terminated;
+            self.update_next(&context.sql, SecureJoinStep::Terminated)
+                .await?;
             return Ok(Some(BobHandshakeStage::Terminated(
                 "Contact confirm message not encrypted",
             )));
@@ -394,7 +396,8 @@ impl BobState {
             // This is not an error affecting the protocol outcome.
             .ok();
 
-        self.next = SecureJoinStep::Completed;
+        self.update_next(&context.sql, SecureJoinStep::Completed)
+            .await?;
         Ok(Some(BobHandshakeStage::Completed))
     }
 
@@ -406,48 +409,60 @@ impl BobState {
         context: &Context,
         step: BobHandshakeMsg,
     ) -> Result<(), SendMsgError> {
-        let mut msg = Message {
-            viewtype: Viewtype::Text,
-            text: Some(step.body_text(&self.invite)),
-            hidden: true,
-            ..Default::default()
-        };
-        msg.param.set_cmd(SystemMessage::SecurejoinMessage);
-
-        // Sends the step in Secure-Join header.
-        msg.param
-            .set(Param::Arg, step.securejoin_header(&self.invite));
-
-        match step {
-            BobHandshakeMsg::Request => {
-                // Sends the Secure-Join-Invitenumber header in mimefactory.rs.
-                msg.param.set(Param::Arg2, self.invite.invitenumber());
-                msg.force_plaintext();
-            }
-            BobHandshakeMsg::RequestWithAuth => {
-                // Sends the Secure-Join-Auth header in mimefactory.rs.
-                msg.param.set(Param::Arg2, self.invite.authcode());
-                msg.param.set_int(Param::GuaranteeE2ee, 1);
-            }
-            BobHandshakeMsg::ContactConfirmReceived => {
-                msg.param.set_int(Param::GuaranteeE2ee, 1);
-            }
-        };
-
-        // Sends our own fingerprint in the Secure-Join-Fingerprint header.
-        let bob_fp = SignedPublicKey::load_self(context).await?.fingerprint();
-        msg.param.set(Param::Arg3, bob_fp.hex());
-
-        // Sends the grpid in the Secure-Join-Group header.
-        if let QrInvite::Group { ref grpid, .. } = self.invite {
-            msg.param.set(Param::Arg4, grpid);
-        }
-
-        chat::send_msg(context, self.chat_id, &mut msg).await?;
-        Ok(())
+        send_handshake_message(context, &self.invite, self.chat_id, step).await
     }
 }
 
+/// Sends the requested handshake message to Alice.
+///
+/// Same as [`BobState::send_handshake_message`] but this variation allows us to send this
+/// message before we create the state in [`BobState::start_protocol`].
+async fn send_handshake_message(
+    context: &Context,
+    invite: &QrInvite,
+    chat_id: ChatId,
+    step: BobHandshakeMsg,
+) -> Result<(), SendMsgError> {
+    let mut msg = Message {
+        viewtype: Viewtype::Text,
+        text: Some(step.body_text(invite)),
+        hidden: true,
+        ..Default::default()
+    };
+    msg.param.set_cmd(SystemMessage::SecurejoinMessage);
+
+    // Sends the step in Secure-Join header.
+    msg.param.set(Param::Arg, step.securejoin_header(invite));
+
+    match step {
+        BobHandshakeMsg::Request => {
+            // Sends the Secure-Join-Invitenumber header in mimefactory.rs.
+            msg.param.set(Param::Arg2, invite.invitenumber());
+            msg.force_plaintext();
+        }
+        BobHandshakeMsg::RequestWithAuth => {
+            // Sends the Secure-Join-Auth header in mimefactory.rs.
+            msg.param.set(Param::Arg2, invite.authcode());
+            msg.param.set_int(Param::GuaranteeE2ee, 1);
+        }
+        BobHandshakeMsg::ContactConfirmReceived => {
+            msg.param.set_int(Param::GuaranteeE2ee, 1);
+        }
+    };
+
+    // Sends our own fingerprint in the Secure-Join-Fingerprint header.
+    let bob_fp = SignedPublicKey::load_self(context).await?.fingerprint();
+    msg.param.set(Param::Arg3, bob_fp.hex());
+
+    // Sends the grpid in the Secure-Join-Group header.
+    if let QrInvite::Group { ref grpid, .. } = invite {
+        msg.param.set(Param::Arg4, grpid);
+    }
+
+    chat::send_msg(context, chat_id, &mut msg).await?;
+    Ok(())
+}
+
 /// Identifies the SecureJoin handshake messages Bob can send.
 enum BobHandshakeMsg {
     /// vc-request or vg-request
@@ -492,8 +507,8 @@ impl BobHandshakeMsg {
 }
 
 /// The next message expected by [`BobState`] in the setup-contact/secure-join protocol.
-#[derive(Debug, Clone, PartialEq)]
-enum SecureJoinStep {
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum SecureJoinStep {
     /// Expecting the auth-required message.
     ///
     /// This corresponds to the `vc-auth-required` or `vg-auth-required` message of step 3d.
@@ -533,3 +548,29 @@ impl SecureJoinStep {
         }
     }
 }
+
+impl rusqlite::types::ToSql for SecureJoinStep {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
+        let num = match &self {
+            SecureJoinStep::AuthRequired => 0,
+            SecureJoinStep::ContactConfirm => 1,
+            SecureJoinStep::Terminated => 2,
+            SecureJoinStep::Completed => 3,
+        };
+        let val = rusqlite::types::Value::Integer(num);
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+impl rusqlite::types::FromSql for SecureJoinStep {
+    fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|val| match val {
+            0 => Ok(SecureJoinStep::AuthRequired),
+            1 => Ok(SecureJoinStep::ContactConfirm),
+            2 => Ok(SecureJoinStep::Terminated),
+            3 => Ok(SecureJoinStep::Completed),
+            _ => Err(rusqlite::types::FromSqlError::OutOfRange(val)),
+        })
+    }
+}

src/securejoin/qrinvite.rs

@@ -8,22 +8,23 @@ use std::convert::TryFrom;
 
 use anyhow::{bail, Error, Result};
 
+use crate::contact::ContactId;
 use crate::key::Fingerprint;
 use crate::qr::Qr;
 
 /// Represents the data from a QR-code scan.
 ///
 /// There are methods to conveniently access fields present in both variants.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub enum QrInvite {
     Contact {
-        contact_id: u32,
+        contact_id: ContactId,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
     },
     Group {
-        contact_id: u32,
+        contact_id: ContactId,
         fingerprint: Fingerprint,
         name: String,
         grpid: String,
@@ -37,7 +38,7 @@ impl QrInvite {
     ///
     /// The actual QR-code contains a URL-encoded email address, but upon scanning this is
     /// translated to a contact ID.
-    pub fn contact_id(&self) -> u32 {
+    pub fn contact_id(&self) -> ContactId {
         match self {
             Self::Contact { contact_id, .. } | Self::Group { contact_id, .. } => *contact_id,
         }
@@ -100,3 +101,22 @@ impl TryFrom<Qr> for QrInvite {
         }
     }
 }
+
+impl rusqlite::types::ToSql for QrInvite {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
+        let json = serde_json::to_string(self)
+            .map_err(|err| rusqlite::Error::ToSqlConversionFailure(Box::new(err)))?;
+        let val = rusqlite::types::Value::Text(json);
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+impl rusqlite::types::FromSql for QrInvite {
+    fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult<Self> {
+        String::column_result(value).and_then(|val| {
+            serde_json::from_str(&val)
+                .map_err(|err| rusqlite::types::FromSqlError::Other(Box::new(err)))
+        })
+    }
+}

src/simplify.rs

@@ -240,7 +240,7 @@ fn render_message(lines: &[&str], is_cut_at_end: bool) -> String {
         ret += " [...]";
     }
     // redo escaping done by escape_message_footer_marks()
-    ret.replace("\u{200B}", "")
+    ret.replace('\u{200B}', "")
 }
 
 /// Returns true if the line contains only whitespace.

src/smtp.rs

@@ -4,14 +4,14 @@ pub mod send;
 
 use std::time::{Duration, SystemTime};
 
-use anyhow::{bail, format_err, Context as _, Result};
+use anyhow::{bail, format_err, Context as _, Error, Result};
 use async_smtp::smtp::client::net::ClientTlsParameters;
 use async_smtp::smtp::response::{Category, Code, Detail};
 use async_smtp::{smtp, EmailAddress, ServerAddress};
+use async_std::task;
 
 use crate::constants::DC_LP_AUTH_OAUTH2;
 use crate::events::EventType;
-use crate::job::Status;
 use crate::login_param::{
     dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam, Socks5Config,
 };
@@ -50,7 +50,10 @@ impl Smtp {
     /// Disconnect the SMTP transport and drop it entirely.
     pub async fn disconnect(&mut self) {
         if let Some(mut transport) = self.transport.take() {
-            transport.close().await.ok();
+            // Closing connection with a QUIT command may take some time, especially if it's a
+            // stale connection and an attempt to send the command times out. Send a command in a
+            // separate task to avoid waiting for reply or timeout.
+            task::spawn(async move { transport.close().await });
         }
         self.last_success = None;
     }
@@ -84,7 +87,7 @@ impl Smtp {
         }
 
         self.connectivity.set_connecting(context).await;
-        let lp = LoginParam::from_database(context, "configured_").await?;
+        let lp = LoginParam::load_configured_params(context).await?;
         self.connect(
             context,
             &lp.smtp,
@@ -196,12 +199,18 @@ impl Smtp {
     }
 }
 
+pub(crate) enum SendResult {
+    /// Message was sent successfully.
+    Success,
+
+    /// Permanent error, message sending has failed.
+    Failure(Error),
+
+    /// Temporary error, the message should be retried later.
+    Retry,
+}
+
 /// Tries to send a message.
-///
-/// Returns Status::Finished if sending the message should not be retried anymore,
-/// Status::RetryLater if sending should be postponed and Status::RetryNow if it is suspected that
-/// temporary failure is caused by stale connection, in which case a second attempt to send the
-/// same message may be done immediately.
 pub(crate) async fn smtp_send(
     context: &Context,
     recipients: &[async_smtp::EmailAddress],
@@ -209,7 +218,7 @@ pub(crate) async fn smtp_send(
     smtp: &mut Smtp,
     msg_id: MsgId,
     rowid: i64,
-) -> Status {
+) -> SendResult {
     if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
         info!(context, "smtp-sending out mime message:");
         println!("{}", message);
@@ -217,6 +226,20 @@ pub(crate) async fn smtp_send(
 
     smtp.connectivity.set_working(context).await;
 
+    if smtp.has_maybe_stale_connection().await {
+        info!(context, "Closing stale connection");
+        smtp.disconnect().await;
+
+        if let Err(err) = smtp
+            .connect_configured(context)
+            .await
+            .context("failed to reopen stale SMTP connection")
+        {
+            smtp.last_send_error = Some(format!("{:#}", err));
+            return SendResult::Retry;
+        }
+    }
+
     let send_result = smtp
         .send(context, recipients, message.as_bytes(), rowid)
         .await;
@@ -225,7 +248,7 @@ pub(crate) async fn smtp_send(
     let status = match send_result {
         Err(crate::smtp::send::Error::SmtpSend(err)) => {
             // Remote error, retry later.
-            warn!(context, "SMTP failed to send: {:?}", &err);
+            info!(context, "SMTP failed to send: {:?}", &err);
 
             let res = match err {
                 async_smtp::smtp::error::Error::Permanent(ref response) => {
@@ -246,19 +269,21 @@ pub(crate) async fn smtp_send(
                             // Other enhanced status codes, such as Postfix
                             // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
                             // are not ignored.
-                            response.first_word() == Some(&"5.5.0".to_string())
+                            response.first_word() == Some("5.5.0")
                         }
                         _ => false,
                     };
 
                     if maybe_transient {
-                        Status::RetryLater
+                        info!(context, "Permanent error that is likely to actually be transient, postponing retry for later");
+                        SendResult::Retry
                     } else {
+                        info!(context, "Permanent error, message sending failed");
                         // If we do not retry, add an info message to the chat.
                         // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
                         // should definitely go here, because user has to open the link to
                         // resume message sending.
-                        Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                        SendResult::Failure(format_err!("Permanent SMTP error: {}", err))
                     }
                 }
                 async_smtp::smtp::error::Error::Transient(ref response) => {
@@ -275,25 +300,34 @@ pub(crate) async fn smtp_send(
                             // receive as a transient error are misconfigurations of the smtp server.
                             // See <https://tools.ietf.org/html/rfc3463#section-3.2>
                             info!(context, "Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", first_word);
-                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            SendResult::Failure(format_err!("Permanent SMTP error: {}", err))
                         } else {
-                            Status::RetryLater
+                            info!(
+                                context,
+                                "Transient error with status code {}, postponing retry for later",
+                                first_word
+                            );
+                            SendResult::Retry
                         }
                     } else {
-                        Status::RetryLater
+                        info!(
+                            context,
+                            "Transient error without status code, postponing retry for later"
+                        );
+                        SendResult::Retry
                     }
                 }
                 _ => {
-                    if smtp.has_maybe_stale_connection().await {
-                        info!(context, "stale connection? immediately reconnecting");
-                        Status::RetryNow
-                    } else {
-                        Status::RetryLater
-                    }
+                    info!(
+                        context,
+                        "Message sending failed without error returned by the server, retry later"
+                    );
+                    SendResult::Retry
                 }
             };
 
             // this clears last_success info
+            info!(context, "Failed to send message over SMTP, disconnecting");
             smtp.disconnect().await;
 
             res
@@ -302,24 +336,24 @@ pub(crate) async fn smtp_send(
             // Local error, job is invalid, do not retry.
             smtp.disconnect().await;
             warn!(context, "SMTP job is invalid: {}", err);
-            Status::Finished(Err(err.into()))
+            SendResult::Failure(err.into())
         }
         Err(crate::smtp::send::Error::NoTransport) => {
             // Should never happen.
             // It does not even make sense to disconnect here.
             error!(context, "SMTP job failed because SMTP has no transport");
-            Status::Finished(Err(format_err!("SMTP has not transport")))
+            SendResult::Failure(format_err!("SMTP has not transport"))
         }
         Err(crate::smtp::send::Error::Other(err)) => {
             // Local error, job is invalid, do not retry.
             smtp.disconnect().await;
             warn!(context, "unable to load job: {}", err);
-            Status::Finished(Err(err))
+            SendResult::Failure(err)
         }
-        Ok(()) => Status::Finished(Ok(())),
+        Ok(()) => SendResult::Success,
     };
 
-    if let Status::Finished(Err(err)) = &status {
+    if let SendResult::Failure(err) = &status {
         // We couldn't send the message, so mark it as failed
         message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
     }
@@ -343,19 +377,52 @@ pub(crate) async fn send_msg_to_smtp(
         return Err(err);
     }
 
-    let (body, recipients, msg_id) = context
+    // Increase retry count as soon as we have an SMTP connection. This ensures that the message is
+    // eventually removed from the queue by exceeding retry limit even in case of an error that
+    // keeps happening early in the message sending code, e.g. failure to read the message from the
+    // database.
+    context
+        .sql
+        .execute(
+            "UPDATE smtp SET retries=retries+1 WHERE id=?",
+            paramsv![rowid],
+        )
+        .await
+        .context("failed to update retries count")?;
+
+    let (body, recipients, msg_id, retries) = context
         .sql
         .query_row(
-            "SELECT mime, recipients, msg_id FROM smtp WHERE id=?",
+            "SELECT mime, recipients, msg_id, retries FROM smtp WHERE id=?",
             paramsv![rowid],
             |row| {
                 let mime: String = row.get(0)?;
                 let recipients: String = row.get(1)?;
                 let msg_id: MsgId = row.get(2)?;
-                Ok((mime, recipients, msg_id))
+                let retries: i64 = row.get(3)?;
+                Ok((mime, recipients, msg_id, retries))
             },
         )
         .await?;
+    if retries > 6 {
+        message::set_msg_failed(
+            context,
+            msg_id,
+            Some("Number of retries exceeded the limit."),
+        )
+        .await;
+        context
+            .sql
+            .execute("DELETE FROM smtp WHERE id=?", paramsv![rowid])
+            .await
+            .context("failed to remove message with exceeded retry limit from smtp table")?;
+        bail!("Number of retries exceeded the limit");
+    }
+    info!(
+        context,
+        "Retry number {} to send message {} over SMTP", retries, msg_id
+    );
+
     let recipients_list = recipients
         .split(' ')
         .filter_map(
@@ -382,7 +449,7 @@ pub(crate) async fn send_msg_to_smtp(
         return Ok(());
     }
 
-    let status = match smtp_send(
+    let status = smtp_send(
         context,
         &recipients_list,
         body.as_str(),
@@ -390,58 +457,25 @@ pub(crate) async fn send_msg_to_smtp(
         msg_id,
         rowid,
     )
-    .await
-    {
-        Status::RetryNow => {
-            // Do a single retry immediately without increasing retry counter in case of stale
-            // connection.
-            info!(context, "Doing immediate retry to send message.");
+    .await;
 
-            // smtp_send just closed stale SMTP connection, reconnect and try again.
-            if let Err(err) = smtp
-                .connect_configured(context)
-                .await
-                .context("failed to reopen stale SMTP connection")
-            {
-                smtp.last_send_error = Some(format!("{:#}", err));
-                return Err(err);
-            }
-
-            smtp_send(
-                context,
-                &recipients_list,
-                body.as_str(),
-                smtp,
-                msg_id,
-                rowid,
-            )
-            .await
-        }
-        status => status,
-    };
     match status {
-        Status::Finished(res) => {
-            if res.is_ok() {
-                msg_id.set_delivered(context).await?;
-
-                context
-                    .sql
-                    .execute("DELETE FROM smtp WHERE id=?", paramsv![rowid])
-                    .await?;
-            }
-            res
-        }
-        Status::RetryNow | Status::RetryLater => {
+        SendResult::Retry => {}
+        SendResult::Success | SendResult::Failure(_) => {
             context
                 .sql
-                .execute(
-                    "UPDATE smtp SET retries=retries+1 WHERE id=?",
-                    paramsv![rowid],
-                )
-                .await
-                .context("failed to update retries count")?;
-            Err(format_err!("Retry"))
+                .execute("DELETE FROM smtp WHERE id=?", paramsv![rowid])
+                .await?;
         }
+    };
+
+    match status {
+        SendResult::Retry => Err(format_err!("Retry")),
+        SendResult::Success => {
+            msg_id.set_delivered(context).await?;
+            Ok(())
+        }
+        SendResult::Failure(err) => Err(format_err!("{}", err)),
     }
 }
 
@@ -449,15 +483,10 @@ pub(crate) async fn send_msg_to_smtp(
 ///
 /// Logs and ignores SMTP errors to ensure that a single SMTP message constantly failing to be sent
 /// does not block other messages in the queue from being sent.
-pub(crate) async fn send_smtp_messages(
-    context: &Context,
-    connection: &mut Smtp,
-) -> anyhow::Result<()> {
+///
+/// Returns true if all messages were sent successfully, false otherwise.
+pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp) -> Result<bool> {
     context.send_sync_msg().await?; // Add sync message to the end of the queue if needed.
-    context
-        .sql
-        .execute("DELETE FROM smtp WHERE retries > 5", paramsv![])
-        .await?;
     let rowids = context
         .sql
         .query_map(
@@ -474,10 +503,12 @@ pub(crate) async fn send_smtp_messages(
             },
         )
         .await?;
+    let mut success = true;
     for rowid in rowids {
         if let Err(err) = send_msg_to_smtp(context, connection, rowid).await {
             info!(context, "Failed to send message over SMTP: {:#}.", err);
+            success = false;
         }
     }
-    Ok(())
+    Ok(success)
 }

src/sql.rs

@@ -3,7 +3,7 @@
 use async_std::path::Path;
 use async_std::sync::RwLock;
 
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::convert::TryFrom;
 use std::time::Duration;
 
@@ -15,11 +15,11 @@ use rusqlite::{config::DbConfig, Connection, OpenFlags};
 use crate::blob::BlobObject;
 use crate::chat::{add_device_msg, update_device_icon, update_saved_messages_icon};
 use crate::config::Config;
-use crate::constants::{Viewtype, DC_CHAT_ID_TRASH};
+use crate::constants::DC_CHAT_ID_TRASH;
 use crate::context::Context;
 use crate::dc_tools::{dc_delete_file, time};
 use crate::ephemeral::start_ephemeral_timers;
-use crate::message::Message;
+use crate::message::{Message, Viewtype};
 use crate::param::{Param, Params};
 use crate::peerstate::{deduplicate_peerstates, Peerstate};
 use crate::stock_str;
@@ -34,6 +34,17 @@ macro_rules! paramsv {
     };
 }
 
+#[macro_export]
+macro_rules! params_iterv {
+    ($($param:expr),+ $(,)?) => {
+        vec![$(&$param as &dyn $crate::ToSql),+]
+    };
+}
+
+pub(crate) fn params_iter(iter: &[impl crate::ToSql]) -> impl Iterator<Item = &dyn crate::ToSql> {
+    iter.iter().map(|item| item as &dyn crate::ToSql)
+}
+
 mod migrations;
 
 /// A wrapper around the underlying Sqlite3 object.
@@ -47,6 +58,8 @@ pub struct Sql {
     /// None if the database is not open, true if it is open with passphrase and false if it is
     /// open without a passphrase.
     is_encrypted: RwLock<Option<bool>>,
+
+    pub(crate) config_cache: RwLock<HashMap<String, Option<String>>>,
 }
 
 impl Sql {
@@ -55,6 +68,7 @@ impl Sql {
             dbfile,
             pool: Default::default(),
             is_encrypted: Default::default(),
+            config_cache: Default::default(),
         }
     }
 
@@ -178,6 +192,7 @@ impl Sql {
                      PRAGMA secure_delete=on;
                      PRAGMA busy_timeout = {};
                      PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
+                     PRAGMA foreign_keys=on;
                      ",
                     Duration::from_secs(10).as_millis()
                 ))?;
@@ -497,6 +512,8 @@ impl Sql {
     /// will already have been logged.
     pub async fn set_raw_config(&self, key: impl AsRef<str>, value: Option<&str>) -> Result<()> {
         let key = key.as_ref();
+
+        let mut lock = self.config_cache.write().await;
         if let Some(value) = value {
             let exists = self
                 .exists(
@@ -522,12 +539,23 @@ impl Sql {
             self.execute("DELETE FROM config WHERE keyname=?;", paramsv![key])
                 .await?;
         }
+        lock.insert(key.to_string(), value.map(|s| s.to_string()));
+        drop(lock);
 
         Ok(())
     }
 
     /// Get configuration options from the database.
     pub async fn get_raw_config(&self, key: impl AsRef<str>) -> Result<Option<String>> {
+        let lock = self.config_cache.read().await;
+        let cached = lock.get(key.as_ref()).cloned();
+        drop(lock);
+
+        if let Some(c) = cached {
+            return Ok(c);
+        }
+
+        let mut lock = self.config_cache.write().await;
         let value = self
             .query_get_value(
                 "SELECT value FROM config WHERE keyname=?;",
@@ -535,6 +563,8 @@ impl Sql {
             )
             .await
             .context(format!("failed to fetch raw config: {}", key.as_ref()))?;
+        lock.insert(key.as_ref().to_string(), value.clone());
+        drop(lock);
 
         Ok(value)
     }
@@ -573,13 +603,63 @@ impl Sql {
             .await
             .map(|s| s.and_then(|r| r.parse().ok()))
     }
+
+    #[cfg(feature = "internals")]
+    pub fn config_cache(&self) -> &RwLock<HashMap<String, Option<String>>> {
+        &self.config_cache
+    }
 }
 
 pub async fn housekeeping(context: &Context) -> Result<()> {
-    if let Err(err) = crate::ephemeral::delete_expired_messages(context).await {
-        warn!(context, "Failed to delete expired messages: {}", err);
+    if let Err(err) = remove_unused_files(context).await {
+        warn!(
+            context,
+            "Housekeeping: cannot remove unusued files: {}", err
+        );
     }
 
+    if let Err(err) = start_ephemeral_timers(context).await {
+        warn!(
+            context,
+            "Housekeeping: cannot start ephemeral timers: {}", err
+        );
+    }
+
+    if let Err(err) = prune_tombstones(&context.sql).await {
+        warn!(
+            context,
+            "Housekeeping: Cannot prune message tombstones: {}", err
+        );
+    }
+
+    if let Err(err) = deduplicate_peerstates(&context.sql).await {
+        warn!(context, "Failed to deduplicate peerstates: {}", err)
+    }
+
+    context.schedule_quota_update().await?;
+
+    // Try to clear the freelist to free some space on the disk. This
+    // only works if auto_vacuum is enabled.
+    if let Err(err) = context
+        .sql
+        .execute("PRAGMA incremental_vacuum", paramsv![])
+        .await
+    {
+        warn!(context, "Failed to run incremental vacuum: {}", err);
+    }
+
+    if let Err(e) = context
+        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
+        .await
+    {
+        warn!(context, "Can't set config: {}", e);
+    }
+
+    info!(context, "Housekeeping done.");
+    Ok(())
+}
+
+pub async fn remove_unused_files(context: &Context) -> Result<()> {
     let mut files_in_use = HashSet::new();
     let mut unreferenced_count = 0;
 
@@ -694,44 +774,6 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         }
     }
 
-    if let Err(err) = start_ephemeral_timers(context).await {
-        warn!(
-            context,
-            "Housekeeping: cannot start ephemeral timers: {}", err
-        );
-    }
-
-    if let Err(err) = prune_tombstones(&context.sql).await {
-        warn!(
-            context,
-            "Housekeeping: Cannot prune message tombstones: {}", err
-        );
-    }
-
-    if let Err(err) = deduplicate_peerstates(&context.sql).await {
-        warn!(context, "Failed to deduplicate peerstates: {}", err)
-    }
-
-    context.schedule_quota_update().await?;
-
-    // Try to clear the freelist to free some space on the disk. This
-    // only works if auto_vacuum is enabled.
-    if let Err(err) = context
-        .sql
-        .execute("PRAGMA incremental_vacuum", paramsv![])
-        .await
-    {
-        warn!(context, "Failed to run incremental vacuum: {}", err);
-    }
-
-    if let Err(e) = context
-        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
-        .await
-    {
-        warn!(context, "Can't set config: {}", e);
-    }
-
-    info!(context, "Housekeeping done.");
     Ok(())
 }
 
@@ -787,7 +829,7 @@ async fn maybe_add_from_param(
 async fn prune_tombstones(sql: &Sql) -> Result<()> {
     sql.execute(
         "DELETE FROM msgs
-         WHERE (chat_id=? OR hidden)
+         WHERE chat_id=?
          AND NOT EXISTS (
          SELECT * FROM imap WHERE msgs.rfc724_mid=rfc724_mid AND target!=''
          )",
@@ -797,6 +839,16 @@ async fn prune_tombstones(sql: &Sql) -> Result<()> {
     Ok(())
 }
 
+/// Helper function to return comma-separated sequence of `?` chars.
+///
+/// Use this together with [`rusqlite::ParamsFromIter`] to use dynamically generated
+/// parameter lists.
+pub fn repeat_vars(count: usize) -> String {
+    let mut s = "?,".repeat(count);
+    s.pop(); // Remove trailing comma
+    s
+}
+
 #[cfg(test)]
 mod tests {
     use async_std::channel;
@@ -906,6 +958,23 @@ mod tests {
         }
     }
 
+    /// Regression test for a bug where housekeeping deleted drafts since their
+    /// `hidden` flag is set.
+    #[async_std::test]
+    async fn test_housekeeping_dont_delete_drafts() {
+        let t = TestContext::new_alice().await;
+
+        let chat = t.create_chat_with_contact("bob", "bob@example.com").await;
+        let mut new_draft = Message::new(Viewtype::Text);
+        new_draft.set_text(Some("This is my draft".to_string()));
+        chat.id.set_draft(&t, Some(&mut new_draft)).await.unwrap();
+
+        housekeeping(&t).await.unwrap();
+
+        let loaded_draft = chat.id.get_draft(&t).await.unwrap();
+        assert_eq!(loaded_draft.unwrap().text.unwrap(), "This is my draft");
+    }
+
     /// Regression test.
     ///
     /// Previously the code checking for existence of `config` table

src/sql/migrations.rs

@@ -36,6 +36,13 @@ pub async fn run(context: &Context, sql: &Sql) -> Result<(bool, bool, bool, bool
             Ok(())
         })
         .await?;
+
+        let mut lock = context.sql.config_cache.write().await;
+        lock.insert(
+            VERSION_CFG.to_string(),
+            Some(format!("{}", dbversion_before_update)),
+        );
+        drop(lock);
     } else {
         exists_before_update = true;
         dbversion_before_update = sql
@@ -388,7 +395,7 @@ UPDATE chats SET protected=1, type=120 WHERE type=130;"#,
 
     if dbversion < 71 {
         info!(context, "[migration] v71");
-        if let Some(addr) = context.get_config(Config::ConfiguredAddr).await? {
+        if let Ok(addr) = context.get_primary_self_addr().await {
             if let Ok(domain) = addr.parse::<EmailAddress>().map(|email| email.domain) {
                 context
                     .set_config(
@@ -579,6 +586,44 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid);
         )
         .await?;
     }
+    if dbversion < 86 {
+        info!(context, "[migration] v86");
+        sql.execute_migration(
+            r#"CREATE TABLE bobstate (
+                   id INTEGER PRIMARY KEY AUTOINCREMENT,
+                   invite TEXT NOT NULL,
+                   next_step INTEGER NOT NULL,
+                   chat_id INTEGER NOT NULL
+            );"#,
+            86,
+        )
+        .await?;
+    }
+    if dbversion < 87 {
+        info!(context, "[migration] v87");
+        // the index is used to speed up delete_expired_messages()
+        sql.execute_migration(
+            "CREATE INDEX IF NOT EXISTS msgs_index8 ON msgs (ephemeral_timestamp);",
+            87,
+        )
+        .await?;
+    }
+    if dbversion < 88 {
+        info!(context, "[migration] v88");
+        sql.execute_migration("DROP TABLE IF EXISTS backup_blobs;", 88)
+            .await?;
+    }
+    if dbversion < 89 {
+        info!(context, "[migration] v89");
+        sql.execute_migration(
+            r#"CREATE TABLE imap_markseen (
+              id INTEGER,
+              FOREIGN KEY(id) REFERENCES imap(id) ON DELETE CASCADE
+            );"#,
+            89,
+        )
+        .await?;
+    }
 
     Ok((
         recalc_fingerprints,
@@ -609,6 +654,10 @@ impl Sql {
         .await
         .with_context(|| format!("execute_migration failed for version {}", version))?;
 
+        let mut lock = self.config_cache.write().await;
+        lock.insert(VERSION_CFG.to_string(), Some(format!("{}", version)));
+        drop(lock);
+
         Ok(())
     }
 }

src/sql/tables.sql

@@ -168,6 +168,14 @@ CREATE TABLE tokens (
     timestamp INTEGER DEFAULT 0
 );
 
+-- The currently running securejoin protocols, joiner-side.
+-- CREATE TABLE bobstate (
+--     id INTEGER PRIMARY KEY AUTOINCREMENT,
+--     invite TEXT NOT NULL,
+--     next_step INTEGER NOT NULL,
+--     chat_id INTEGER NOT NULL
+-- );
+
 CREATE TABLE locations (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     latitude REAL DEFAULT 0.0,

src/stock_str.rs

@@ -10,11 +10,10 @@ use strum_macros::EnumProperty;
 use crate::blob::BlobObject;
 use crate::chat::{self, Chat, ChatId, ProtectionStatus};
 use crate::config::Config;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
-use crate::contact::{Contact, Origin};
+use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::dc_tools::dc_timestamp_to_str;
-use crate::message::Message;
+use crate::message::{Message, Viewtype};
 use crate::param::Param;
 use humansize::{file_size_opts, FileSize};
 
@@ -288,9 +287,6 @@ pub enum StockMessage {
     #[strum(props(fallback = "Storage on %1$s"))]
     StorageOnDomain = 105,
 
-    #[strum(props(fallback = "One moment…"))]
-    OneMoment = 106,
-
     #[strum(props(fallback = "Connected"))]
     Connected = 107,
 
@@ -333,6 +329,9 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Scan to join group %1$s"))]
     SecureJoinGroupQRDescription = 120,
+
+    #[strum(props(fallback = "Not connected"))]
+    NotConnected = 121,
 }
 
 impl StockMessage {
@@ -389,7 +388,7 @@ trait StockStringMods: AsRef<str> + Sized {
     fn action_by_contact<'a>(
         self,
         context: &'a Context,
-        contact_id: u32,
+        contact_id: ContactId,
     ) -> Pin<Box<dyn Future<Output = String> + Send + 'a>>
     where
         Self: Send + 'a,
@@ -397,7 +396,7 @@ trait StockStringMods: AsRef<str> + Sized {
         Box::pin(async move {
             let message = self.as_ref().trim_end_matches('.');
             match contact_id {
-                DC_CONTACT_ID_SELF => msg_action_by_me(context, message).await,
+                ContactId::SELF => msg_action_by_me(context, message).await,
                 _ => {
                     let displayname = Contact::get_by_id(context, contact_id)
                         .await
@@ -457,7 +456,7 @@ pub(crate) async fn msg_grp_name(
     context: &Context,
     from_group: impl AsRef<str>,
     to_group: impl AsRef<str>,
-    by_contact: u32,
+    by_contact: ContactId,
 ) -> String {
     translated(context, StockMessage::MsgGrpName)
         .await
@@ -468,7 +467,7 @@ pub(crate) async fn msg_grp_name(
 }
 
 /// Stock string: `Group image changed.`.
-pub(crate) async fn msg_grp_img_changed(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn msg_grp_img_changed(context: &Context, by_contact: ContactId) -> String {
     translated(context, StockMessage::MsgGrpImgChanged)
         .await
         .action_by_contact(context, by_contact)
@@ -482,7 +481,7 @@ pub(crate) async fn msg_grp_img_changed(context: &Context, by_contact: u32) -> S
 pub(crate) async fn msg_add_member(
     context: &Context,
     added_member_addr: impl AsRef<str>,
-    by_contact: u32,
+    by_contact: ContactId,
 ) -> String {
     let addr = added_member_addr.as_ref();
     let who = match Contact::lookup_id_by_addr(context, addr, Origin::Unknown).await {
@@ -506,7 +505,7 @@ pub(crate) async fn msg_add_member(
 pub(crate) async fn msg_del_member(
     context: &Context,
     removed_member_addr: impl AsRef<str>,
-    by_contact: u32,
+    by_contact: ContactId,
 ) -> String {
     let addr = removed_member_addr.as_ref();
     let who = match Contact::lookup_id_by_addr(context, addr, Origin::Unknown).await {
@@ -524,7 +523,7 @@ pub(crate) async fn msg_del_member(
 }
 
 /// Stock string: `Group left.`.
-pub(crate) async fn msg_group_left(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn msg_group_left(context: &Context, by_contact: ContactId) -> String {
     translated(context, StockMessage::MsgGroupLeft)
         .await
         .action_by_contact(context, by_contact)
@@ -574,7 +573,7 @@ pub(crate) async fn read_rcpt_mail_body(context: &Context, message: impl AsRef<s
 }
 
 /// Stock string: `Group image deleted.`.
-pub(crate) async fn msg_grp_img_deleted(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn msg_grp_img_deleted(context: &Context, by_contact: ContactId) -> String {
     translated(context, StockMessage::MsgGrpImgDeleted)
         .await
         .action_by_contact(context, by_contact)
@@ -587,7 +586,10 @@ pub(crate) async fn e2e_preferred(context: &Context) -> String {
 }
 
 /// Stock string: `%1$s invited you to join this group. Waiting for the device of %2$s to reply…`.
-pub(crate) async fn secure_join_started(context: &Context, inviter_contact_id: u32) -> String {
+pub(crate) async fn secure_join_started(
+    context: &Context,
+    inviter_contact_id: ContactId,
+) -> String {
     if let Ok(contact) = Contact::get_by_id(context, inviter_contact_id).await {
         translated(context, StockMessage::SecureJoinStarted)
             .await
@@ -602,7 +604,7 @@ pub(crate) async fn secure_join_started(context: &Context, inviter_contact_id: u
 }
 
 /// Stock string: `%1$s replied, waiting for being added to the group…`.
-pub(crate) async fn secure_join_replies(context: &Context, contact_id: u32) -> String {
+pub(crate) async fn secure_join_replies(context: &Context, contact_id: ContactId) -> String {
     if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
         translated(context, StockMessage::SecureJoinReplies)
             .await
@@ -636,20 +638,19 @@ pub(crate) async fn secure_join_group_qr_description(context: &Context, chat: &C
 }
 
 /// Stock string: `%1$s verified.`.
-pub(crate) async fn contact_verified(context: &Context, contact_addr: impl AsRef<str>) -> String {
+pub(crate) async fn contact_verified(context: &Context, contact: &Contact) -> String {
+    let addr = contact.get_name_n_addr();
     translated(context, StockMessage::ContactVerified)
         .await
-        .replace1(contact_addr)
+        .replace1(addr)
 }
 
 /// Stock string: `Cannot verify %1$s`.
-pub(crate) async fn contact_not_verified(
-    context: &Context,
-    contact_addr: impl AsRef<str>,
-) -> String {
+pub(crate) async fn contact_not_verified(context: &Context, contact: &Contact) -> String {
+    let addr = contact.get_name_n_addr();
     translated(context, StockMessage::ContactNotVerified)
         .await
-        .replace1(contact_addr)
+        .replace1(addr)
 }
 
 /// Stock string: `Changed setup for %1$s`.
@@ -719,7 +720,7 @@ pub(crate) async fn msg_location_enabled(context: &Context) -> String {
 }
 
 /// Stock string: `Location streaming enabled by ...`.
-pub(crate) async fn msg_location_enabled_by(context: &Context, contact: u32) -> String {
+pub(crate) async fn msg_location_enabled_by(context: &Context, contact: ContactId) -> String {
     translated(context, StockMessage::MsgLocationEnabled)
         .await
         .action_by_contact(context, contact)
@@ -785,7 +786,10 @@ pub(crate) async fn failed_sending_to(context: &Context, name: impl AsRef<str>)
 }
 
 /// Stock string: `Message deletion timer is disabled.`.
-pub(crate) async fn msg_ephemeral_timer_disabled(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn msg_ephemeral_timer_disabled(
+    context: &Context,
+    by_contact: ContactId,
+) -> String {
     translated(context, StockMessage::MsgEphemeralTimerDisabled)
         .await
         .action_by_contact(context, by_contact)
@@ -796,7 +800,7 @@ pub(crate) async fn msg_ephemeral_timer_disabled(context: &Context, by_contact:
 pub(crate) async fn msg_ephemeral_timer_enabled(
     context: &Context,
     timer: impl AsRef<str>,
-    by_contact: u32,
+    by_contact: ContactId,
 ) -> String {
     translated(context, StockMessage::MsgEphemeralTimerEnabled)
         .await
@@ -806,7 +810,7 @@ pub(crate) async fn msg_ephemeral_timer_enabled(
 }
 
 /// Stock string: `Message deletion timer is set to 1 minute.`.
-pub(crate) async fn msg_ephemeral_timer_minute(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn msg_ephemeral_timer_minute(context: &Context, by_contact: ContactId) -> String {
     translated(context, StockMessage::MsgEphemeralTimerMinute)
         .await
         .action_by_contact(context, by_contact)
@@ -814,7 +818,7 @@ pub(crate) async fn msg_ephemeral_timer_minute(context: &Context, by_contact: u3
 }
 
 /// Stock string: `Message deletion timer is set to 1 hour.`.
-pub(crate) async fn msg_ephemeral_timer_hour(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn msg_ephemeral_timer_hour(context: &Context, by_contact: ContactId) -> String {
     translated(context, StockMessage::MsgEphemeralTimerHour)
         .await
         .action_by_contact(context, by_contact)
@@ -822,7 +826,7 @@ pub(crate) async fn msg_ephemeral_timer_hour(context: &Context, by_contact: u32)
 }
 
 /// Stock string: `Message deletion timer is set to 1 day.`.
-pub(crate) async fn msg_ephemeral_timer_day(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn msg_ephemeral_timer_day(context: &Context, by_contact: ContactId) -> String {
     translated(context, StockMessage::MsgEphemeralTimerDay)
         .await
         .action_by_contact(context, by_contact)
@@ -830,7 +834,7 @@ pub(crate) async fn msg_ephemeral_timer_day(context: &Context, by_contact: u32)
 }
 
 /// Stock string: `Message deletion timer is set to 1 week.`.
-pub(crate) async fn msg_ephemeral_timer_week(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn msg_ephemeral_timer_week(context: &Context, by_contact: ContactId) -> String {
     translated(context, StockMessage::MsgEphemeralTimerWeek)
         .await
         .action_by_contact(context, by_contact)
@@ -875,7 +879,7 @@ pub(crate) async fn error_no_network(context: &Context) -> String {
 }
 
 /// Stock string: `Chat protection enabled.`.
-pub(crate) async fn protection_enabled(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn protection_enabled(context: &Context, by_contact: ContactId) -> String {
     translated(context, StockMessage::ProtectionEnabled)
         .await
         .action_by_contact(context, by_contact)
@@ -883,7 +887,7 @@ pub(crate) async fn protection_enabled(context: &Context, by_contact: u32) -> St
 }
 
 /// Stock string: `Chat protection disabled.`.
-pub(crate) async fn protection_disabled(context: &Context, by_contact: u32) -> String {
+pub(crate) async fn protection_disabled(context: &Context, by_contact: ContactId) -> String {
     translated(context, StockMessage::ProtectionDisabled)
         .await
         .action_by_contact(context, by_contact)
@@ -909,7 +913,7 @@ pub(crate) async fn delete_server_turned_off(context: &Context) -> String {
 pub(crate) async fn msg_ephemeral_timer_minutes(
     context: &Context,
     minutes: impl AsRef<str>,
-    by_contact: u32,
+    by_contact: ContactId,
 ) -> String {
     translated(context, StockMessage::MsgEphemeralTimerMinutes)
         .await
@@ -922,7 +926,7 @@ pub(crate) async fn msg_ephemeral_timer_minutes(
 pub(crate) async fn msg_ephemeral_timer_hours(
     context: &Context,
     hours: impl AsRef<str>,
-    by_contact: u32,
+    by_contact: ContactId,
 ) -> String {
     translated(context, StockMessage::MsgEphemeralTimerHours)
         .await
@@ -935,7 +939,7 @@ pub(crate) async fn msg_ephemeral_timer_hours(
 pub(crate) async fn msg_ephemeral_timer_days(
     context: &Context,
     days: impl AsRef<str>,
-    by_contact: u32,
+    by_contact: ContactId,
 ) -> String {
     translated(context, StockMessage::MsgEphemeralTimerDays)
         .await
@@ -948,7 +952,7 @@ pub(crate) async fn msg_ephemeral_timer_days(
 pub(crate) async fn msg_ephemeral_timer_weeks(
     context: &Context,
     weeks: impl AsRef<str>,
-    by_contact: u32,
+    by_contact: ContactId,
 ) -> String {
     translated(context, StockMessage::MsgEphemeralTimerWeeks)
         .await
@@ -1005,9 +1009,9 @@ pub(crate) async fn storage_on_domain(context: &Context, domain: impl AsRef<str>
         .replace1(domain)
 }
 
-/// Stock string: `One moment…`.
-pub(crate) async fn one_moment(context: &Context) -> String {
-    translated(context, StockMessage::OneMoment).await
+/// Stock string: `Not connected`.
+pub(crate) async fn not_connected(context: &Context) -> String {
+    translated(context, StockMessage::NotConnected).await
 }
 
 /// Stock string: `Connected`.
@@ -1105,7 +1109,7 @@ impl Context {
     pub(crate) async fn stock_protection_msg(
         &self,
         protect: ProtectionStatus,
-        from_id: u32,
+        from_id: ContactId,
     ) -> String {
         match protect {
             ProtectionStatus::Unprotected => protection_enabled(self, from_id).await,
@@ -1124,7 +1128,7 @@ impl Context {
             self.sql
                 .set_raw_config_bool("self-chat-added", true)
                 .await?;
-            ChatId::create_for_contact(self, DC_CONTACT_ID_SELF).await?;
+            ChatId::create_for_contact(self, ContactId::SELF).await?;
         }
 
         // add welcome-messages. by the label, this is done only once,
@@ -1144,14 +1148,13 @@ impl Context {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use crate::test_utils::TestContext;
-
-    use crate::constants::DC_CONTACT_ID_SELF;
+    use num_traits::ToPrimitive;
 
     use crate::chat::Chat;
     use crate::chatlist::Chatlist;
-    use num_traits::ToPrimitive;
+    use crate::test_utils::TestContext;
+
+    use super::*;
 
     #[test]
     fn test_enum_mapping() {
@@ -1197,8 +1200,15 @@ mod tests {
     #[async_std::test]
     async fn test_stock_string_repl_str() {
         let t = TestContext::new().await;
+        let contact_id = Contact::create(&t.ctx, "Someone", "someone@example.org")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
         // uses %1$s substitution
-        assert_eq!(contact_verified(&t, "Foo").await, "Foo verified.");
+        assert_eq!(
+            contact_verified(&t, &contact).await,
+            "Someone (someone@example.org) verified."
+        );
         // We have no string using %1$d to test...
     }
 
@@ -1221,7 +1231,7 @@ mod tests {
     async fn test_stock_system_msg_add_member_by_me() {
         let t = TestContext::new().await;
         assert_eq!(
-            msg_add_member(&t, "alice@example.org", DC_CONTACT_ID_SELF).await,
+            msg_add_member(&t, "alice@example.org", ContactId::SELF).await,
             "Member alice@example.org added by me."
         )
     }
@@ -1233,7 +1243,7 @@ mod tests {
             .await
             .expect("failed to create contact");
         assert_eq!(
-            msg_add_member(&t, "alice@example.org", DC_CONTACT_ID_SELF).await,
+            msg_add_member(&t, "alice@example.org", ContactId::SELF).await,
             "Member Alice (alice@example.org) added by me."
         );
     }

src/summary.rs

@@ -1,11 +1,11 @@
 //! # Message summary for chatlist.
 
 use crate::chat::Chat;
-use crate::constants::{Chattype, Viewtype, DC_CONTACT_ID_SELF};
-use crate::contact::Contact;
+use crate::constants::Chattype;
+use crate::contact::{Contact, ContactId};
 use crate::context::Context;
 use crate::dc_tools::dc_truncate;
-use crate::message::{Message, MessageState};
+use crate::message::{Message, MessageState, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::stock_str;
@@ -60,7 +60,7 @@ impl Summary {
     ) -> Self {
         let prefix = if msg.state == MessageState::OutDraft {
             Some(SummaryPrefix::Draft(stock_str::draft(context).await))
-        } else if msg.from_id == DC_CONTACT_ID_SELF {
+        } else if msg.from_id == ContactId::SELF {
             if msg.is_info() || chat.is_self_talk() {
                 None
             } else {

src/sync.rs

@@ -2,10 +2,11 @@
 
 use crate::chat::{Chat, ChatId};
 use crate::config::Config;
-use crate::constants::{Blocked, Viewtype, DC_CONTACT_ID_SELF};
+use crate::constants::Blocked;
+use crate::contact::ContactId;
 use crate::context::Context;
 use crate::dc_tools::time;
-use crate::message::{Message, MsgId};
+use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::sync::SyncData::{AddQrToken, DeleteQrToken};
@@ -126,7 +127,7 @@ impl Context {
     pub async fn send_sync_msg(&self) -> Result<Option<MsgId>> {
         if let Some((json, ids)) = self.build_sync_json().await? {
             let chat_id =
-                ChatId::create_for_contact_with_blocked(self, DC_CONTACT_ID_SELF, Blocked::Yes)
+                ChatId::create_for_contact_with_blocked(self, ContactId::SELF, Blocked::Yes)
                     .await?;
             let mut msg = Message {
                 chat_id,
@@ -483,7 +484,7 @@ mod tests {
         // check that the used self-talk is not visible to the user
         // but that creation will still work (in this case, the chat is empty)
         assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 0);
-        let chat_id = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
+        let chat_id = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
         let chat = Chat::load_from_db(&alice, chat_id).await?;
         assert!(chat.is_self_talk());
         assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 1);

src/test_utils.rs

@@ -22,14 +22,14 @@ use crate::chat::{self, Chat, ChatId};
 use crate::chatlist::Chatlist;
 use crate::config::Config;
 use crate::constants::Chattype;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF, DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
-use crate::contact::{Contact, Origin};
+use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
+use crate::contact::{Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_receive_imf::dc_receive_imf;
 use crate::dc_tools::EmailAddress;
 use crate::events::{Event, EventType};
 use crate::key::{self, DcKey, KeyPair, KeyPairUse};
-use crate::message::{update_msg_state, Message, MessageState, MsgId};
+use crate::message::{update_msg_state, Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::MimeMessage;
 
 #[allow(non_upper_case_globals)]
@@ -39,6 +39,34 @@ pub const AVATAR_900x900_BYTES: &[u8] = include_bytes!("../test-data/image/avata
 static CONTEXT_NAMES: Lazy<std::sync::RwLock<BTreeMap<u32, String>>> =
     Lazy::new(|| std::sync::RwLock::new(BTreeMap::new()));
 
+pub struct TestContextManager {
+    log_tx: Sender<Event>,
+    _log_sink: LogSink,
+}
+
+impl TestContextManager {
+    pub async fn new() -> Self {
+        let (log_tx, _log_sink) = LogSink::create();
+        Self { log_tx, _log_sink }
+    }
+
+    pub async fn alice(&mut self) -> TestContext {
+        TestContext::builder()
+            .configure_alice()
+            .with_log_sink(self.log_tx.clone())
+            .build()
+            .await
+    }
+
+    pub async fn bob(&mut self) -> TestContext {
+        TestContext::builder()
+            .configure_bob()
+            .with_log_sink(self.log_tx.clone())
+            .build()
+            .await
+    }
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct TestContextBuilder {
     key_pair: Option<KeyPair>,
@@ -60,6 +88,13 @@ impl TestContextBuilder {
         self.with_key_pair(bob_keypair())
     }
 
+    /// Configures as fiona@example.net with fixed secret key.
+    ///
+    /// This is a shortcut for `.with_key_pair(bob_keypair()).
+    pub fn configure_fiona(self) -> Self {
+        self.with_key_pair(fiona_keypair())
+    }
+
     /// Configures the new [`TestContext`] with the provided [`KeyPair`].
     ///
     /// This will extract the email address from the key and configure the context with the
@@ -155,6 +190,13 @@ impl TestContext {
         Self::builder().configure_bob().build().await
     }
 
+    /// Creates a new configured [`TestContext`].
+    ///
+    /// This is a shortcut which configures fiona@example.net with a fixed key.
+    pub async fn new_fiona() -> Self {
+        Self::builder().configure_fiona().build().await
+    }
+
     /// Internal constructor.
     ///
     /// `name` is used to identify this context in e.g. log output.  This is useful mostly
@@ -336,7 +378,7 @@ impl TestContext {
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n"
                 .to_owned()
                 + msg.payload();
-        dc_receive_imf(&self.ctx, received_msg.as_bytes(), "INBOX", false)
+        dc_receive_imf(&self.ctx, received_msg.as_bytes(), false)
             .await
             .unwrap();
     }
@@ -371,31 +413,51 @@ impl TestContext {
             .expect("failed to load msg")
     }
 
+    /// Returns the [`Contact`] for the other [`TestContext`], creating it if necessary.
+    pub async fn add_or_lookup_contact(&self, other: &TestContext) -> Contact {
+        let name = other
+            .ctx
+            .get_config(Config::Displayname)
+            .await
+            .unwrap_or_default()
+            .unwrap_or_default();
+        let addr = other.ctx.get_primary_self_addr().await.unwrap();
+        // MailinglistAddress is the lowest allowed origin, we'd prefer to not modify the
+        // origin when creating this contact.
+        let (contact_id, modified) =
+            Contact::add_or_lookup(self, &name, &addr, Origin::MailinglistAddress)
+                .await
+                .unwrap();
+        match modified {
+            Modifier::None => (),
+            Modifier::Modified => warn!(&self.ctx, "Contact {} modified by TestContext", &addr),
+            Modifier::Created => warn!(&self.ctx, "Contact {} created by TestContext", &addr),
+        }
+        Contact::load_from_db(&self.ctx, contact_id).await.unwrap()
+    }
+
+    /// Returns 1:1 [`Chat`] with another account, if it exists.
+    ///
+    /// This first creates a contact using the configured details on the other account, then
+    /// creates a 1:1 chat with this contact.
+    pub async fn get_chat(&self, other: &TestContext) -> Option<Chat> {
+        let contact = self.add_or_lookup_contact(other).await;
+        match ChatId::lookup_by_contact(&self.ctx, contact.id)
+            .await
+            .unwrap()
+        {
+            Some(id) => Some(Chat::load_from_db(&self.ctx, id).await.unwrap()),
+            None => None,
+        }
+    }
+
     /// Creates or returns an existing 1:1 [`Chat`] with another account.
     ///
     /// This first creates a contact using the configured details on the other account, then
     /// creates a 1:1 chat with this contact.
     pub async fn create_chat(&self, other: &TestContext) -> Chat {
-        let (contact_id, _modified) = Contact::add_or_lookup(
-            self,
-            &other
-                .ctx
-                .get_config(Config::Displayname)
-                .await
-                .unwrap_or_default()
-                .unwrap_or_default(),
-            &other
-                .ctx
-                .get_config(Config::ConfiguredAddr)
-                .await
-                .unwrap()
-                .unwrap(),
-            Origin::ManuallyCreated,
-        )
-        .await
-        .unwrap();
-
-        let chat_id = ChatId::create_for_contact(self, contact_id).await.unwrap();
+        let contact = self.add_or_lookup_contact(other).await;
+        let chat_id = ChatId::create_for_contact(self, contact.id).await.unwrap();
         Chat::load_from_db(self, chat_id).await.unwrap()
     }
 
@@ -413,7 +475,7 @@ impl TestContext {
 
     /// Retrieves the "self" chat.
     pub async fn get_self_chat(&self) -> Chat {
-        let chat_id = ChatId::create_for_contact(self, DC_CONTACT_ID_SELF)
+        let chat_id = ChatId::create_for_contact(self, ContactId::SELF)
             .await
             .unwrap();
         Chat::load_from_db(self, chat_id).await.unwrap()
@@ -643,6 +705,24 @@ pub fn bob_keypair() -> KeyPair {
     }
 }
 
+/// Load a pre-generated keypair for fiona@example.net from disk.
+///
+/// Like [alice_keypair] but a different key and identity.
+pub fn fiona_keypair() -> key::KeyPair {
+    let addr = EmailAddress::new("fiona@example.net").unwrap();
+    let public = key::SignedPublicKey::from_asc(include_str!("../test-data/key/fiona-public.asc"))
+        .unwrap()
+        .0;
+    let secret = key::SignedSecretKey::from_asc(include_str!("../test-data/key/fiona-secret.asc"))
+        .unwrap()
+        .0;
+    key::KeyPair {
+        addr,
+        public,
+        secret,
+    }
+}
+
 /// Utility to help wait for and retrieve events.
 ///
 /// This buffers the events in order they are emitted.  This allows consuming events in
@@ -798,7 +878,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.get_from_id() == 1u32 {
+        if msg.get_from_id() == ContactId::SELF {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -849,17 +929,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_with_both() {
-        let (log_sender, _log_sink) = LogSink::create();
-        let alice = TestContext::builder()
-            .configure_alice()
-            .with_log_sink(log_sender.clone())
-            .build()
-            .await;
-        let bob = TestContext::builder()
-            .configure_bob()
-            .with_log_sink(log_sender)
-            .build()
-            .await;
+        let mut tcm = TestContextManager::new().await;
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+
         alice.ctx.emit_event(EventType::Info("hello".into()));
         bob.ctx.emit_event(EventType::Info("there".into()));
         // panic!("Both fail");

src/update_helper.rs

@@ -1,7 +1,7 @@
 //! # Functions to update timestamps.
 
 use crate::chat::{Chat, ChatId};
-use crate::contact::Contact;
+use crate::contact::{Contact, ContactId};
 use crate::context::Context;
 use crate::param::{Param, Params};
 use anyhow::Result;
@@ -12,7 +12,7 @@ impl Context {
     /// (if we have a ContactId type at some point, the function should go there)
     pub(crate) async fn update_contacts_timestamp(
         &self,
-        contact_id: u32,
+        contact_id: ContactId,
         scope: Param,
         new_timestamp: i64,
     ) -> Result<bool> {
@@ -99,7 +99,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 23:37:57 +0000\n\
                  \n\
                  second message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -113,7 +112,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
                  \n\
                  first message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -143,7 +141,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 01:00:00 +0000\n\
                  \n\
                  first message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -163,7 +160,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 03:00:00 +0000\n\
                  \n\
                  third message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -179,7 +175,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 02:00:00 +0000\n\
                  \n\
                  second message\n",
-            "INBOX",
             false,
         )
         .await?;

src/webxdc.rs

@@ -1,24 +1,30 @@
 //! # Handle webxdc messages.
 
 use crate::chat::Chat;
-use crate::constants::Viewtype;
 use crate::context::Context;
 use crate::dc_tools::{dc_create_smeared_timestamp, dc_open_file_std};
-use crate::message::{Message, MessageState, MsgId};
+use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::{chat, EventType};
 use anyhow::{bail, ensure, format_err, Result};
 use async_std::path::PathBuf;
-use lettre_email::mime::{self};
+use deltachat_derive::FromSql;
+use lettre_email::mime;
 use lettre_email::PartBuilder;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::convert::TryFrom;
 use std::fs::File;
-use std::io::Read;
+use std::io::{Read, Seek, SeekFrom};
 use zip::ZipArchive;
 
+/// The current API version.
+/// If `min_api` in manifest.toml is set to a larger value,
+/// the Webxdc's index.html is replaced by an error message.
+/// In the future, that may be useful to avoid new Webxdc being loaded on old Delta Chats.
+const WEBXDC_API_VERSION: u32 = 1;
+
 pub const WEBXDC_SUFFIX: &str = "xdc";
 const WEBXDC_DEFAULT_ICON: &str = "__webxdc__/default-icon.png";
 
@@ -32,18 +38,19 @@ const WEBXDC_DEFAULT_ICON: &str = "__webxdc__/default-icon.png";
 ///
 /// The limit is also an experiment to see how small we can go;
 /// it is planned to raise that limit as needed in subsequent versions.
-const WEBXDC_SENDING_LIMIT: usize = 655360;
+const WEBXDC_SENDING_LIMIT: u64 = 655360;
 
 /// Be more tolerant for .xdc sizes on receiving -
 /// might be, the senders version uses already a larger limit
 /// and not showing the .xdc on some devices would be even worse ux.
-const WEBXDC_RECEIVING_LIMIT: usize = 4194304;
+const WEBXDC_RECEIVING_LIMIT: u64 = 4194304;
 
 /// Raw information read from manifest.toml
 #[derive(Debug, Deserialize)]
 #[non_exhaustive]
 struct WebxdcManifest {
     name: Option<String>,
+    min_api: Option<u32>,
 }
 
 /// Parsed information from WebxdcManifest and fallbacks.
@@ -56,14 +63,26 @@ pub struct WebxdcInfo {
 
 /// Status Update ID.
 #[derive(
-    Debug, Copy, Clone, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize,
+    Debug,
+    Copy,
+    Clone,
+    Default,
+    PartialEq,
+    Eq,
+    Hash,
+    PartialOrd,
+    Ord,
+    Serialize,
+    Deserialize,
+    FromSql,
+    FromPrimitive,
 )]
-pub struct StatusUpdateId(u32);
+pub struct StatusUpdateSerial(u32);
 
-impl StatusUpdateId {
+impl StatusUpdateSerial {
     /// Create a new [MsgId].
-    pub fn new(id: u32) -> StatusUpdateId {
-        StatusUpdateId(id)
+    pub fn new(id: u32) -> StatusUpdateSerial {
+        StatusUpdateSerial(id)
     }
 
     /// Gets StatusUpdateId as untyped integer.
@@ -73,9 +92,9 @@ impl StatusUpdateId {
     }
 }
 
-impl rusqlite::types::ToSql for StatusUpdateId {
+impl rusqlite::types::ToSql for StatusUpdateSerial {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-        let val = rusqlite::types::Value::Integer(self.0 as i64);
+        let val = rusqlite::types::Value::Integer(i64::from(self.0));
         let out = rusqlite::types::ToSqlOutput::Owned(val);
         Ok(out)
     }
@@ -99,48 +118,68 @@ pub(crate) struct StatusUpdateItem {
     summary: Option<String>,
 }
 
+/// Update items as passed to the UIs.
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct StatusUpdateItemAndSerial {
+    #[serde(flatten)]
+    item: StatusUpdateItem,
+
+    serial: StatusUpdateSerial,
+    max_serial: StatusUpdateSerial,
+}
+
 impl Context {
     /// check if a file is an acceptable webxdc for sending or receiving.
-    pub(crate) async fn is_webxdc_file(&self, filename: &str, buf: &[u8]) -> Result<bool> {
-        if filename.ends_with(WEBXDC_SUFFIX) {
-            let reader = std::io::Cursor::new(buf);
-            if let Ok(mut archive) = zip::ZipArchive::new(reader) {
-                if let Ok(_index_html) = archive.by_name("index.html") {
-                    if buf.len() <= WEBXDC_RECEIVING_LIMIT {
-                        return Ok(true);
-                    } else {
-                        info!(
-                            self,
-                            "{} exceeds receiving limit of {} bytes",
-                            &filename,
-                            WEBXDC_RECEIVING_LIMIT
-                        );
-                    }
-                } else {
-                    info!(self, "{} misses index.html", &filename);
-                }
-            } else {
-                info!(self, "{} cannot be opened as zip-file", &filename);
-            }
+    pub(crate) async fn is_webxdc_file<R>(&self, filename: &str, mut reader: R) -> Result<bool>
+    where
+        R: Read + Seek,
+    {
+        if !filename.ends_with(WEBXDC_SUFFIX) {
+            return Ok(false);
         }
-        Ok(false)
+
+        let size = reader.seek(SeekFrom::End(0))?;
+        if size > WEBXDC_RECEIVING_LIMIT {
+            info!(
+                self,
+                "{} exceeds receiving limit of {} bytes", &filename, WEBXDC_RECEIVING_LIMIT
+            );
+            return Ok(false);
+        }
+
+        reader.seek(SeekFrom::Start(0))?;
+        let mut archive = match zip::ZipArchive::new(reader) {
+            Ok(archive) => archive,
+            Err(_) => {
+                info!(self, "{} cannot be opened as zip-file", &filename);
+                return Ok(false);
+            }
+        };
+
+        if archive.by_name("index.html").is_err() {
+            info!(self, "{} misses index.html", &filename);
+            return Ok(false);
+        }
+
+        Ok(true)
     }
 
     /// ensure that a file is an acceptable webxdc for sending
     /// (sending has more strict size limits).
     pub(crate) async fn ensure_sendable_webxdc_file(&self, path: &PathBuf) -> Result<()> {
         let mut file = std::fs::File::open(path)?;
-        let mut buf = Vec::new();
-        file.read_to_end(&mut buf)?;
         if !self
-            .is_webxdc_file(path.to_str().unwrap_or_default(), &buf)
+            .is_webxdc_file(path.to_str().unwrap_or_default(), &mut file)
             .await?
         {
             bail!(
                 "{} is not a valid webxdc file",
                 path.to_str().unwrap_or_default()
             );
-        } else if buf.len() > WEBXDC_SENDING_LIMIT {
+        }
+
+        let size = file.seek(SeekFrom::End(0))?;
+        if size > WEBXDC_SENDING_LIMIT {
             bail!(
                 "webxdc {} exceeds acceptable size of {} bytes",
                 path.to_str().unwrap_or_default(),
@@ -157,7 +196,7 @@ impl Context {
         instance: &mut Message,
         update_str: &str,
         timestamp: i64,
-    ) -> Result<StatusUpdateId> {
+    ) -> Result<StatusUpdateSerial> {
         let update_str = update_str.trim();
         if update_str.is_empty() {
             bail!("create_status_update_record: empty update.");
@@ -176,13 +215,7 @@ impl Context {
                     _ => item,
                 }
             } else {
-                // TODO: this fallback (legacy `PAYLOAD`) should be deleted soon, together with the test below
-                let payload: Value = serde_json::from_str(update_str)?; // checks if input data are valid json
-                StatusUpdateItem {
-                    payload,
-                    info: None,
-                    summary: None,
-                }
+                bail!("create_status_update_record: no valid update item.");
             }
         };
 
@@ -193,6 +226,7 @@ impl Context {
                 info.as_str(),
                 SystemMessage::Unknown,
                 timestamp,
+                None,
                 Some(instance),
             )
             .await?;
@@ -205,10 +239,7 @@ impl Context {
             {
                 instance.param.set(Param::WebxdcSummary, summary);
                 instance.update_param(self).await;
-                self.emit_event(EventType::MsgsChanged {
-                    chat_id: instance.chat_id,
-                    msg_id: instance.id,
-                });
+                self.emit_msgs_changed(instance.chat_id, instance.id);
             }
         }
 
@@ -219,14 +250,15 @@ impl Context {
                 paramsv![instance.id, serde_json::to_string(&status_update_item)?],
             )
             .await?;
-        let status_update_id = StatusUpdateId(u32::try_from(rowid)?);
+
+        let status_update_serial = StatusUpdateSerial(u32::try_from(rowid)?);
 
         self.emit_event(EventType::WebxdcStatusUpdate {
             msg_id: instance.id,
-            status_update_id,
+            status_update_serial,
         });
 
-        Ok(status_update_id)
+        Ok(status_update_serial)
     }
 
     /// Sends a status update for an webxdc instance.
@@ -250,7 +282,7 @@ impl Context {
         let chat = Chat::load_from_db(self, instance.chat_id).await?;
         ensure!(chat.can_send(self).await?, "cannot send to {}", chat.id);
 
-        let status_update_id = self
+        let status_update_serial = self
             .create_status_update_record(
                 &mut instance,
                 update_str,
@@ -279,7 +311,7 @@ impl Context {
                     Param::Arg,
                     self.render_webxdc_status_update_object(
                         instance_msg_id,
-                        Some(status_update_id),
+                        Some(status_update_serial),
                     )
                     .await?
                     .ok_or_else(|| format_err!("Status object expected."))?,
@@ -338,24 +370,79 @@ impl Context {
         Ok(())
     }
 
-    /// Returns status updates as an JSON-array.
+    /// Returns status updates as an JSON-array, ready to be consumed by a webxdc.
     ///
-    /// Example: `[{"payload":"any update data"},{"payload":"another update data"}]`
-    /// The updates may be filtered by a given status_update_id;
-    /// if no updates are available, an empty JSON-array is returned.
+    /// Example: `[{"serial":1, "max_serial":3, "payload":"any update data"},
+    ///            {"serial":3, "max_serial":3, "payload":"another update data"}]`
+    /// Updates with serials larger than `last_known_serial` are returned.
+    /// If no last serial is known, set `last_known_serial` to 0.
+    /// If no updates are available, an empty JSON-array is returned.
     pub async fn get_webxdc_status_updates(
         &self,
         instance_msg_id: MsgId,
-        status_update_id: Option<StatusUpdateId>,
+        last_known_serial: StatusUpdateSerial,
     ) -> Result<String> {
         let json = self
             .sql
             .query_map(
-                "SELECT update_item FROM msgs_status_updates WHERE msg_id=? AND (1=? OR id=?)",
+                "SELECT update_item, id FROM msgs_status_updates WHERE msg_id=? AND id>? ORDER BY id",
+                paramsv![instance_msg_id, last_known_serial],
+                |row| {
+                    let update_item_str = row.get::<_, String>(0)?;
+                    let serial = row.get::<_, StatusUpdateSerial>(1)?;
+                    Ok((update_item_str, serial))
+                },
+                |rows| {
+                    let mut rows_copy : Vec<(String, StatusUpdateSerial)> = Vec::new(); // `rows_copy` needed as `rows` cannot be iterated twice.
+                    let mut max_serial = StatusUpdateSerial(0);
+                    for row in rows {
+                        let row = row?;
+                        if row.1 > max_serial {
+                            max_serial = row.1;
+                        }
+                        rows_copy.push(row);
+                    }
+
+                    let mut json = String::default();
+                    for row in rows_copy {
+                        let (update_item_str, serial) = row;
+                        let update_item = StatusUpdateItemAndSerial
+                        {
+                            item: serde_json::from_str(&*update_item_str)?,
+                            serial,
+                            max_serial,
+                        };
+
+                        if !json.is_empty() {
+                            json.push_str(",\n");
+                        }
+                        json.push_str(&*serde_json::to_string(&update_item)?);
+                    }
+                    Ok(json)
+                },
+            )
+            .await?;
+        Ok(format!("[{}]", json))
+    }
+
+    /// Renders JSON-object for status updates as used on the wire.
+    ///
+    /// Example: `{"updates": [{"payload":"any update data"},
+    ///                        {"payload":"another update data"}]}`
+    /// If `status_update_serial` is set, exactly that update is rendered, otherwise all updates are rendered.
+    pub(crate) async fn render_webxdc_status_update_object(
+        &self,
+        instance_msg_id: MsgId,
+        status_update_serial: Option<StatusUpdateSerial>,
+    ) -> Result<Option<String>> {
+        let json = self
+            .sql
+            .query_map(
+                "SELECT update_item FROM msgs_status_updates WHERE msg_id=? AND (1=? OR id=?) ORDER BY id",
                 paramsv![
                     instance_msg_id,
-                    if status_update_id.is_some() { 0 } else { 1 },
-                    status_update_id.unwrap_or(StatusUpdateId(0))
+                    if status_update_serial.is_some() { 0 } else { 1 },
+                    status_update_serial.unwrap_or(StatusUpdateSerial(0))
                 ],
                 |row| row.get::<_, String>(0),
                 |rows| {
@@ -371,22 +458,10 @@ impl Context {
                 },
             )
             .await?;
-        Ok(format!("[{}]", json))
-    }
-
-    /// Render JSON-object for status updates as used on the wire.
-    pub(crate) async fn render_webxdc_status_update_object(
-        &self,
-        instance_msg_id: MsgId,
-        status_update_id: Option<StatusUpdateId>,
-    ) -> Result<Option<String>> {
-        let updates_array = self
-            .get_webxdc_status_updates(instance_msg_id, status_update_id)
-            .await?;
-        if updates_array == "[]" {
+        if json.is_empty() {
             Ok(None)
         } else {
-            Ok(Some(format!(r#"{{"updates":{}}}"#, updates_array)))
+            Ok(Some(format!(r#"{{"updates":[{}]}}"#, json)))
         }
     }
 }
@@ -433,6 +508,21 @@ impl Message {
         };
 
         let mut archive = self.get_webxdc_archive(context).await?;
+
+        if name == "index.html" {
+            if let Ok(bytes) = get_blob(&mut archive, "manifest.toml").await {
+                if let Ok(manifest) = parse_webxdc_manifest(&bytes).await {
+                    if let Some(min_api) = manifest.min_api {
+                        if min_api > WEBXDC_API_VERSION {
+                            return Ok(Vec::from(
+                                "<!DOCTYPE html>This Webxdc requires a newer Delta Chat version.",
+                            ));
+                        }
+                    }
+                }
+            }
+        }
+
         get_blob(&mut archive, name).await
     }
 
@@ -445,10 +535,16 @@ impl Message {
             if let Ok(manifest) = parse_webxdc_manifest(&bytes).await {
                 manifest
             } else {
-                WebxdcManifest { name: None }
+                WebxdcManifest {
+                    name: None,
+                    min_api: None,
+                }
             }
         } else {
-            WebxdcManifest { name: None }
+            WebxdcManifest {
+                name: None,
+                min_api: None,
+            }
         };
 
         if let Some(ref name) = manifest.name {
@@ -483,7 +579,11 @@ impl Message {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use std::io::Cursor;
+
+    use async_std::fs::File;
+    use async_std::io::WriteExt;
+
     use crate::chat::{
         add_contact_to_chat, create_group_chat, forward_msgs, send_msg, send_text_msg, ChatId,
         ProtectionStatus,
@@ -492,8 +592,8 @@ mod tests {
     use crate::contact::Contact;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils::TestContext;
-    use async_std::fs::File;
-    use async_std::io::WriteExt;
+
+    use super::*;
 
     #[allow(clippy::assertions_on_constants)]
     #[async_std::test]
@@ -511,35 +611,35 @@ mod tests {
         assert!(
             !t.is_webxdc_file(
                 "bad-ext-no-zip.txt",
-                include_bytes!("../test-data/message/issue_523.txt")
+                Cursor::new(include_bytes!("../test-data/message/issue_523.txt"))
             )
             .await?
         );
         assert!(
             !t.is_webxdc_file(
                 "bad-ext-good-zip.txt",
-                include_bytes!("../test-data/webxdc/minimal.xdc")
+                Cursor::new(include_bytes!("../test-data/webxdc/minimal.xdc"))
             )
             .await?
         );
         assert!(
             !t.is_webxdc_file(
                 "good-ext-no-zip.xdc",
-                include_bytes!("../test-data/message/issue_523.txt")
+                Cursor::new(include_bytes!("../test-data/message/issue_523.txt"))
             )
             .await?
         );
         assert!(
             !t.is_webxdc_file(
                 "good-ext-no-index-html.xdc",
-                include_bytes!("../test-data/webxdc/no-index-html.xdc")
+                Cursor::new(include_bytes!("../test-data/webxdc/no-index-html.xdc"))
             )
             .await?
         );
         assert!(
             t.is_webxdc_file(
                 "good-ext-good-zip.xdc",
-                include_bytes!("../test-data/webxdc/minimal.xdc")
+                Cursor::new(include_bytes!("../test-data/webxdc/minimal.xdc"))
             )
             .await?
         );
@@ -635,8 +735,9 @@ mod tests {
         .await?;
         assert!(!instance.is_forwarded());
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, None).await?,
-            r#"[{"payload":42,"info":"foo","summary":"bar"}]"#
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":42,"info":"foo","summary":"bar","serial":1,"max_serial":1}]"#
         );
         assert_eq!(chat_id.get_msg_cnt(&t).await?, 2); // instance and info
         let info = Message::load_from_db(&t, instance.id)
@@ -649,7 +750,11 @@ mod tests {
         forward_msgs(&t, &[instance.get_id()], chat_id).await?;
         let instance2 = t.get_last_msg_in(chat_id).await;
         assert!(instance2.is_forwarded());
-        assert_eq!(t.get_webxdc_status_updates(instance2.id, None).await?, "[]");
+        assert_eq!(
+            t.get_webxdc_status_updates(instance2.id, StatusUpdateSerial(0))
+                .await?,
+            "[]"
+        );
         assert_eq!(chat_id.get_msg_cnt(&t).await?, 3); // two instances, only one info
         let info = Message::load_from_db(&t, instance2.id)
             .await?
@@ -666,7 +771,6 @@ mod tests {
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/webxdc_good_extension.eml"),
-            "INBOX",
             false,
         )
         .await?;
@@ -677,7 +781,6 @@ mod tests {
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/webxdc_bad_extension.eml"),
-            "INBOX",
             false,
         )
         .await?;
@@ -712,7 +815,8 @@ mod tests {
             .await
             .is_err());
         assert_eq!(
-            bob.get_webxdc_status_updates(bob_instance.id, None).await?,
+            bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0))
+                .await?,
             "[]"
         );
 
@@ -723,8 +827,9 @@ mod tests {
             .await
             .is_ok());
         assert_eq!(
-            bob.get_webxdc_status_updates(bob_instance.id, None).await?,
-            r#"[{"payload":42}]"#
+            bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":42,"serial":1,"max_serial":1}]"#
         );
 
         Ok(())
@@ -746,14 +851,16 @@ mod tests {
         t.send_webxdc_status_update(instance.id, r#"{"payload": 42}"#, "descr")
             .await?;
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, None).await?,
-            r#"[{"payload":42}]"#.to_string()
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":42,"serial":1,"max_serial":1}]"#.to_string()
         );
 
         // set_draft(None) deletes the message without the need to simulate network
         chat_id.set_draft(&t, None).await?;
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, None).await?,
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
             "[]".to_string()
         );
         assert_eq!(
@@ -772,9 +879,13 @@ mod tests {
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "foo").await?;
         let mut instance = send_webxdc_instance(&t, chat_id).await?;
 
-        assert_eq!(t.get_webxdc_status_updates(instance.id, None).await?, "[]");
+        assert_eq!(
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
+            "[]"
+        );
 
-        let id = t
+        let update_id1 = t
             .create_status_update_record(
                 &mut instance,
                 "\n\n{\"payload\": {\"foo\":\"bar\"}}\n",
@@ -782,8 +893,9 @@ mod tests {
             )
             .await?;
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, Some(id)).await?,
-            r#"[{"payload":{"foo":"bar"}}]"#
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":1}]"#
         );
 
         assert!(t
@@ -795,15 +907,12 @@ mod tests {
             .await
             .is_err());
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, Some(id)).await?,
-            r#"[{"payload":{"foo":"bar"}}]"#
-        );
-        assert_eq!(
-            t.get_webxdc_status_updates(instance.id, None).await?,
-            r#"[{"payload":{"foo":"bar"}}]"#
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":1}]"#
         );
 
-        let id = t
+        let update_id2 = t
             .create_status_update_record(
                 &mut instance,
                 r#"{"payload" : { "foo2":"bar2"}}"#,
@@ -811,19 +920,20 @@ mod tests {
             )
             .await?;
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, Some(id)).await?,
-            r#"[{"payload":{"foo2":"bar2"}}]"#
+            t.get_webxdc_status_updates(instance.id, update_id1).await?,
+            r#"[{"payload":{"foo2":"bar2"},"serial":2,"max_serial":2}]"#
         );
         t.create_status_update_record(&mut instance, r#"{"payload":true}"#, 1640178619)
             .await?;
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, None).await?,
-            r#"[{"payload":{"foo":"bar"}},
-{"payload":{"foo2":"bar2"}},
-{"payload":true}]"#
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":3},
+{"payload":{"foo2":"bar2"},"serial":2,"max_serial":3},
+{"payload":true,"serial":3,"max_serial":3}]"#
         );
 
-        let id = t
+        let _update_id3 = t
             .create_status_update_record(
                 &mut instance,
                 r#"{"payload" : 1, "sender": "that is not used"}"#,
@@ -831,17 +941,9 @@ mod tests {
             )
             .await?;
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, Some(id)).await?,
-            r#"[{"payload":1}]"#
-        );
-
-        // TODO: legacy `PAYLOAD` support should be deleted soon
-        let id = t
-            .create_status_update_record(&mut instance, r#"{"foo" : 1}"#, 1640178619)
-            .await?;
-        assert_eq!(
-            t.get_webxdc_status_updates(instance.id, Some(id)).await?,
-            r#"[{"payload":{"foo":1}}]"#
+            t.get_webxdc_status_updates(instance.id, update_id2).await?,
+            r#"[{"payload":true,"serial":3,"max_serial":4},
+{"payload":1,"serial":4,"max_serial":4}]"#
         );
 
         Ok(())
@@ -873,8 +975,9 @@ mod tests {
         t.receive_status_update(instance.id, r#"{"updates":[{"payload":{"foo":"bar"}}]}"#)
             .await?;
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, None).await?,
-            r#"[{"payload":{"foo":"bar"}}]"#
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":1}]"#
         );
 
         t.receive_status_update(
@@ -883,10 +986,11 @@ mod tests {
         )
         .await?;
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, None).await?,
-            r#"[{"payload":{"foo":"bar"}},
-{"payload":42},
-{"payload":23}]"#
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":3},
+{"payload":42,"serial":2,"max_serial":3},
+{"payload":23,"serial":3,"max_serial":3}]"#
         );
 
         t.receive_status_update(
@@ -895,11 +999,12 @@ mod tests {
         )
         .await?; // ignore members that may be added in the future
         assert_eq!(
-            t.get_webxdc_status_updates(instance.id, None).await?,
-            r#"[{"payload":{"foo":"bar"}},
-{"payload":42},
-{"payload":23},
-{"payload":"ok"}]"#
+            t.get_webxdc_status_updates(instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":4},
+{"payload":42,"serial":2,"max_serial":4},
+{"payload":23,"serial":3,"max_serial":4},
+{"payload":"ok","serial":4,"max_serial":4}]"#
         );
 
         Ok(())
@@ -913,13 +1018,8 @@ mod tests {
         match event {
             EventType::WebxdcStatusUpdate {
                 msg_id,
-                status_update_id,
+                status_update_serial: _,
             } => {
-                assert_eq!(
-                    t.get_webxdc_status_updates(msg_id, Some(status_update_id))
-                        .await?,
-                    r#"[{"payload":{"foo":"bar"}}]"#
-                );
                 assert_eq!(msg_id, instance_id);
             }
             _ => unreachable!(),
@@ -964,9 +1064,9 @@ mod tests {
         assert!(sent2.payload().contains("descr text"));
         assert_eq!(
             alice
-                .get_webxdc_status_updates(alice_instance.id, None)
+                .get_webxdc_status_updates(alice_instance.id, StatusUpdateSerial(0))
                 .await?,
-            r#"[{"payload":{"foo":"bar"}}]"#
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":1}]"#
         );
 
         alice
@@ -979,10 +1079,10 @@ mod tests {
             .unwrap();
         assert_eq!(
             alice
-                .get_webxdc_status_updates(alice_instance.id, None)
+                .get_webxdc_status_updates(alice_instance.id, StatusUpdateSerial(0))
                 .await?,
-            r#"[{"payload":{"foo":"bar"}},
-{"payload":{"snipp":"snapp"}}]"#
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":2},
+{"payload":{"snipp":"snapp"},"serial":2,"max_serial":2}]"#
         );
 
         // Bob receives all messages
@@ -998,8 +1098,9 @@ mod tests {
         assert_eq!(bob_chat_id.get_msg_cnt(&bob).await?, 1);
 
         assert_eq!(
-            bob.get_webxdc_status_updates(bob_instance.id, None).await?,
-            r#"[{"payload":{"foo":"bar"}}]"#
+            bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":1}]"#
         );
 
         // Alice has a second device and also receives messages there
@@ -1091,9 +1192,10 @@ mod tests {
         assert!(sent1.payload().contains("status-update.json"));
         assert!(sent1.payload().contains(r#""payload":{"foo":"bar"}"#));
         assert_eq!(
-            bob.get_webxdc_status_updates(bob_instance.id, None).await?,
-            r#"[{"payload":{"foo":"bar"}},
-{"payload":42}]"# // 'info: "i"' ignored as sent in draft mode
+            bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":{"foo":"bar"},"serial":1,"max_serial":2},
+{"payload":42,"serial":2,"max_serial":2}]"# // 'info: "i"' ignored as sent in draft mode
         );
 
         Ok(())
@@ -1224,6 +1326,38 @@ sth_for_the = "future""#
         )
         .await?;
         assert_eq!(manifest.name, Some("foz".to_string()));
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_parse_webxdc_manifest_min_api() -> Result<()> {
+        let manifest = parse_webxdc_manifest(r#"min_api = 3"#.as_bytes()).await?;
+        assert_eq!(manifest.min_api, Some(3));
+
+        let result = parse_webxdc_manifest(r#"min_api = "1""#.as_bytes()).await;
+        assert!(result.is_err());
+
+        let result = parse_webxdc_manifest(r#"min_api = 1.2"#.as_bytes()).await;
+        assert!(result.is_err());
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_webxdc_min_api_too_large() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "chat").await?;
+        let mut instance = create_webxdc_instance(
+            &t,
+            "with-min-api-1001.xdc",
+            include_bytes!("../test-data/webxdc/with-min-api-1001.xdc"),
+        )
+        .await?;
+        send_msg(&t, chat_id, &mut instance).await?;
+
+        let instance = t.get_last_msg().await;
+        let html = instance.get_webxdc_blob(&t, "index.html").await?;
+        assert!(String::from_utf8_lossy(&*html).contains("requires a newer Delta Chat version"));
 
         Ok(())
     }
@@ -1411,9 +1545,9 @@ sth_for_the = "future""#
         assert!(info_msg.quoted_message(&alice).await?.is_none());
         assert_eq!(
             alice
-                .get_webxdc_status_updates(alice_instance.id, None)
+                .get_webxdc_status_updates(alice_instance.id, StatusUpdateSerial(0))
                 .await?,
-            r#"[{"payload":"sth. else","info":"this appears in-chat"}]"#
+            r#"[{"payload":"sth. else","info":"this appears in-chat","serial":1,"max_serial":1}]"#
         );
 
         // Bob receives all messages
@@ -1431,8 +1565,9 @@ sth_for_the = "future""#
         assert_eq!(info_msg.parent(&bob).await?.unwrap().id, bob_instance.id);
         assert!(info_msg.quoted_message(&bob).await?.is_none());
         assert_eq!(
-            bob.get_webxdc_status_updates(bob_instance.id, None).await?,
-            r#"[{"payload":"sth. else","info":"this appears in-chat"}]"#
+            bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":"sth. else","info":"this appears in-chat","serial":1,"max_serial":1}]"#
         );
 
         // Alice has a second device and also receives the info message there
@@ -1455,9 +1590,9 @@ sth_for_the = "future""#
         assert!(info_msg.quoted_message(&alice2).await?.is_none());
         assert_eq!(
             alice2
-                .get_webxdc_status_updates(alice2_instance.id, None)
+                .get_webxdc_status_updates(alice2_instance.id, StatusUpdateSerial(0))
                 .await?,
-            r#"[{"payload":"sth. else","info":"this appears in-chat"}]"#
+            r#"[{"payload":"sth. else","info":"this appears in-chat","serial":1,"max_serial":1}]"#
         );
 
         Ok(())

standards.md

@@ -16,7 +16,7 @@ Seen status synchronization      | IMAP CONDSTORE extension ([RFC 7162](https://
 Authorization                    | OAuth2 ([RFC 6749](https://tools.ietf.org/html/rfc6749))
 End-to-end encryption            | [Autocrypt Level 1](https://autocrypt.org/level1.html), OpenPGP ([RFC 4880](https://tools.ietf.org/html/rfc4880)), Security Multiparts for MIME ([RFC 1847](https://tools.ietf.org/html/rfc1847)) and [“Mixed Up” Encryption repairing](https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html)
 Header encryption                | [Protected Headers for Cryptographic E-mail](https://datatracker.ietf.org/doc/draft-autocrypt-lamps-protected-headers/)
-Configuration assistance         | [Autoconfigure](https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover](https://technet.microsoft.com/library/bb124251(v=exchg.150).aspx)
+Configuration assistance         | [Autoconfigure](https://web.archive.org/web/20210402044801/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover](https://technet.microsoft.com/library/bb124251(v=exchg.150).aspx)
 Messenger functions              | [Chat-over-Email](https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#chat-mail-specification)
 Detect mailing list              | List-Id ([RFC 2919](https://tools.ietf.org/html/rfc2919)) and Precedence ([RFC 3834](https://tools.ietf.org/html/rfc3834))
 User and chat colors             | [XEP-0392](https://xmpp.org/extensions/xep-0392.html): Consistent Color Generation

test-data/message/ms_exchange_report_disposition_notification.eml

@@ -1,43 +1,43 @@
-Return-Path: <anonymous@example.org>
+Return-Path: <bob@example.net>
 Delivered-To: anonymous@posteo.de
 Received: from proxy02.posteo.name ([127.0.0.1])
-	by dovecot16.posteo.name (Dovecot) with LMTP id Cp2uFxP1sWHbCQEAchYRkQ
-	for <anonymous@posteo.de>; Thu, 09 Dec 2021 13:25:38 +0100
+	by dovecot16.posteo.name (Dovecot) with LMTP id GaxcARout2HxiwMAchYRkQ
+	for <anonymous@posteo.de>; Mon, 13 Dec 2021 12:35:32 +0100
 Received: from proxy02.posteo.de ([127.0.0.1])
-	by proxy02.posteo.name (Dovecot) with LMTP id MWsaCwrvsWG0wgEAGFAyLg
-	; Thu, 09 Dec 2021 13:25:38 +0100
-Received: from mailin06.posteo.de (unknown [10.0.1.6])
-	by proxy02.posteo.de (Postfix) with ESMTPS id 4J8tXy0KkMz120l
-	for <anonymous@posteo.de>; Thu,  9 Dec 2021 13:25:38 +0100 (CET)
-Received: from mx04.posteo.de (mailin06.posteo.de [127.0.0.1])
-	by mailin06.posteo.de (Postfix) with ESMTPS id F24DE215B8
-	for <anonymous@posteo.de>; Thu,  9 Dec 2021 13:25:37 +0100 (CET)
+	by proxy02.posteo.name (Dovecot) with LMTP id q/LiCqwqt2FMTQEAGFAyLg
+	; Mon, 13 Dec 2021 12:35:32 +0100
+Received: from mailin05.posteo.de (unknown [10.0.1.5])
+	by proxy02.posteo.de (Postfix) with ESMTPS id 4JCKFJ1LCLz1214
+	for <anonymous@posteo.de>; Mon, 13 Dec 2021 12:35:32 +0100 (CET)
+Received: from mx03.posteo.de (mailin05.posteo.de [127.0.0.1])
+	by mailin05.posteo.de (Postfix) with ESMTPS id 1B26420012
+	for <anonymous@posteo.de>; Mon, 13 Dec 2021 12:35:32 +0100 (CET)
 X-Virus-Scanned: amavisd-new at posteo.de
 X-Spam-Flag: NO
 X-Spam-Score: 0.011
 X-Spam-Level: 
 X-Spam-Status: No, score=0.011 tagged_above=-1000 required=7
 	tests=[HTML_MESSAGE=0.001, T_POSTEO_TLSINY=0.01] autolearn=disabled
-X-Posteo-Antispam-Signature: v=1; e=base64; a=aes-256-gcm; d=27yedFdXeAzOobR4x685XJ/5e6WQmX8PP5pSnOlGU2a9Ismhk38wb5AS44xh1yeL5PUxla78UEsHwGkPR0IyPRlHWaLMFLd5CJZN3GzFfrj/2CuB+cd1hOLpp9hRmCebc3rchuDr
+X-Posteo-Antispam-Signature: v=1; e=base64; a=aes-256-gcm; d=RzB41PpqvrD+cuxf3UAqQLiXQL4MHazHZcKeOYJw75deIl7zxtrLXqfAZCeq2IPKt/njRRONUbfuvNdvLxg4mBJ0Rnb53wFOKOqtEpTzxYoQff3yqBpGSohr0DBG26PyBHi7ba/7
 Authentication-Results: posteo.de; dmarc=none (p=none dis=none) header.from=example.org
 X-Posteo-TLS-Received-Status: TLSv1.2
 Received: from mail.example.org (mail.example.org [0.0.0.0])
-	by mx04.posteo.de (Postfix) with ESMTPS id 4J8tXx38vRz10yw
-	for <anonymous@posteo.at>; Thu,  9 Dec 2021 13:25:37 +0100 (CET)
-Received: from [192.168.1.11] (port=22105 helo=mail.example.org)
+	by mx03.posteo.de (Postfix) with ESMTPS id 4JCKFH2ZM7zyx0
+	for <alice@example.org>; Mon, 13 Dec 2021 12:35:31 +0100 (CET)
+Received: from [192.168.1.11] (port=27040 helo=mail.example.org)
 	by mail.example.org with esmtps  (TLS1.2) tls TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384
 	(Exim 4.94.2)
-	(envelope-from <anonymous@example.org>)
-	id 1mvIUG-0007VC-2U
-	for anonymous@posteo.at; Thu, 09 Dec 2021 13:25:24 +0100
-From: Anonymous <anonymous@example.org>
-To: Anonymous <anonymous@posteo.at>
+	(envelope-from <bob@example.net>)
+	id 1mwjc6-0003hM-2K
+	for alice@example.org; Mon, 13 Dec 2021 12:35:26 +0100
+From: Anonymous_2 <bob@example.net>
+To: Anonymous_1 <alice@example.org>
 Subject: Gelesen: Test message
 Thread-Topic: Test message
-Thread-Index: AQHX7Dt/+5f88Aokk0KrqG0hbF8dN6wqFvxh
-Date: Thu, 9 Dec 2021 12:25:24 +0000
-Message-ID: <1711fc3548cd4b2699ccd4fffac17713@anonymous>
-In-Reply-To: <75dd051097b02468183707ad0dd62ebd@posteo.de>
+Thread-Index: AQHX8BVZ9B3+kB6CNUCtJ9eQIONSNawwSqpi
+Date: Mon, 13 Dec 2021 11:35:26 +0000
+Message-ID: <59b1d0c94a8d4834b7ab779a76647d44@mail.example.org>
+In-Reply-To: <d5904dc344eeb5deaf9bb44603f0c716@posteo.de>
 Accept-Language: de-AT, de-DE, en-US
 Content-Language: de-DE
 X-MS-Has-Attach:
@@ -45,29 +45,29 @@ X-MS-TNEF-Correlator:
 x-ms-exchange-transport-fromentityheader: Hosted
 x-originating-ip: [192.168.120.215]
 Content-Type: multipart/report;
-	boundary="_000_1711fc3548cd4b2699ccd4fffac17713anonymous_";
+	boundary="_000_59b1d0c94a8d4834b7ab779a76647d44mailexampleorg_";
 	report-type=disposition-notification
 MIME-Version: 1.0
 
---_000_1711fc3548cd4b2699ccd4fffac17713anonymous_
+--_000_59b1d0c94a8d4834b7ab779a76647d44mailexampleorg_
 Content-Type: multipart/alternative;
-	boundary="_002_1711fc3548cd4b2699ccd4fffac17713anonymous_"
+	boundary="_002_59b1d0c94a8d4834b7ab779a76647d44mailexampleorg_"
 
---_002_1711fc3548cd4b2699ccd4fffac17713anonymous_
+--_002_59b1d0c94a8d4834b7ab779a76647d44mailexampleorg_
 Content-Type: text/plain; charset="iso-8859-1"
 Content-Transfer-Encoding: quoted-printable
 
 Ihre Nachricht
 
-   An: Anonymous
+   An: Anonymous_2
    Betreff: Test message
-   Gesendet: Mittwoch, 8. Dezember 2021 14:57:05 (UTC+01:00) Amsterdam, Ber=
-lin, Bern, Rom, Stockholm, Wien
+   Gesendet: Montag, 13. Dezember 2021 12:33:58 (UTC+01:00) Amsterdam, Berl=
+in, Bern, Rom, Stockholm, Wien
 
- wurde am Donnerstag, 9. Dezember 2021 13:24:34 (UTC+01:00) Amsterdam, Berl=
-in, Bern, Rom, Stockholm, Wien gelesen.
+ wurde am Montag, 13. Dezember 2021 12:34:40 (UTC+01:00) Amsterdam, Berlin,=
+ Bern, Rom, Stockholm, Wien gelesen.
 
---_002_1711fc3548cd4b2699ccd4fffac17713anonymous_
+--_002_59b1d0c94a8d4834b7ab779a76647d44mailexampleorg_
 Content-Type: text/html; charset="iso-8859-1"
 Content-Transfer-Encoding: quoted-printable
 
@@ -83,26 +83,26 @@ ding-left: 4pt; border-left: #800000 2px solid; } --></style>
 <font size=3D"2"><span style=3D"font-size:10pt;">
 <div class=3D"PlainText">Ihre Nachricht <br>
 <br>
-&nbsp;&nbsp; An: Anonymous<br>
+&nbsp;&nbsp; An: Anonymous_2<br>
 &nbsp;&nbsp; Betreff: Test message<br>
-&nbsp;&nbsp; Gesendet: Mittwoch, 8. Dezember 2021 14:57:05 (UTC&#43;01:00) =
-Amsterdam, Berlin, Bern, Rom, Stockholm, Wien<br>
+&nbsp;&nbsp; Gesendet: Montag, 13. Dezember 2021 12:33:58 (UTC&#43;01:00) A=
+msterdam, Berlin, Bern, Rom, Stockholm, Wien<br>
 <br>
-&nbsp;wurde am Donnerstag, 9. Dezember 2021 13:24:34 (UTC&#43;01:00) Amster=
-dam, Berlin, Bern, Rom, Stockholm, Wien gelesen.</div>
+&nbsp;wurde am Montag, 13. Dezember 2021 12:34:40 (UTC&#43;01:00) Amsterdam=
+, Berlin, Bern, Rom, Stockholm, Wien gelesen.</div>
 </span></font>
 </body>
 </html>
 
---_002_1711fc3548cd4b2699ccd4fffac17713anonymous_--
+--_002_59b1d0c94a8d4834b7ab779a76647d44mailexampleorg_--
 
---_000_1711fc3548cd4b2699ccd4fffac17713anonymous_
+--_000_59b1d0c94a8d4834b7ab779a76647d44mailexampleorg_
 Content-Type: message/disposition-notification
 
-Final-recipient: RFC822; anonymous@example.org
+Final-recipient: RFC822; bob@example.net
 Disposition: automatic-action/MDN-sent-automatically; displayed
-X-MSExch-Correlation-Key: coNC5vaCQkiAOjek1v1Uew==
-X-Display-Name: Anonymous
+X-MSExch-Correlation-Key: nf7/jgN6Qk+WzsrkY5s9WA==
+X-Display-Name: Anonymous_2
 
 
---_000_1711fc3548cd4b2699ccd4fffac17713anonymous_--
+--_000_59b1d0c94a8d4834b7ab779a76647d44mailexampleorg_--

test-data/message/ms_exchange_report_original_message.eml

@@ -0,0 +1,34 @@
+Received: We have to put a Received header here. Otherwise, the message would be ignored
+	because DC thinks it's a draft, and the test fails.
+	Alternatively, we could configure the Sentobox folder in the test.
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+ boundary="=_3293d145f71bf71c4bb97415536759d6"
+Date: Mon, 13 Dec 2021 12:33:58 +0100
+From: Anonymous_1 <alice@example.org>
+To: Anonymous_2 <bob@example.net>
+Subject: Test message
+Return-Receipt-To: Anonymous_1 <alice@example.org>
+Disposition-Notification-To: Anonymous_1 <alice@example.org>
+Message-ID: <d5904dc344eeb5deaf9bb44603f0c716@posteo.de>
+X-Sender: alice@example.org
+User-Agent: Posteo Webmail
+
+--=_3293d145f71bf71c4bb97415536759d6
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain; charset=US-ASCII
+
+ This is a test!
+
+Best regards 
+--=_3293d145f71bf71c4bb97415536759d6
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html; charset=UTF-8
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
+<html><body style=3D'font-size: 10pt; font-family: Verdana,Geneva,sans-seri=
+f'>
+This is a test!<br /><br />Best regards
+</body></html>
+
+--=_3293d145f71bf71c4bb97415536759d6--