prepare 1.79 (#3281)

* update changelog for 1.79.0

* bump version to 1.79.0

.github/dependabot.yml

@@ -3,7 +3,7 @@ updates:
   - package-ecosystem: "cargo"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "monthly"
     commit-message:
       prefix: "cargo"
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 50

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples
+          args: --workspace --tests --examples --benches
 
   docs:
     name: Rust doc comments
@@ -77,10 +77,10 @@ jobs:
         include:
           # Currently used Rust version, same as in `rust-toolchain` file.
           - os: ubuntu-latest
-            rust: 1.59.0
+            rust: 1.60.0
             python: 3.9
           - os: windows-latest
-            rust: 1.59.0
+            rust: 1.60.0
             python: false # Python bindings compilation on Windows is not supported.
 
           # Minimum Supported Rust Version = 1.56.0

.github/workflows/dependabot.yml

@@ -0,0 +1,21 @@
+name: Dependabot auto-approve
+on: pull_request
+
+permissions:
+  pull-requests: write
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v1.1.1
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Approve a PR
+        run: gh pr review --approve "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.gitignore

@@ -29,3 +29,7 @@ deltachat-ffi/xml
 
 coverage/
 .DS_Store
+.vscode/launch.json
+python/accounts.txt
+python/all-testaccounts.txt
+tmp/

CHANGELOG.md

@@ -1,19 +1,56 @@
 # Changelog
 
-## Unreleased
+## 1.79.0
+
+### Changes
+- Send locations in the background regardless of SMTP loop activity #3247
+- refactorings #3268
+- improve tests and ci #3266 #3271
+
+### Fixes
+- simplify `dc_stop_io()` and remove potential panics and race conditions #3273
+- fix correct message escaping consisting of a dot in SMTP protocol #3265
+
+
+## 1.78.0
+
+### API-Changes
+- replaced stock string `DC_STR_ONE_MOMENT` by `DC_STR_NOT_CONNECTED` #3222
+- add `dc_resend_msgs()` #3238
+- `dc_provider_new_from_email()` does no longer do an DNS lookup for checking custom domains,
+  this is done by `dc_provider_new_from_email_with_dns()` now #3256
+
+### Changes
+- introduce multiple self addresses with the "configured" address always being the primary one #2896
+- Further improve finding the correct server after logging in #3208
+- `get_connectivity_html()` returns HTML as non-scalable #3213
+- add update-serial to `DC_EVENT_WEBXDC_STATUS_UPDATE` #3215
+- Speed up message receiving via IMAP a bit #3225
+- mark messages as seen on IMAP in batches #3223
+- remove Received: based draft detection heuristic #3230
+- Use pkgconfig for building Python package #2590
+- don't start io on unconfigured context #2664
+- do not assign group IDs to ad-hoc groups #2798
+- dynamic libraries use dylib extension on Darwin #3226
+- refactorings #3217 #3219 #3224 #3235 #3239 #3244 #3254
+- improve documentation #3214 #3220 #3237
+- improve tests and ci #3212 #3233 #3241 #3242 #3252 #3250 #3255 #3260
+
+### Fixes
+- Take `delete_device_after` into account when calculating ephemeral loop timeout #3211 #3221
+- Fix a bug where a blocked contact could send a contact request #3218
+- Make sure, videochat-room-names are always URL-safe #3231
+- Try removing account folder multiple times in case of failure #3229
+- Ignore messages from all spam folders if there are many #3246
+- Hide location-only messages instead of displaying empty bubbles #3248
+
+
+## 1.77.0
 
 ### API changes
 - change semantics of `dc_get_webxdc_status_updates()` second parameter
   and remove update-id from `DC_EVENT_WEBXDC_STATUS_UPDATE` #3081
 
-### Fixes
-- Hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
-- do not delete messages without Message-IDs as duplicates #3095
-- Assign replies from a different email address to the correct chat #3119
-- start ephemeral timer when seen status is synchronized via IMAP #3122
-- do not delete duplicate messages on IMAP immediately to accidentally deleting
-  the last copy #3138
-
 ### Changes
 - add more SMTP logging #3093
 - place common headers like `From:` before the large `Autocrypt:` header #3079
@@ -22,12 +59,32 @@
 - improve speed by caching config values #3131 #3145
 - optimize `markseen_msgs` #3141
 - automatically accept chats with outgoing messages #3143
-- `dc_receive_imf` refactorings #3154 #3156
+- `dc_receive_imf` refactorings #3154 #3156 #3159
 - add index to speedup deletion of expired ephemeral messages #3155
-
+- muted chats stay archived on new messages #3184
+- support `min_api` from Webxdc manifests #3206
+- do not read whole webxdc file into memory #3109
+- improve tests, refactorings #3073 #3096 #3102 #3108 #3139 #3128 #3133 #3142 #3153 #3151 #3174 #3170 #3148 #3179 #3185
+- improve documentation #2983 #3112 #3103 #3118 #3120
 
 ### Fixes
-- Fix a bug where sometimes the file extension of a long filename containing a dot was cropped #3098
+- speed up loading of chat messages by a factor of 20 #3171 #3194 #3173
+- fix an issue where the app crashes when trying to export a backup #3195
+- hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
+- do not delete messages without Message-IDs as duplicates #3095
+- assign replies from a different email address to the correct chat #3119
+- assing outgoing private replies to the correct chat #3177
+- start ephemeral timer when seen status is synchronized via IMAP #3122
+- do not create empty contact requests with "setup changed" messages;
+  instead, send a "setup changed" message into all chats we share with the peer #3187
+- do not delete duplicate messages on IMAP immediately to accidentally deleting
+  the last copy #3138
+- clear more columns when message expires due to `delete_device_after` setting #3181
+- do not try to use stale SMTP connections #3180
+- slightly improve finding the correct server after logging in #3207
+- retry message sending automatically if loop is not interrupted #3183
+- fix a bug where sometimes the file extension of a long filename containing a dot was cropped #3098
+
 
 ## 1.76.0
 

CMakeLists.txt

@@ -4,10 +4,18 @@ include(GNUInstallDirs)
 
 find_program(CARGO cargo)
 
+if(APPLE)
+       set(DYNAMIC_EXT "dylib")
+elseif(UNIX)
+       set(DYNAMIC_EXT "so")
+else()
+       set(DYNAMIC_EXT "dll")
+endif()
+
 add_custom_command(
 	OUTPUT
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
+	"target/release/libdeltachat.${DYNAMIC_EXT}"
 	"target/release/pkgconfig/deltachat.pc"
         COMMAND
         PREFIX=${CMAKE_INSTALL_PREFIX}
@@ -32,11 +40,11 @@ add_custom_target(
 	ALL
 	DEPENDS
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
+	"target/release/libdeltachat.${DYNAMIC_EXT}"
 	"target/release/pkgconfig/deltachat.pc"
 )
 
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.76.0"
+version = "1.79.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -19,7 +19,7 @@ ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
 async-imap = { git = "https://github.com/async-email/async-imap" }
 async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
+async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", default-features=false, features = ["smtp-transport", "socks5"] }
 async-std-resolver = "0.21"
 async-std = { version = "1" }
 async-tar = { version = "0.4", default-features=false }
@@ -62,7 +62,6 @@ serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
 smallvec = "1"
-stop-token = "0.7"
 strum = "0.24"
 strum_macros = "0.24"
 surf = { version = "2.3", default-features = false, features = ["h1-client"] }
@@ -75,7 +74,7 @@ humansize = "1"
 qrcodegen = "1.7.0"
 tagger = "4.3.3"
 textwrap = "0.15.0"
-zip = { version = "0.6.0", default-features = false, features = ["deflate"] }
+zip = { version = "0.6.2", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
@@ -124,6 +123,10 @@ harness = false
 name = "get_chat_msgs"
 harness = false
 
+[[bench]]
+name = "get_chatlist"
+harness = false
+
 [features]
 default = ["vendored"]
 internals = []

benches/get_chat_msgs.rs

@@ -27,10 +27,12 @@ fn criterion_benchmark(c: &mut Criterion) {
             (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
         });
 
-        c.bench_function("Load all chats", |b| {
+        c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
             b.to_async(AsyncStdExecutor)
-                .iter(|| get_chat_msgs_benchmark(black_box(&path.as_ref()), black_box(&chats)))
+                .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
         });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
     }
 }
 

benches/get_chatlist.rs

@@ -0,0 +1,27 @@
+use criterion::async_executor::AsyncStdExecutor;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+use deltachat::chatlist::Chatlist;
+use deltachat::context::Context;
+
+async fn get_chat_list_benchmark(context: &Context) {
+    Chatlist::try_load(context, 0, None, None).await.unwrap();
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let context =
+            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
+        c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
+            b.to_async(AsyncStdExecutor)
+                .iter(|| get_chat_list_benchmark(black_box(&context)))
+        });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/receive_emails.rs

@@ -31,7 +31,7 @@ Hello {i}",
             i = i,
             i_dec = i - 1,
         );
-        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), "INBOX", false)
+        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), false)
             .await
             .unwrap();
     }

benches/search_msgs.rs

@@ -20,6 +20,8 @@ fn criterion_benchmark(c: &mut Criterion) {
         c.bench_function("search hello", |b| {
             b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
         });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
     }
 }
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.76.0"
+version = "1.79.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -1801,6 +1801,25 @@ void            dc_delete_msgs               (dc_context_t* context, const uint3
 void            dc_forward_msgs              (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt, uint32_t chat_id);
 
 
+/**
+ * Resend messages and make information available for newly added chat members.
+ * Resending sends out the original message, however, recipients and webxdc-status may differ.
+ * Clients that already have the original message can still ignore the resent message as
+ * they have tracked the state by dedicated updates.
+ *
+ * Some messages cannot be resent, eg. info-messages, drafts, already pending messages or messages that are not sent by SELF.
+ * In this case, the return value indicates an error and the error string from dc_get_last_error() should be shown to the user.
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @param msg_ids An array of uint32_t containing all message IDs that should be resend.
+ *     All messages must belong to the same chat.
+ * @param msg_cnt The number of messages IDs in the msg_ids array.
+ * @return 1=all messages are queued for resending, 0=error
+ */
+int             dc_resend_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
+
+
 /**
  * Mark messages as presented to the user.
  * Typically, UIs call this function on scrolling through the message list,
@@ -4629,9 +4648,10 @@ int             dc_contact_is_verified       (dc_contact_t* contact);
 
 
 /**
- * Create a provider struct for the given e-mail address.
+ * Create a provider struct for the given e-mail address by local lookup.
  *
- * The provider is extracted from the e-mail address and it's information is returned.
+ * Lookup is done from the local database by extracting the domain from the e-mail address.
+ * Therefore the provider for custom domains cannot be identified.
  *
  * @memberof dc_provider_t
  * @param context The context object.
@@ -4643,6 +4663,23 @@ int             dc_contact_is_verified       (dc_contact_t* contact);
 dc_provider_t*  dc_provider_new_from_email            (const dc_context_t* context, const char* email);
 
 
+/**
+ * Create a provider struct for the given e-mail address by local and DNS lookup.
+ *
+ * First lookup is done from the local database as of dc_provider_new_from_email().
+ * If the first lookup fails, an additional DNS lookup is done,
+ * trying to figure out the provider belonging to custom domains.
+ *
+ * @memberof dc_provider_t
+ * @param context The context object.
+ * @param email The user's e-mail address to extract the provider info form.
+ * @return A dc_provider_t struct which can be used with the dc_provider_get_*
+ *     accessor functions. If no provider info is found, NULL will be
+ *     returned.
+ */
+dc_provider_t*  dc_provider_new_from_email_with_dns    (const dc_context_t* context, const char* email);
+
+
 /**
  * URL of the overview page.
  *
@@ -5606,11 +5643,16 @@ void dc_event_unref(dc_event_t* event);
 /**
  * webxdc status update received.
  * To get the received status update, use dc_get_webxdc_status_updates() with
- * `serial` set to the last known update.
+ * `serial` set to the last known update
+ * (in case of special bots, `status_update_serial` from `data2`
+ * may help to calculate the last known update for dc_get_webxdc_status_updates();
+ * UIs must not peek at this parameter to avoid races in the status replication
+ * eg. when events arrive while initial updates are played back).
+ *
  * To send status updates, use dc_send_webxdc_status_update().
  *
  * @param data1 (int) msg_id
- * @param data2 (int) 0
+ * @param data2 (int) status_update_serial - must not be used by UI implementations.
  */
 #define DC_EVENT_WEBXDC_STATUS_UPDATE                2120
 
@@ -6218,9 +6260,7 @@ void dc_event_unref(dc_event_t* event);
 /// `%1$s` will be replaced by the domain of the configured e-mail address.
 #define DC_STR_STORAGE_ON_DOMAIN          105
 
-/// "One moment…"
-///
-/// Used in the connectivity view when some information are not yet there.
+/// @deprecated Deprecated 2022-04-16, this string is no longer needed.
 #define DC_STR_ONE_MOMENT                 106
 
 /// "Connected"
@@ -6309,6 +6349,11 @@ void dc_event_unref(dc_event_t* event);
 /// `%1$s` will be replaced with the group name.
 #define DC_STR_SECURE_JOIN_GROUP_QR_DESC  120
 
+/// "Not connected"
+///
+/// Used as status in the connectivity view.
+#define DC_STR_NOT_CONNECTED              121
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -503,7 +503,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::SecurejoinJoinerProgress { contact_id, .. } => {
             contact_id.to_u32() as libc::c_int
         }
-        EventType::WebxdcStatusUpdate(msg_id) => msg_id.to_u32() as libc::c_int,
+        EventType::WebxdcStatusUpdate { msg_id, .. } => msg_id.to_u32() as libc::c_int,
     }
 }
 
@@ -535,9 +535,8 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ImexFileWritten(_)
         | EventType::MsgsNoticed(_)
         | EventType::ConnectivityChanged
-        | EventType::SelfavatarChanged
-        | EventType::WebxdcStatusUpdate(_)
-        | EventType::ChatModified(_) => 0,
+        | EventType::SelfavatarChanged => 0,
+        EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
         | EventType::MsgDelivered { msg_id, .. }
@@ -546,6 +545,10 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         EventType::SecurejoinInviterProgress { progress, .. }
         | EventType::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
         EventType::ChatEphemeralTimerModified { timer, .. } => timer.to_u32() as libc::c_int,
+        EventType::WebxdcStatusUpdate {
+            status_update_serial,
+            ..
+        } => status_update_serial.to_u32() as libc::c_int,
     }
 }
 
@@ -587,7 +590,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SecurejoinJoinerProgress { .. }
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
-        | EventType::WebxdcStatusUpdate(_)
+        | EventType::WebxdcStatusUpdate { .. }
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -1748,6 +1751,27 @@ pub unsafe extern "C" fn dc_forward_msgs(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_resend_msgs(
+    context: *mut dc_context_t,
+    msg_ids: *const u32,
+    msg_cnt: libc::c_int,
+) -> libc::c_int {
+    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
+        eprintln!("ignoring careless call to dc_resend_msgs()");
+        return 0;
+    }
+    let ctx = &*context;
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
+
+    if let Err(err) = block_on(chat::resend_msgs(ctx, &msg_ids)) {
+        error!(ctx, "Resending failed: {}", err);
+        0
+    } else {
+        1
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_markseen_msgs(
     context: *mut dc_context_t,
@@ -1949,18 +1973,19 @@ pub unsafe extern "C" fn dc_block_contact(
     contact_id: u32,
     block: libc::c_int,
 ) {
-    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL.to_u32() {
+    let contact_id = ContactId::new(contact_id);
+    if context.is_null() || contact_id.is_special() {
         eprintln!("ignoring careless call to dc_block_contact()");
         return;
     }
     let ctx = &*context;
     block_on(async move {
         if block == 0 {
-            Contact::unblock(ctx, ContactId::new(contact_id))
+            Contact::unblock(ctx, contact_id)
                 .await
                 .ok_or_log_msg(ctx, "Can't unblock contact");
         } else {
-            Contact::block(ctx, ContactId::new(contact_id))
+            Contact::block(ctx, contact_id)
                 .await
                 .ok_or_log_msg(ctx, "Can't block contact");
         }
@@ -1994,14 +2019,15 @@ pub unsafe extern "C" fn dc_delete_contact(
     context: *mut dc_context_t,
     contact_id: u32,
 ) -> libc::c_int {
-    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL.to_u32() {
+    let contact_id = ContactId::new(contact_id);
+    if context.is_null() || contact_id.is_special() {
         eprintln!("ignoring careless call to dc_delete_contact()");
         return 0;
     }
     let ctx = &*context;
 
     block_on(async move {
-        match Contact::delete(ctx, ContactId::new(contact_id)).await {
+        match Contact::delete(ctx, contact_id).await {
             Ok(_) => 1,
             Err(_) => 0,
         }
@@ -3947,6 +3973,25 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
     }
     let addr = to_string_lossy(addr);
 
+    let ctx = &*context;
+
+    match block_on(provider::get_provider_info(ctx, addr.as_str(), true)) {
+        Some(provider) => provider,
+        None => ptr::null_mut(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_new_from_email_with_dns(
+    context: *const dc_context_t,
+    addr: *const libc::c_char,
+) -> *const dc_provider_t {
+    if context.is_null() || addr.is_null() {
+        eprintln!("ignoring careless call to dc_provider_new_from_email_with_dns()");
+        return ptr::null();
+    }
+    let addr = to_string_lossy(addr);
+
     let ctx = &*context;
     let socks5_enabled = block_on(async move {
         ctx.get_config_bool(config::Config::Socks5Enabled)

draft/webxdc-dev-reference.md

@@ -54,12 +54,13 @@ and you won't get the update by `setUpdateListener()`.
 ### setUpdateListener()
 
 ```js
-window.webxdc.setUpdateListener((update) => {}, serial);
+let promise = window.webxdc.setUpdateListener((update) => {}, serial);
 ```
 
 With `setUpdateListener()` you define a callback that receives the updates
 sent by `sendUpdate()`. The callback is called for updates sent by you or other peers.
 The `serial` specifies the last serial that you know about (defaults to 0). 
+The returned promise resolves when the listener has processed all the update messages known at the time when  `setUpdateListener` was called. 
 
 Each `update` which is passed to the callback comes with the following properties: 
 
@@ -176,6 +177,9 @@ just clone and start adapting things to your need.
 
 - older devices might not have the newest js features in their webview,
   you may want to transpile your code down to an older js version eg. with https://babeljs.io
+- viewport and scaling features are implementation specific,
+  if you want to have an explicit behavior, you can add eg.
+  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
 - there are tons of ideas for enhancements of the API and the file format,
   eg. in the future, we will may define icon- and manifest-files,
   allow to aggregate the state or add metadata.

examples/repl/cmdline.rs

@@ -21,7 +21,6 @@ use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
-use deltachat::EventType;
 use deltachat::{config, provider};
 use std::fs;
 use std::time::{Duration, SystemTime};
@@ -93,16 +92,13 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_event(EventType::MsgsChanged {
-        chat_id: ChatId::new(0),
-        msg_id: MsgId::new(0),
-    });
+    context.emit_msgs_changed_without_ids();
 }
 
 async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
     let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", false).await {
+    if let Err(err) = dc_receive_imf(context, &data, false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -164,10 +160,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
+        context.emit_msgs_changed_without_ids();
     }
     true
 }
@@ -210,7 +203,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         contact_id,
         msgtext.unwrap_or_default(),
         if msg.has_html() { "[HAS-HTML]️" } else { "" },
-        if msg.get_from_id() == DC_CONTACT_ID_SELF {
+        if msg.get_from_id() == ContactId::SELF {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -297,7 +290,7 @@ async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()
         let peerstate = Peerstate::from_addr(context, addr)
             .await
             .expect("peerstate error");
-        if peerstate.is_some() && *contact_id != DC_CONTACT_ID_SELF {
+        if peerstate.is_some() && *contact_id != ContactId::SELF {
             line2 = format!(
                 ", prefer-encrypt={}",
                 peerstate.as_ref().unwrap().prefer_encrypt
@@ -406,6 +399,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
+                 resend <msg-id>\n\
                  markseen <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
@@ -932,13 +926,24 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
+            let query = format!("{} {}", arg1, arg2).trim().to_string();
             let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
             let time_start = std::time::SystemTime::now();
-            let msglist = context.search_msgs(chat, arg1).await?;
+            let msglist = context.search_msgs(chat, &query).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
-            println!("{} messages.", msglist.len());
+            println!(
+                "{}{} messages for {}search of \"{}\"",
+                msglist.len(),
+                if msglist.len() == 1000 { "+" } else { "" },
+                if chat.is_none() {
+                    "global "
+                } else {
+                    "in-chat-"
+                },
+                query,
+            );
             println!("{:?} to create this list", time_needed);
         }
         "draft" => {
@@ -1095,6 +1100,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(&context, &msg_ids, chat_id).await?;
         }
+        "resend" => {
+            ensure!(!arg1.is_empty(), "Arguments <msg-id> expected");
+
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
+            chat::resend_msgs(&context, &msg_ids).await?;
+        }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];

examples/repl/main.rs

@@ -207,11 +207,12 @@ const CHAT_COMMANDS: [&str; 36] = [
     "accept",
     "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 7] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
+    "resend",
     "markseen",
     "delmsg",
     "download",

python/install_python_bindings.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
 
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
         subprocess.check_call([

python/mypy.ini

@@ -17,3 +17,7 @@ ignore_missing_imports = True
 
 [mypy-_pytest.*]
 ignore_missing_imports = True
+
+[mypy-imap_tools.*]
+ignore_missing_imports = True
+

python/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
+requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]

python/setup.py

@@ -11,8 +11,11 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
-        setup_requires=['setuptools_scm'], # required for compatibility with `python3 setup.py sdist`
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
+        setup_requires=[
+            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
+            'pkgconfig',
+        ],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/_build.py

@@ -8,9 +8,9 @@ import shutil
 import subprocess
 import tempfile
 import textwrap
-import types
 
 import cffi
+import pkgconfig  # type: ignore
 
 
 def local_build_flags(projdir, target):
@@ -19,36 +19,31 @@ def local_build_flags(projdir, target):
     :param projdir: The root directory of the deltachat-core-rust project.
     :param target: The rust build target, `debug` or `release`.
     """
-    flags = types.SimpleNamespace()
+    flags = {}
     if platform.system() == 'Darwin':
-        flags.libs = ['resolv', 'dl']
-        flags.extra_link_args = [
+        flags['libraries'] = ['resolv', 'dl']
+        flags['extra_link_args'] = [
                 '-framework', 'CoreFoundation',
                 '-framework', 'CoreServices',
                 '-framework', 'Security',
         ]
     elif platform.system() == 'Linux':
-        flags.libs = ['rt', 'dl', 'm']
-        flags.extra_link_args = []
+        flags['libraries'] = ['rt', 'dl', 'm']
+        flags['extra_link_args'] = []
     else:
         raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
     target_dir = os.environ.get("CARGO_TARGET_DIR")
     if target_dir is None:
         target_dir = os.path.join(projdir, 'target')
-    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags.objs[0]), flags.objs
-    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    flags['extra_objects'] = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags['extra_objects'][0]), flags['extra_objects']
+    flags['include_dirs'] = [os.path.join(projdir, 'deltachat-ffi')]
     return flags
 
 
 def system_build_flags():
     """Construct build flags for building against an installed libdeltachat."""
-    flags = types.SimpleNamespace()
-    flags.libs = ['deltachat']
-    flags.objs = []
-    flags.incs = []
-    flags.extra_link_args = []
-    return flags
+    return pkgconfig.parse('deltachat')
 
 
 def extract_functions(flags):
@@ -69,7 +64,7 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags.incs,
+                      include_dirs=flags['include_dirs'],
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
             return dst_fp.read()
@@ -105,7 +100,7 @@ def find_header(flags):
         try:
             os.chdir(tmpdir)
             cc.compile(sources=["where.c"],
-                       include_dirs=flags.incs,
+                       include_dirs=flags['include_dirs'],
                        macros=[("PY_CFFI_INC", "1")])
         finally:
             os.chdir(cwd)
@@ -183,10 +178,7 @@ def ffibuilder():
                 return DC_EVENT_DATA2_IS_STRING(e);
             }
         """,
-        include_dirs=flags.incs,
-        libraries=flags.libs,
-        extra_objects=flags.objs,
-        extra_link_args=flags.extra_link_args,
+        **flags,
     )
     builder.cdef("""
         typedef int... time_t;

python/src/deltachat/chat.py

@@ -5,7 +5,7 @@ import calendar
 import json
 from datetime import datetime, timezone
 import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
+from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
@@ -517,7 +517,7 @@ class Chat(object):
                     lib.dc_array_get_timestamp(dc_array, i),
                     timezone.utc
                 ),
-                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+                marker=from_optional_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]

python/src/deltachat/direct_imap.py

@@ -4,25 +4,22 @@ and for cleaning up inbox/mvbox for each test function run.
 """
 
 import io
-import email
 import ssl
 import pathlib
-from imapclient import IMAPClient
-from imapclient.exceptions import IMAPClientError
+from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
 import imaplib
 import deltachat
 from deltachat import const, Account
+from typing import List
 
 
-SEEN = b'\\Seen'
-DELETED = b'\\Deleted'
 FLAGS = b'FLAGS'
 FETCH = b'FETCH'
 ALL = "1:*"
 
 
 @deltachat.global_hookimpl
-def dc_account_extra_configure(account):
+def dc_account_extra_configure(account: Account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
@@ -36,12 +33,10 @@ def dc_account_extra_configure(account):
                     assert imap.select_folder(folder)
                     imap.delete(ALL, expunge=True)
                 else:
-                    imap.conn.delete_folder(folder)
+                    imap.conn.folder.delete(folder)
                     # We just deleted the folder, so we have to make DC forget about it, too
                     if account.get_config("configured_sentbox_folder") == folder:
                         account.set_config("configured_sentbox_folder", None)
-                    if account.get_config("configured_spam_folder") == folder:
-                        account.set_config("configured_spam_folder", None)
 
             setattr(account, "direct_imap", imap)
 
@@ -88,37 +83,34 @@ class DirectImap:
             ssl_context.verify_mode = ssl.CERT_NONE
 
         if security == const.DC_SOCKET_STARTTLS:
-            self.conn = IMAPClient(host, port, ssl=False)
-            self.conn.starttls(ssl_context)
-        elif security == const.DC_SOCKET_PLAIN:
-            self.conn = IMAPClient(host, port, ssl=False)
-        elif security == const.DC_SOCKET_SSL:
-            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
+            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
+        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
+            self.conn = MailBox(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
 
     def shutdown(self):
         try:
-            self.conn.idle_done()
-        except (OSError, IMAPClientError):
+            self.idle_done()
+        except (OSError, imaplib.IMAP4.abort):
             pass
         try:
             self.conn.logout()
-        except (OSError, IMAPClientError):
+        except (OSError, imaplib.IMAP4.abort):
             print("Could not logout direct_imap conn")
 
     def create_folder(self, foldername):
         try:
-            self.conn.create_folder(foldername)
-        except imaplib.IMAP4.error as e:
+            self.conn.folder.create(foldername)
+        except errors.MailboxFolderCreateError as e:
             print("Can't create", foldername, "probably it already exists:", str(e))
 
-    def select_folder(self, foldername):
+    def select_folder(self, foldername: str) -> tuple:
         assert not self._idling
-        return self.conn.select_folder(foldername)
+        return self.conn.folder.set(foldername)
 
-    def select_config_folder(self, config_name):
+    def select_config_folder(self, config_name: str):
         """ Return info about selected folder if it is
         configured, otherwise None. """
         if "_" not in config_name:
@@ -127,50 +119,36 @@ class DirectImap:
         if foldername:
             return self.select_folder(foldername)
 
-    def list_folders(self):
+    def list_folders(self) -> List[str]:
         """ return list of all existing folder names"""
         assert not self._idling
-        folders = []
-        for meta, sep, foldername in self.conn.list_folders():
-            folders.append(foldername)
-        return folders
+        return [folder.name for folder in self.conn.folder.list()]
 
-    def delete(self, range, expunge=True):
+    def delete(self, uid_list: str, expunge=True):
         """ delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.set_flags(range, [DELETED])
+        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
         if expunge:
             self.conn.expunge()
 
-    def get_all_messages(self):
+    def get_all_messages(self) -> List[MailMessage]:
         assert not self._idling
+        return [mail for mail in self.conn.fetch()]
 
-        # Flush unsolicited responses. IMAPClient has problems
-        # dealing with them: https://github.com/mjs/imapclient/issues/334
-        # When this NOOP was introduced, next FETCH returned empty
-        # result instead of a single message, even though IMAP server
-        # can only return more untagged responses than required, not
-        # less.
-        self.conn.noop()
-
-        return self.conn.fetch(ALL, [FLAGS])
-
-    def get_unread_messages(self):
+    def get_unread_messages(self) -> List[str]:
         assert not self._idling
-        res = self.conn.fetch(ALL, [FLAGS])
-        return [uid for uid in res
-                if SEEN not in res[uid][FLAGS]]
+        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
         if messages:
-            res = self.conn.set_flags(messages, [SEEN])
+            res = self.conn.flag(messages, MailMessageFlags.SEEN, True)
             print("marked seen:", messages, res)
 
-    def get_unread_cnt(self):
+    def get_unread_cnt(self) -> int:
         return len(self.get_unread_messages())
 
     def dump_imap_structures(self, dir, logfile):
@@ -192,21 +170,20 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[]', FLAGS]
-            for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[]']
-                if not body_bytes:
-                    log("Message", uid, "has empty body")
+            for msg in self.conn.fetch(mark_seen=False):
+                body = getattr(msg.obj, "text", None)
+                if not body:
+                    body = getattr(msg.obj, "html", None)
+                if not body:
+                    log("Message", msg.uid, "has empty body")
                     continue
 
-                flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(uid))
-                fn.write_bytes(body_bytes)
-                log("Message", uid, fn)
-                email_message = email.message_from_bytes(body_bytes)
-                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+                fn = path.joinpath(str(msg.uid))
+                fn.write_bytes(body)
+                log("Message", msg.uid, fn)
+                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
@@ -216,49 +193,58 @@ class DirectImap:
     def idle_start(self):
         """ switch this connection to idle mode. non-blocking. """
         assert not self._idling
-        res = self.conn.idle()
+        res = self.conn.idle.start()
         self._idling = True
         return res
 
-    def idle_check(self, terminate=False):
+    def idle_check(self, terminate=False, timeout=None) -> List[bytes]:
         """ (blocking) wait for next idle message from server. """
         assert self._idling
         self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check()
+        res = self.conn.idle.poll(timeout=timeout)
         if terminate:
             self.idle_done()
         self.account.log("imap-direct: idle_check returned {!r}".format(res))
         return res
 
-    def idle_wait_for_seen(self):
-        """ Return first message with SEEN flag
-        from a running idle-stream REtiurn.
+    def idle_wait_for_new_message(self, terminate=False, timeout=None) -> bytes:
+        while 1:
+            for item in self.idle_check(timeout=timeout):
+                if b'EXISTS' in item or b'RECENT' in item:
+                    if terminate:
+                        self.idle_done()
+                    return item
+
+    def idle_wait_for_seen(self, terminate=False, timeout=None) -> int:
+        """ Return first message with SEEN flag from a running idle-stream.
         """
         while 1:
-            for item in self.idle_check():
-                if item[1] == FETCH:
-                    if item[2][0] == FLAGS:
-                        if SEEN in item[2][1]:
-                            return item[0]
+            for item in self.idle_check(timeout=timeout):
+                if FETCH in item:
+                    self.account.log(str(item))
+                    if FLAGS in item and rb'\Seen' in item:
+                        if terminate:
+                            self.idle_done()
+                        return int(item.split(b' ')[1])
 
     def idle_done(self):
         """ send idle-done to server if we are currently in idle mode. """
         if self._idling:
-            res = self.conn.idle_done()
+            res = self.conn.idle.stop()
             self._idling = False
             return res
 
-    def append(self, folder, msg):
+    def append(self, folder: str, msg: str):
         """Upload a message to *folder*.
         Trailing whitespace or a linebreak at the beginning will be removed automatically.
         """
         if msg.startswith("\n"):
             msg = msg[1:]
         msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(folder, msg)
+        self.conn.append(bytes(msg, encoding='ascii'), folder)
 
-    def get_uid_by_message_id(self, message_id):
-        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
+    def get_uid_by_message_id(self, message_id) -> str:
+        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
         if len(msgs) == 0:
             raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
         return msgs[0]

python/tests/bench_test_setup.py

@@ -0,0 +1,18 @@
+"""
+run with:
+
+     pytest -vv --durations=10 bench_empty.py
+
+to see timings of test setups.
+"""
+
+import pytest
+
+
+class TestEmpty:
+    def test_prepare_setup_measurings(self, acfactory):
+        acfactory.get_many_online_accounts(5)
+
+    @pytest.mark.parametrize("num", range(0, 5))
+    def test_setup_online_accounts(self, acfactory, num):
+        acfactory.get_many_online_accounts(num)

python/tests/test_account.py

@@ -11,6 +11,7 @@ from deltachat.hookspec import account_hookimpl
 from deltachat.capi import ffi, lib
 from deltachat.cutil import iter_array
 from datetime import datetime, timedelta, timezone
+from imap_tools import AND, U
 
 
 @pytest.mark.parametrize("msgtext,res", [
@@ -959,6 +960,18 @@ class TestOnlineAccount:
         chat.send_text("hello")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    def test_send_dot(self, acfactory, lp):
+        """Test that a single dot is properly escaped in SMTP protocol"""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("sending message")
+        msg_out = chat.send_text(".")
+
+        lp.sec("receiving message")
+        msg_in = ac2._evtracker.wait_next_incoming_message()
+        assert msg_in.text == msg_out.text
+
     def test_send_and_receive_message_markseen(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -1013,7 +1026,7 @@ class TestOnlineAccount:
         assert ev.data1 == msg2.chat.id
         assert ev.data2 == 0
 
-        ac2.direct_imap.idle_check(terminate=True)
+        ac2.direct_imap.idle_wait_for_new_message(terminate=True)
         lp.step("1")
         for i in range(2):
             ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
@@ -1044,8 +1057,7 @@ class TestOnlineAccount:
 
         ac1.create_chat(ac2).send_text("Hello!")
 
-        # Wait for the message to arrive.
-        ac2.direct_imap.idle_check(terminate=True)
+        ac2.direct_imap.idle_wait_for_new_message(terminate=True)
 
         # Emulate moving of the message to DeltaChat folder by Sieve rule.
         # mailcow server contains this rule by default.
@@ -1059,13 +1071,9 @@ class TestOnlineAccount:
         # Accept the contact request.
         msg.chat.accept()
         ac2.mark_seen_messages([msg])
-        ac2.direct_imap.idle_wait_for_seen()
-        ac2.direct_imap.idle_done()
+        uid = ac2.direct_imap.idle_wait_for_seen(terminate=True)
 
-        fetch = list(ac2.direct_imap.conn.fetch("*", b'FLAGS').values())
-        flags = fetch[-1][b'FLAGS']
-        is_seen = b'\\Seen' in flags
-        assert is_seen
+        assert len([a for a in ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*")))]) == 1
 
     def test_multidevice_sync_seen(self, acfactory, lp):
         """Test that message marked as seen on one device is marked as seen on another."""
@@ -1437,6 +1445,7 @@ class TestOnlineAccount:
         ac1.direct_imap.create_folder("Drafts")
         ac1.direct_imap.create_folder("Sent")
         ac1.direct_imap.create_folder("Spam")
+        ac1.direct_imap.create_folder("Junk")
 
         acfactory.wait_configure_and_start_io()
         # Wait until each folder was selected once and we are IDLEing again:
@@ -1470,6 +1479,15 @@ class TestOnlineAccount:
 
             Unknown message in Spam
         """.format(ac1.get_config("configured_addr")))
+        ac1.direct_imap.append("Junk", """
+            From: unknown.address@junk.org
+            Subject: subj
+            To: {}
+            Message-ID: <spam.message@junk.org>
+            Content-Type: text/plain; charset=utf-8
+
+            Unknown message in Junk
+        """.format(ac1.get_config("configured_addr")))
 
         ac1.set_config("scan_all_folders_debounce_secs", "0")
         lp.sec("All prepared, now let DC find the message")
@@ -2211,7 +2229,7 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.create_chat(ac1).send_text("Hi")
 
-        ac1.direct_imap.idle_check(terminate=False)
+        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
         ac1.maybe_network()
 
         ac1._evtracker.wait_for_all_work_done()
@@ -2223,8 +2241,6 @@ class TestOnlineAccount:
 
         ac2.create_chat(ac1).send_text("Hi 2")
 
-        ac1.direct_imap.idle_check(terminate=True)
-        ac1.maybe_network()
         ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTED, const.DC_CONNECTIVITY_WORKING)
         ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_WORKING, const.DC_CONNECTIVITY_CONNECTED)
 
@@ -2248,7 +2264,7 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.create_chat(ac1).send_text("Hi")
 
-        ac1.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
         ac1.maybe_network()
 
         while 1:
@@ -2438,25 +2454,23 @@ class TestOnlineAccount:
 
         acfactory.wait_configure_and_start_io()
 
-        imap2 = ac2.direct_imap
-        imap2.idle_start()
-
         lp.sec("ac1: create chat with ac2")
         chat1 = ac1.create_chat(ac2)
         ac2.create_chat(ac1)
 
+        lp.sec("ac1: send message to ac2")
         sent_msg = chat1.send_text("hello")
-        imap2.idle_check(terminate=False)
 
         msg = ac2._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        imap2.idle_check(terminate=True)
+        lp.sec("ac2: wait for close/expunge on autodelete")
         ac2._evtracker.get_info_contains("close/expunge succeeded")
 
-        assert len(imap2.get_all_messages()) == 0
+        lp.sec("ac2: check that message was autodeleted on server")
+        assert len(ac2.direct_imap.get_all_messages()) == 0
 
-        # Mark deleted message as seen and check that read receipt arrives
+        lp.sec("ac2: Mark deleted message as seen and check that read receipt arrives")
         msg.mark_seen()
         ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
         assert ev.data1 == chat1.id
@@ -2544,10 +2558,9 @@ class TestOnlineAccount:
 
         ac2._evtracker.get_info_contains("close/expunge succeeded")
 
-        lp.sec("imap2: test that only one message is left")
-        imap2 = ac2.direct_imap
-
-        assert len(imap2.get_all_messages()) == 1
+        lp.sec("ac2: test that only one message is left")
+        ac2.direct_imap.select_config_folder("inbox")
+        assert len(ac2.direct_imap.get_all_messages()) == 1
 
     def test_configure_error_msgs(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2717,13 +2730,7 @@ class TestOnlineAccount:
         ac1.direct_imap.select_config_folder("inbox")
         ac1.direct_imap.idle_start()
         acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
-        while True:
-            if len(ac1.direct_imap.idle_check(terminate=True)) > 1:
-                # If length is 1, it's [(b'OK', b'Still here')]
-                # Could happen on very slow network.
-                #
-                # More is usually [(1, b'EXISTS'), (1, b'RECENT')]
-                break
+        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
         ac1.direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
 
         lp.sec("Everything prepared, now see if DeltaChat finds the message (" + variant + ")")
@@ -2772,10 +2779,7 @@ class TestOnlineAccount:
         acfactory.wait_configure_and_start_io()
         assert_folders_configured(ac1)
 
-        if mvbox_move:
-            ac1.direct_imap.select_config_folder("mvbox")
-        else:
-            ac1.direct_imap.select_folder("INBOX")
+        assert ac1.direct_imap.select_config_folder("mvbox" if mvbox_move else "inbox")
         ac1.direct_imap.idle_start()
 
         lp.sec("send out message with bcc to ourselves")
@@ -2784,22 +2788,24 @@ class TestOnlineAccount:
         chat.send_text("message text")
         assert_folders_configured(ac1)
 
-        # now wait until the bcc_self message arrives
-        # Also test that bcc_self messages moved to the mvbox are marked as read.
+        lp.sec("wait until the bcc_self message arrives in correct folder and is marked seen")
         assert ac1.direct_imap.idle_wait_for_seen()
         assert_folders_configured(ac1)
 
+        lp.sec("create a cloned ac1 and fetch contact history during configure")
         ac1_clone = acfactory.clone_online_account(ac1)
         ac1_clone.set_config("fetch_existing_msgs", "1")
         ac1_clone._configtracker.wait_finish()
         ac1_clone.start_io()
         assert_folders_configured(ac1_clone)
 
+        lp.sec("check that ac2 contact was fetchted during configure")
         ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
         ac2_addr = ac2.get_config("addr")
         assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
         assert_folders_configured(ac1_clone)
 
+        lp.sec("check that messages changed events arrive for the correct message")
         msg = ac1_clone._evtracker.wait_next_messages_changed()
         assert msg.text == "message text"
         assert_folders_configured(ac1)
@@ -2858,15 +2864,15 @@ class TestOnlineAccount:
         ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure(ac1)
 
-        ac1.direct_imap.conn.delete_folder("DeltaChat")
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 0
+        ac1.direct_imap.conn.folder.delete("DeltaChat")
+        assert "DeltaChat" not in ac1.direct_imap.list_folders()
         acfactory.wait_configure_and_start_io()
 
         ac2.create_chat(ac1).send_text("hello")
         msg = ac1._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 1
+        assert "DeltaChat" in ac1.direct_imap.list_folders()
 
 
 class TestGroupStressTests:

python/tox.ini

@@ -8,7 +8,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n1 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 

rust-toolchain

@@ -1 +1 @@
-1.59.0
+1.60.0

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.59.0
+RUST_VERSION=1.60.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.59.0
+RUST_VERSION=1.60.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

src/accounts.rs

@@ -54,7 +54,11 @@ impl Accounts {
         ensure!(dir.exists().await, "directory does not exist");
 
         let config_file = dir.join(CONFIG_NAME);
-        ensure!(config_file.exists().await, "accounts.toml does not exist");
+        ensure!(
+            config_file.exists().await,
+            "{:?} does not exist",
+            config_file
+        );
 
         let config = Config::from_file(config_file)
             .await
@@ -144,9 +148,27 @@ impl Accounts {
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id).await {
-            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
-                .await
-                .context("failed to remove account data")?;
+            // Spend up to 1 minute trying to remove the files.
+            // Files may remain locked up to 30 seconds due to r2d2 bug:
+            // https://github.com/sfackler/r2d2/issues/99
+            let mut counter = 0;
+            loop {
+                counter += 1;
+
+                if let Err(err) = fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                    .await
+                    .context("failed to remove account data")
+                {
+                    if counter > 60 {
+                        return Err(err);
+                    }
+
+                    // Wait 1 second and try again.
+                    async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
+                } else {
+                    break;
+                }
+            }
         }
         self.config.remove_account(id).await?;
 

src/blob.rs

@@ -940,7 +940,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_recode_image() {
+    async fn test_recode_image_1() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
         send_image_check_mediaquality(Some("0"), bytes, 1000, 1000, 0, 1000, 1000)
@@ -957,7 +957,10 @@ mod tests {
         )
         .await
         .unwrap();
+    }
 
+    #[async_std::test]
+    async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(
@@ -978,18 +981,24 @@ mod tests {
             .write_to(&mut buf, image::ImageFormat::Jpeg)
             .unwrap();
         let bytes = buf.into_inner();
-        let img_rotated = send_image_check_mediaquality(
-            Some("0"),
-            &bytes,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-            0,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-        assert_correct_rotation(&img_rotated);
+
+        // Do this in parallel to speed up the test a bit
+        // (it still takes very long though)
+        let bytes2 = bytes.clone();
+        let join_handle = async_std::task::spawn(async move {
+            let img_rotated = send_image_check_mediaquality(
+                Some("0"),
+                &bytes2,
+                BALANCED_IMAGE_SIZE * 1800 / 2000,
+                BALANCED_IMAGE_SIZE,
+                0,
+                BALANCED_IMAGE_SIZE * 1800 / 2000,
+                BALANCED_IMAGE_SIZE,
+            )
+            .await
+            .unwrap();
+            assert_correct_rotation(&img_rotated);
+        });
 
         let img_rotated = send_image_check_mediaquality(
             Some("1"),
@@ -1004,6 +1013,11 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
+        join_handle.await;
+    }
+
+    #[async_std::test]
+    async fn test_recode_image_3() {
         let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
             .await

src/chatlist.rs

@@ -4,13 +4,11 @@ use anyhow::{ensure, Context as _, Result};
 
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
 use crate::constants::{
-    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CONTACT_ID_DEVICE,
-    DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT, DC_GCL_ARCHIVED_ONLY,
-    DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_GCL_ADD_ALLDONE_HINT,
+    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
 };
 use crate::contact::{Contact, ContactId};
 use crate::context::Context;
-use crate::ephemeral::delete_expired_messages;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock_str;
 use crate::summary::Summary;
@@ -92,12 +90,6 @@ impl Chatlist {
         let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
         let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
 
-        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
-        // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_expired_messages(context).await {
-            warn!(context, "Failed to hide expired messages: {}", err);
-        }
-
         let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
@@ -112,7 +104,7 @@ impl Chatlist {
         };
 
         let skip_id = if flag_for_forwarding {
-            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+            ChatId::lookup_by_contact(context, ContactId::DEVICE)
                 .await?
                 .unwrap_or_default()
         } else {
@@ -216,7 +208,7 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+                ChatId::lookup_by_contact(context, ContactId::SELF)
                     .await?
                     .unwrap_or_default()
             } else {
@@ -326,7 +318,7 @@ impl Chatlist {
 
         let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
             let lastmsg = Message::load_from_db(context, lastmsg_id).await?;
-            if lastmsg.from_id == DC_CONTACT_ID_SELF {
+            if lastmsg.from_id == ContactId::SELF {
                 (Some(lastmsg), None)
             } else {
                 match chat.typ {
@@ -343,7 +335,7 @@ impl Chatlist {
 
         if chat.id.is_archived_link() {
             Ok(Default::default())
-        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED) {
+        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != ContactId::UNDEFINED) {
             Ok(Summary::new(context, &lastmsg, chat, lastcontact.as_ref()).await)
         } else {
             Ok(Summary {
@@ -356,6 +348,10 @@ impl Chatlist {
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
         self.ids.iter().position(|(chat_id, _)| chat_id == &id)
     }
+
+    pub fn iter(&self) -> impl Iterator<Item = &(ChatId, Option<MsgId>)> {
+        self.ids.iter()
+    }
 }
 
 /// Returns the number of archived chats
@@ -507,7 +503,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
                  \n\
                  hello foo\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -568,7 +563,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:38:57 +0000\n\
                  \n\
                  hello foo\n",
-            "INBOX",
             false,
         )
         .await?;

src/config.rs

@@ -1,16 +1,15 @@
 //! # Key-value configuration management.
 
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Context as _, Result};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
-use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
-use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
 
@@ -113,6 +112,7 @@ pub enum Config {
     DeleteDeviceAfter,
 
     SaveMimeHeaders,
+    /// The primary email address. Also see `SecondaryAddrs`.
     ConfiguredAddr,
     ConfiguredMailServer,
     ConfiguredMailUser,
@@ -131,11 +131,14 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
-    ConfiguredSpamFolder,
     ConfiguredTimestamp,
     ConfiguredProvider,
     Configured,
 
+    /// All secondary self addresses separated by spaces
+    /// (`addr1@example.org addr2@exapmle.org addr3@example.org`)
+    SecondaryAddrs,
+
     #[strum(serialize = "sys.version")]
     SysVersion,
 
@@ -293,11 +296,8 @@ impl Context {
             }
             Config::DeleteDeviceAfter => {
                 let ret = self.sql.set_raw_config(key, value).await;
-                // Force chatlist reload to delete old messages immediately.
-                self.emit_event(EventType::MsgsChanged {
-                    msg_id: MsgId::new(0),
-                    chat_id: ChatId::new(0),
-                });
+                // Interrupt ephemeral loop to delete old messages immediately.
+                self.interrupt_ephemeral_task().await;
                 ret?
             }
             Config::Displayname => {
@@ -333,6 +333,73 @@ impl Context {
     }
 }
 
+// Separate impl block for self address handling
+impl Context {
+    /// Determine whether the specified addr maps to the/a self addr.
+    /// Returns `false` if no addresses are configured.
+    pub(crate) async fn is_self_addr(&self, addr: &str) -> Result<bool> {
+        Ok(self
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .iter()
+            .any(|a| addr_cmp(addr, a))
+            || self
+                .get_secondary_self_addrs()
+                .await?
+                .iter()
+                .any(|a| addr_cmp(addr, a)))
+    }
+
+    /// Sets `primary_new` as the new primary self address and saves the old
+    /// primary address (if exists) as a secondary address.
+    ///
+    /// This should only be used by test code and during configure.
+    pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
+        // add old primary address (if exists) to secondary addresses
+        let mut secondary_addrs = self.get_all_self_addrs().await?;
+        // never store a primary address also as a secondary
+        secondary_addrs.retain(|a| !addr_cmp(a, primary_new));
+        self.set_config(
+            Config::SecondaryAddrs,
+            Some(secondary_addrs.join(" ").as_str()),
+        )
+        .await?;
+
+        self.set_config(Config::ConfiguredAddr, Some(primary_new))
+            .await?;
+
+        Ok(())
+    }
+
+    /// Returns all primary and secondary self addresses.
+    pub(crate) async fn get_all_self_addrs(&self) -> Result<Vec<String>> {
+        let primary_addrs = self.get_config(Config::ConfiguredAddr).await?.into_iter();
+        let secondary_addrs = self.get_secondary_self_addrs().await?.into_iter();
+
+        Ok(primary_addrs.chain(secondary_addrs).collect())
+    }
+
+    /// Returns all secondary self addresses.
+    pub(crate) async fn get_secondary_self_addrs(&self) -> Result<Vec<String>> {
+        let secondary_addrs = self
+            .get_config(Config::SecondaryAddrs)
+            .await?
+            .unwrap_or_default();
+        Ok(secondary_addrs
+            .split_ascii_whitespace()
+            .map(|s| s.to_string())
+            .collect())
+    }
+
+    /// Returns the primary self address.
+    /// Returns an error if no self addr is configured.
+    pub async fn get_primary_self_addr(&self) -> Result<String> {
+        self.get_config(Config::ConfiguredAddr)
+            .await?
+            .context("No self addr configured")
+    }
+}
+
 /// Returns all available configuration keys concated together.
 fn get_config_keys_string() -> String {
     let keys = Config::iter().fold(String::new(), |mut acc, key| {
@@ -417,4 +484,57 @@ mod tests {
         assert_eq!(t.get_config_bool(c).await?, false);
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_self_addrs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        assert!(alice.is_self_addr("alice@example.org").await?);
+        assert_eq!(alice.get_all_self_addrs().await?, vec!["alice@example.org"]);
+        assert!(!alice.is_self_addr("alice@alice.com").await?);
+
+        // Test adding the same primary address
+        alice.set_primary_self_addr("alice@example.org").await?;
+        alice.set_primary_self_addr("Alice@Example.Org").await?;
+        assert_eq!(alice.get_all_self_addrs().await?, vec!["Alice@Example.Org"]);
+
+        // Test adding a new (primary) self address
+        // The address is trimmed during by `LoginParam::from_database()`,
+        // so `set_primary_self_addr()` doesn't have to trim it.
+        alice.set_primary_self_addr(" Alice@alice.com ").await?;
+        assert!(alice.is_self_addr("    aliCe@example.org").await?);
+        assert!(alice.is_self_addr("alice@alice.com").await?);
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec![" Alice@alice.com ", "Alice@Example.Org"]
+        );
+
+        // Check that the entry is not duplicated
+        alice.set_primary_self_addr("alice@alice.com").await?;
+        alice.set_primary_self_addr("alice@alice.com").await?;
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec!["alice@alice.com", "Alice@Example.Org"]
+        );
+
+        // Test switching back
+        alice.set_primary_self_addr("alice@example.org").await?;
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec!["alice@example.org", "alice@alice.com"]
+        );
+
+        // Test setting a new primary self address, the previous self address
+        // should be kept as a secondary self address
+        alice.set_primary_self_addr("alice@alice.xyz").await?;
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec!["alice@alice.xyz", "alice@example.org", "alice@alice.com"]
+        );
+        assert!(alice.is_self_addr("alice@example.org").await?);
+        assert!(alice.is_self_addr("alice@alice.com").await?);
+        assert!(alice.is_self_addr("Alice@alice.xyz").await?);
+
+        Ok(())
+    }
 }

src/configure.rs

@@ -85,7 +85,7 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let mut param = LoginParam::from_database(self, "").await?;
+        let mut param = LoginParam::load_candidate_params(self).await?;
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
@@ -453,8 +453,14 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     drop(imap);
 
     progress!(ctx, 910);
+
+    if ctx.get_config(Config::ConfiguredAddr).await?.as_deref() != Some(&param.addr) {
+        // Switched account, all server UIDs we know are invalid
+        job::schedule_resync(ctx).await?;
+    }
+
     // the trailing underscore is correct
-    param.save_to_database(ctx, "configured_").await?;
+    param.save_as_configured_params(ctx).await?;
     ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
@@ -699,11 +705,11 @@ pub enum Error {
         error: quick_xml::Error,
     },
 
-    #[error("Failed to get URL: {0}")]
-    ReadUrl(#[from] self::read_url::Error),
-
     #[error("Number of redirection is exceeded")]
     Redirection,
+
+    #[error("{0:#}")]
+    Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]

src/configure/read_url.rs

@@ -1,20 +1,40 @@
 use crate::context::Context;
 
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("URL request error")]
-    GetError(surf::Error),
-}
+use anyhow::format_err;
+use anyhow::Context as _;
 
-pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
-    info!(context, "Requesting URL {}", url);
-
-    match surf::get(url).recv_string().await {
-        Ok(res) => Ok(res),
-        Err(err) => {
-            info!(context, "Can\'t read URL {}: {}", url, err);
-
-            Err(Error::GetError(err))
+pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
+    match read_url_inner(context, url).await {
+        Ok(s) => {
+            info!(context, "Successfully read url {}", url);
+            Ok(s)
+        }
+        Err(e) => {
+            info!(context, "Can't read URL {}: {:#}", url, e);
+            Err(format_err!("Can't read URL {}: {:#}", url, e))
         }
     }
 }
+
+pub async fn read_url_inner(context: &Context, mut url: &str) -> anyhow::Result<String> {
+    let mut _temp; // For the borrow checker
+
+    // Follow up to 10 http-redirects
+    for _i in 0..10 {
+        let mut response = surf::get(url).send().await.map_err(|e| e.into_inner())?;
+        if response.status().is_redirection() {
+            _temp = response
+                .header("location")
+                .context("Redirection doesn't have a target location")?
+                .last()
+                .to_string();
+            info!(context, "Following redirect to {}", _temp);
+            url = &_temp;
+            continue;
+        }
+
+        return response.body_string().await.map_err(|e| e.into_inner());
+    }
+
+    Err(format_err!("Followed 10 redirections"))
+}

src/configure/server_params.rs

@@ -52,10 +52,8 @@ impl ServerParams {
     fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
         if self.hostname.is_empty() {
             vec![
-                Self {
-                    hostname: param_domain.to_string(),
-                    ..self.clone()
-                },
+                // Try "imap.ex.org"/"smtp.ex.org" and "mail.ex.org" first because if a server exists
+                // under this address, it's likely the correct one.
                 Self {
                     hostname: match self.protocol {
                         Protocol::Imap => "imap.".to_string() + param_domain,
@@ -65,6 +63,12 @@ impl ServerParams {
                 },
                 Self {
                     hostname: "mail.".to_string() + param_domain,
+                    ..self.clone()
+                },
+                // Try "ex.org" last because if it's wrong and the server is configured to
+                // not answer at all, configuration may be stuck for several minutes.
+                Self {
+                    hostname: param_domain.to_string(),
                     ..self
                 },
             ]
@@ -296,5 +300,48 @@ mod tests {
                 strict_tls: Some(true)
             }],
         );
+
+        // Test that "example.net" is tried after "*.example.net".
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Imap,
+                hostname: "".to_string(),
+                port: 10480,
+                socket: Socket::Ssl,
+                username: "foobar".to_string(),
+                strict_tls: Some(true),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+        assert_eq!(
+            v,
+            vec![
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "imap.example.net".to_string(),
+                    port: 10480,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                },
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "mail.example.net".to_string(),
+                    port: 10480,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                },
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "example.net".to_string(),
+                    port: 10480,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                }
+            ],
+        );
     }
 }

src/constants.rs

@@ -4,7 +4,6 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::ChatId;
-use crate::contact::ContactId;
 
 pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION").to_string());
 
@@ -180,16 +179,6 @@ pub const DC_ELLIPSIS: &str = "[...]";
 /// `char`s), not Unicode Grapheme Clusters.
 pub const DC_DESIRED_TEXT_LEN: usize = 5000;
 
-pub const DC_CONTACT_ID_UNDEFINED: ContactId = ContactId::new(0);
-pub const DC_CONTACT_ID_SELF: ContactId = ContactId::new(1);
-pub const DC_CONTACT_ID_INFO: ContactId = ContactId::new(2);
-pub const DC_CONTACT_ID_DEVICE: ContactId = ContactId::new(5);
-pub const DC_CONTACT_ID_LAST_SPECIAL: ContactId = ContactId::new(9);
-
-// decorative address that is used for DC_CONTACT_ID_DEVICE
-// when an api that returns an email is called.
-pub const DC_CONTACT_ID_DEVICE_ADDR: &str = "device@localhost";
-
 // Flags for empty server job
 
 pub const DC_EMPTY_MVBOX: u32 = 0x01;

src/contact.rs

@@ -14,10 +14,7 @@ use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
 use crate::color::str_to_color;
 use crate::config::Config;
-use crate::constants::{
-    Blocked, Chattype, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_DEVICE_ADDR, DC_CONTACT_ID_LAST_SPECIAL,
-    DC_CONTACT_ID_SELF, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY,
-};
+use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input, EmailAddress};
 use crate::events::EventType;
@@ -27,24 +24,51 @@ use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
+use crate::sql::{self, params_iter};
 use crate::{chat, stock_str};
 
 /// Contact ID, including reserved IDs.
 ///
 /// Some contact IDs are reserved to identify special contacts.  This
 /// type can represent both the special as well as normal contacts.
-#[derive(
-    Debug, Copy, Clone, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize,
-)]
+#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct ContactId(u32);
 
 impl ContactId {
+    pub const UNDEFINED: ContactId = ContactId::new(0);
+    /// The owner of the account.
+    ///
+    /// The email-address is set by `dc_set_config` using "addr".
+    pub const SELF: ContactId = ContactId::new(1);
+    pub const INFO: ContactId = ContactId::new(2);
+    pub const DEVICE: ContactId = ContactId::new(5);
+    const LAST_SPECIAL: ContactId = ContactId::new(9);
+
+    /// Address to go with [`ContactId::DEVICE`].
+    ///
+    /// This is used by APIs which need to return an email address for this contact.
+    pub const DEVICE_ADDR: &'static str = "device@localhost";
+
     /// Creates a new [`ContactId`].
     pub const fn new(id: u32) -> ContactId {
         ContactId(id)
     }
 
-    /// Bad evil escape hatch, do not use.
+    /// Whether this is a special [`ContactId`].
+    ///
+    /// Some [`ContactId`]s are reserved for special contacts like [`ContactId::SELF`],
+    /// [`ContactId::INFO`] and [`ContactId::DEVICE`].  This function indicates whether this
+    /// [`ContactId`] is any of the reserved special [`ContactId`]s (`true`) or whether it
+    /// is the [`ContactId`] of a real contact (`false`).
+    pub fn is_special(&self) -> bool {
+        self.0 <= Self::LAST_SPECIAL.0
+    }
+
+    /// Numerical representation of the [`ContactId`].
+    ///
+    /// Each contact ID has a unique numerical representation which is used in the database
+    /// (via [`rusqlite::ToSql`]) and also for FFI purposes.  In Rust code you should never
+    /// need to use this directly.
     pub const fn to_u32(&self) -> u32 {
         self.0
     }
@@ -52,20 +76,19 @@ impl ContactId {
 
 impl fmt::Display for ContactId {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.0)
-        // if *self == DC_CONTACT_ID_UNDEFINED {
-        //     write!(f, "Contact#Undefined")
-        // } else if *self == DC_CONTACT_ID_SELF {
-        //     write!(f, "Contact#Self")
-        // } else if *self == DC_CONTACT_ID_INFO {
-        //     write!(f, "Contact#Info")
-        // } else if *self == DC_CONTACT_ID_DEVICE {
-        //     write!(f, "Contact#Device")
-        // } else if *self <= DC_CONTACT_ID_LAST_SPECIAL {
-        //     write!(f, "Contact#Special{}", self.0)
-        // } else {
-        //     write!(f, "Contact#{}", self.0)
-        // }
+        if *self == ContactId::UNDEFINED {
+            write!(f, "Contact#Undefined")
+        } else if *self == ContactId::SELF {
+            write!(f, "Contact#Self")
+        } else if *self == ContactId::INFO {
+            write!(f, "Contact#Info")
+        } else if *self == ContactId::DEVICE {
+            write!(f, "Contact#Device")
+        } else if self.is_special() {
+            write!(f, "Contact#Special{}", self.0)
+        } else {
+            write!(f, "Contact#{}", self.0)
+        }
     }
 }
 
@@ -104,12 +127,6 @@ impl rusqlite::types::FromSql for ContactId {
 #[derive(Debug)]
 pub struct Contact {
     /// The contact ID.
-    ///
-    /// Special message IDs:
-    /// - DC_CONTACT_ID_SELF (1) - this is the owner of the account with the email-address set by
-    ///   `dc_set_config` using "addr".
-    ///
-    /// Normal contact IDs are larger than these special ones (larger than DC_CONTACT_ID_LAST_SPECIAL).
     pub id: ContactId,
 
     /// Contact name. It is recommended to use `Contact::get_name`,
@@ -278,7 +295,7 @@ impl Contact {
                 },
             )
             .await?;
-        if contact_id == DC_CONTACT_ID_SELF {
+        if contact_id == ContactId::SELF {
             contact.name = stock_str::self_msg(context).await;
             contact.addr = context
                 .get_config(Config::ConfiguredAddr)
@@ -288,9 +305,9 @@ impl Contact {
                 .get_config(Config::Selfstatus)
                 .await?
                 .unwrap_or_default();
-        } else if contact_id == DC_CONTACT_ID_DEVICE {
+        } else if contact_id == ContactId::DEVICE {
             contact.name = stock_str::device_messages(context).await;
-            contact.addr = DC_CONTACT_ID_DEVICE_ADDR.to_string();
+            contact.addr = ContactId::DEVICE_ADDR.to_string();
             contact.status = stock_str::device_messages_hint(context).await;
         }
         Ok(contact)
@@ -382,22 +399,17 @@ impl Contact {
 
         let addr_normalized = addr_normalize(addr);
 
-        if let Some(addr_self) = context.get_config(Config::ConfiguredAddr).await? {
-            if addr_cmp(addr_normalized, &addr_self) {
-                return Ok(Some(DC_CONTACT_ID_SELF));
-            }
+        if context.is_self_addr(addr_normalized).await? {
+            return Ok(Some(ContactId::SELF));
         }
+
         let id = context
             .sql
             .query_get_value(
                 "SELECT id FROM contacts \
             WHERE addr=?1 COLLATE NOCASE \
             AND id>?2 AND origin>=?3 AND blocked=0;",
-                paramsv![
-                    addr_normalized,
-                    DC_CONTACT_ID_LAST_SPECIAL,
-                    min_origin as u32,
-                ],
+                paramsv![addr_normalized, ContactId::LAST_SPECIAL, min_origin as u32,],
             )
             .await?;
         Ok(id)
@@ -440,13 +452,9 @@ impl Contact {
         ensure!(origin != Origin::Unknown, "Missing valid origin");
 
         let addr = addr_normalize(addr).to_string();
-        let addr_self = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
 
-        if addr_cmp(&addr, &addr_self) {
-            return Ok((DC_CONTACT_ID_SELF, sth_modified));
+        if context.is_self_addr(&addr).await? {
+            return Ok((ContactId::SELF, sth_modified));
         }
 
         if !may_be_valid_addr(&addr) {
@@ -680,11 +688,7 @@ impl Contact {
         listflags: u32,
         query: Option<impl AsRef<str>>,
     ) -> Result<Vec<ContactId>> {
-        let self_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
-
+        let self_addrs = context.get_all_self_addrs().await?;
         let mut add_self = false;
         let mut ret = Vec::new();
         let flag_verified_only = (listflags & DC_GCL_VERIFIED_ONLY) != 0;
@@ -695,23 +699,25 @@ impl Contact {
             context
                 .sql
                 .query_map(
-                    "SELECT c.id FROM contacts c \
+                    format!(
+                        "SELECT c.id FROM contacts c \
                  LEFT JOIN acpeerstates ps ON c.addr=ps.addr  \
-                 WHERE c.addr!=?1 \
-                 AND c.id>?2 \
-                 AND c.origin>=?3 \
+                 WHERE c.addr NOT IN ({})
+                 AND c.id>? \
+                 AND c.origin>=? \
                  AND c.blocked=0 \
-                 AND (iif(c.name='',c.authname,c.name) LIKE ?4 OR c.addr LIKE ?5) \
-                 AND (1=?6 OR LENGTH(ps.verified_key_fingerprint)!=0)  \
+                 AND (iif(c.name='',c.authname,c.name) LIKE ? OR c.addr LIKE ?) \
+                 AND (1=? OR LENGTH(ps.verified_key_fingerprint)!=0)  \
                  ORDER BY LOWER(iif(c.name='',c.authname,c.name)||c.addr),c.id;",
-                    paramsv![
-                        self_addr,
-                        DC_CONTACT_ID_LAST_SPECIAL,
+                        sql::repeat_vars(self_addrs.len())
+                    ),
+                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
+                        ContactId::LAST_SPECIAL,
                         Origin::IncomingReplyTo,
                         s3str_like_cmd,
                         s3str_like_cmd,
                         if flag_verified_only { 0i32 } else { 1i32 },
-                    ],
+                    ])),
                     |row| row.get::<_, ContactId>(0),
                     |ids| {
                         for id in ids {
@@ -722,13 +728,17 @@ impl Contact {
                 )
                 .await?;
 
-            let self_name = context
-                .get_config(Config::Displayname)
-                .await?
-                .unwrap_or_default();
-            let self_name2 = stock_str::self_msg(context);
-
             if let Some(query) = query {
+                let self_addr = context
+                    .get_config(Config::ConfiguredAddr)
+                    .await?
+                    .unwrap_or_default();
+                let self_name = context
+                    .get_config(Config::Displayname)
+                    .await?
+                    .unwrap_or_default();
+                let self_name2 = stock_str::self_msg(context);
+
                 if self_addr.contains(query.as_ref())
                     || self_name.contains(query.as_ref())
                     || self_name2.await.contains(query.as_ref())
@@ -744,17 +754,19 @@ impl Contact {
             context
                 .sql
                 .query_map(
-                    "SELECT id FROM contacts
-                 WHERE addr!=?1
-                 AND id>?2
-                 AND origin>=?3
+                    format!(
+                        "SELECT id FROM contacts
+                 WHERE addr NOT IN ({})
+                 AND id>?
+                 AND origin>=?
                  AND blocked=0
                  ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                    paramsv![
-                        self_addr,
-                        DC_CONTACT_ID_LAST_SPECIAL,
+                        sql::repeat_vars(self_addrs.len())
+                    ),
+                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
+                        ContactId::LAST_SPECIAL,
                         Origin::IncomingReplyTo
-                    ],
+                    ])),
                     |row| row.get::<_, ContactId>(0),
                     |ids| {
                         for id in ids {
@@ -767,7 +779,7 @@ impl Contact {
         }
 
         if flag_add_self && add_self {
-            ret.push(DC_CONTACT_ID_SELF);
+            ret.push(ContactId::SELF);
         }
 
         Ok(ret)
@@ -822,7 +834,7 @@ impl Contact {
             .sql
             .count(
                 "SELECT COUNT(*) FROM contacts WHERE id>? AND blocked!=0",
-                paramsv![DC_CONTACT_ID_LAST_SPECIAL],
+                paramsv![ContactId::LAST_SPECIAL],
             )
             .await?;
         Ok(count as usize)
@@ -838,7 +850,7 @@ impl Contact {
             .sql
             .query_map(
                 "SELECT id FROM contacts WHERE id>? AND blocked!=0 ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                paramsv![DC_CONTACT_ID_LAST_SPECIAL],
+                paramsv![ContactId::LAST_SPECIAL],
                 |row| row.get::<_, ContactId>(0),
                 |ids| {
                     ids.collect::<std::result::Result<Vec<_>, _>>()
@@ -856,13 +868,13 @@ impl Contact {
     /// fingerprints of the keys involved.
     pub async fn get_encrinfo(context: &Context, contact_id: ContactId) -> Result<String> {
         ensure!(
-            contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+            !contact_id.is_special(),
             "Can not provide encryption info for special contact"
         );
 
         let mut ret = String::new();
         if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            let loginparam = LoginParam::from_database(context, "configured_").await?;
+            let loginparam = LoginParam::load_configured_params(context).await?;
             let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
 
             if let Some(peerstate) = peerstate.filter(|peerstate| {
@@ -924,10 +936,7 @@ impl Contact {
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
     pub async fn delete(context: &Context, contact_id: ContactId) -> Result<()> {
-        ensure!(
-            contact_id > DC_CONTACT_ID_LAST_SPECIAL,
-            "Can not delete special contact"
-        );
+        ensure!(!contact_id.is_special(), "Can not delete special contact");
 
         let count_chats = context
             .sql
@@ -963,7 +972,7 @@ impl Contact {
 
     /// Get a single contact object.  For a list, see eg. dc_get_contacts().
     ///
-    /// For contact DC_CONTACT_ID_SELF (1), the function returns sth.
+    /// For contact ContactId::SELF (1), the function returns sth.
     /// like "Me" in the selected language and the email address
     /// defined by dc_set_config().
     pub async fn get_by_id(context: &Context, contact_id: ContactId) -> Result<Contact> {
@@ -1056,7 +1065,7 @@ impl Contact {
     /// This is the image set by each remote user on their own
     /// using dc_set_config(context, "selfavatar", image).
     pub async fn get_profile_image(&self, context: &Context) -> Result<Option<PathBuf>> {
-        if self.id == DC_CONTACT_ID_SELF {
+        if self.id == ContactId::SELF {
             if let Some(p) = context.get_config(Config::Selfavatar).await? {
                 return Ok(Some(PathBuf::from(p)));
             }
@@ -1102,7 +1111,7 @@ impl Contact {
     ) -> Result<VerifiedStatus> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
-        if self.id == DC_CONTACT_ID_SELF {
+        if self.id == ContactId::SELF {
             return Ok(VerifiedStatus::BidirectVerified);
         }
 
@@ -1121,26 +1130,6 @@ impl Contact {
         Ok(VerifiedStatus::Unverified)
     }
 
-    pub async fn addr_equals_contact(
-        context: &Context,
-        addr: &str,
-        contact_id: ContactId,
-    ) -> Result<bool> {
-        if addr.is_empty() {
-            return Ok(false);
-        }
-
-        let contact = Contact::load_from_db(context, contact_id).await?;
-        if !contact.addr.is_empty() {
-            let normalized_addr = addr_normalize(addr);
-            if contact.addr == normalized_addr {
-                return Ok(true);
-            }
-        }
-
-        Ok(false)
-    }
-
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
         if !context.sql.is_open().await {
             return Ok(0);
@@ -1150,14 +1139,14 @@ impl Contact {
             .sql
             .count(
                 "SELECT COUNT(*) FROM contacts WHERE id>?;",
-                paramsv![DC_CONTACT_ID_LAST_SPECIAL],
+                paramsv![ContactId::LAST_SPECIAL],
             )
             .await?;
         Ok(count)
     }
 
     pub async fn real_exists_by_id(context: &Context, contact_id: ContactId) -> Result<bool> {
-        if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+        if contact_id.is_special() {
             return Ok(false);
         }
 
@@ -1230,7 +1219,7 @@ async fn set_block_contact(
     new_blocking: bool,
 ) -> Result<()> {
     ensure!(
-        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+        !contact_id.is_special(),
         "Can't block special contact {}",
         contact_id
     );
@@ -1300,7 +1289,7 @@ pub(crate) async fn set_profile_image(
     let mut contact = Contact::load_from_db(context, contact_id).await?;
     let changed = match profile_image {
         AvatarAction::Change(profile_image) => {
-            if contact_id == DC_CONTACT_ID_SELF {
+            if contact_id == ContactId::SELF {
                 if was_encrypted {
                     context
                         .set_config(Config::Selfavatar, Some(profile_image))
@@ -1314,7 +1303,7 @@ pub(crate) async fn set_profile_image(
             true
         }
         AvatarAction::Delete => {
-            if contact_id == DC_CONTACT_ID_SELF {
+            if contact_id == ContactId::SELF {
                 if was_encrypted {
                     context.set_config(Config::Selfavatar, None).await?;
                 } else {
@@ -1345,7 +1334,7 @@ pub(crate) async fn set_status(
     encrypted: bool,
     has_chat_version: bool,
 ) -> Result<()> {
-    if contact_id == DC_CONTACT_ID_SELF {
+    if contact_id == ContactId::SELF {
         if encrypted && has_chat_version {
             context
                 .set_config(Config::Selfstatus, Some(&status))
@@ -1370,7 +1359,7 @@ pub(crate) async fn update_last_seen(
     timestamp: i64,
 ) -> Result<()> {
     ensure!(
-        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+        !contact_id.is_special(),
         "Can not update special contact last seen timestamp"
     );
 
@@ -1431,17 +1420,6 @@ fn cat_fingerprint(
     }
 }
 
-impl Context {
-    /// determine whether the specified addr maps to the/a self addr
-    pub async fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        if let Some(self_addr) = self.get_config(Config::ConfiguredAddr).await? {
-            Ok(addr_cmp(&self_addr, addr))
-        } else {
-            Ok(false)
-        }
-    }
-}
-
 pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
     let norm1 = addr_normalize(addr1).to_lowercase();
     let norm2 = addr_normalize(addr2).to_lowercase();
@@ -1474,6 +1452,17 @@ mod tests {
     use crate::message::Message;
     use crate::test_utils::{self, TestContext};
 
+    #[test]
+    fn test_contact_id_values() {
+        // Some FFI users need to have the values of these fixed, how naughty.  But let's
+        // make sure we don't modify them anyway.
+        assert_eq!(ContactId::UNDEFINED.to_u32(), 0);
+        assert_eq!(ContactId::SELF.to_u32(), 1);
+        assert_eq!(ContactId::INFO.to_u32(), 2);
+        assert_eq!(ContactId::DEVICE.to_u32(), 5);
+        assert_eq!(ContactId::LAST_SPECIAL.to_u32(), 9);
+    }
+
     #[test]
     fn test_may_be_valid_addr() {
         assert_eq!(may_be_valid_addr(""), false);
@@ -1526,6 +1515,8 @@ mod tests {
     async fn test_get_contacts() -> Result<()> {
         let context = TestContext::new().await;
 
+        assert!(context.get_all_self_addrs().await?.is_empty());
+
         // Bob is not in the contacts yet.
         let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).await?;
         assert_eq!(contacts.len(), 0);
@@ -1537,7 +1528,7 @@ mod tests {
             Origin::IncomingReplyTo,
         )
         .await?;
-        assert_ne!(id, ContactId::new(0));
+        assert_ne!(id, ContactId::UNDEFINED);
 
         let contact = Contact::load_from_db(&context.ctx, id).await.unwrap();
         assert_eq!(contact.get_name(), "");
@@ -1615,7 +1606,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bla foo", "one@eins.org", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_id(), contact_id);
@@ -1642,7 +1633,7 @@ mod tests {
             Contact::add_or_lookup(&t, "", "three@drei.sam", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_name(), "");
@@ -1682,7 +1673,7 @@ mod tests {
             Contact::add_or_lookup(&t, "", "alice@w.de", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_name(), "Wonderland, Alice");
@@ -1691,8 +1682,7 @@ mod tests {
         assert_eq!(contact.get_name_n_addr(), "Wonderland, Alice (alice@w.de)");
 
         // check SELF
-        let contact = Contact::load_from_db(&t, DC_CONTACT_ID_SELF).await.unwrap();
-        assert_eq!(DC_CONTACT_ID_SELF, ContactId::new(1));
+        let contact = Contact::load_from_db(&t, ContactId::SELF).await.unwrap();
         assert_eq!(contact.get_name(), stock_str::self_msg(&t).await);
         assert_eq!(contact.get_addr(), ""); // we're not configured
         assert!(!contact.is_blocked());
@@ -1702,7 +1692,7 @@ mod tests {
     async fn test_delete() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
-        assert!(Contact::delete(&alice, DC_CONTACT_ID_SELF).await.is_err());
+        assert!(Contact::delete(&alice, ContactId::SELF).await.is_err());
 
         // Create Bob contact
         let (contact_id, _) =
@@ -1735,7 +1725,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob1", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::Created);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob1");
@@ -1747,7 +1737,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob2", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob2");
@@ -1758,7 +1748,7 @@ mod tests {
         let contact_id = Contact::create(&t, "bob3", "bob@example.org")
             .await
             .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob2");
         assert_eq!(contact.get_name(), "bob3");
@@ -1769,7 +1759,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob4", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob4");
@@ -1783,7 +1773,7 @@ mod tests {
 
         // manually create "claire@example.org" without a given name
         let contact_id = Contact::create(&t, "", "claire@example.org").await.unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert!(!contact_id.is_special());
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "");
         assert_eq!(contact.get_name(), "");
@@ -1957,7 +1947,7 @@ mod tests {
         let id = Contact::lookup_id_by_addr(&alice.ctx, "alice@example.org", Origin::Unknown)
             .await
             .unwrap();
-        assert_eq!(id, Some(DC_CONTACT_ID_SELF));
+        assert_eq!(id, Some(ContactId::SELF));
     }
 
     #[async_std::test]
@@ -1965,9 +1955,9 @@ mod tests {
         let alice = TestContext::new_alice().await;
 
         // Return error for special IDs
-        let encrinfo = Contact::get_encrinfo(&alice, DC_CONTACT_ID_SELF).await;
+        let encrinfo = Contact::get_encrinfo(&alice, ContactId::SELF).await;
         assert!(encrinfo.is_err());
-        let encrinfo = Contact::get_encrinfo(&alice, DC_CONTACT_ID_DEVICE).await;
+        let encrinfo = Contact::get_encrinfo(&alice, ContactId::DEVICE).await;
         assert!(encrinfo.is_err());
 
         let (contact_bob_id, _modified) =
@@ -2148,7 +2138,7 @@ Chat-Version: 1.0
 Date: Sun, 22 Mar 2020 22:37:55 +0000
 
 Hi."#;
-        dc_receive_imf(&alice, mime, "Inbox", false).await?;
+        dc_receive_imf(&alice, mime, false).await?;
         let msg = alice.get_last_msg().await;
 
         let timestamp = msg.get_timestamp();

src/context.rs

@@ -10,7 +10,6 @@ use async_std::{
     channel::{self, Receiver, Sender},
     path::{Path, PathBuf},
     sync::{Arc, Mutex, RwLock},
-    task,
 };
 
 use crate::chat::{get_chat_cnt, ChatId};
@@ -56,7 +55,6 @@ pub struct InnerContext {
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
-    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
     /// Recently loaded quota information, if any.
     /// Set to `None` if quota was never tried to load.
@@ -176,7 +174,6 @@ impl Context {
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
-            ephemeral_task: RwLock::new(None),
             quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
@@ -192,17 +189,14 @@ impl Context {
 
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
-        info!(self, "starting IO");
-        if self.inner.is_io_running().await {
-            info!(self, "IO is already running");
+        if let Ok(false) = self.is_configured().await {
+            warn!(self, "can not start io on a context that is not configured");
             return;
         }
 
-        {
-            let l = &mut *self.inner.scheduler.write().await;
-            if let Err(err) = l.start(self.clone()).await {
-                error!(self, "Failed to start IO: {}", err)
-            }
+        info!(self, "starting IO");
+        if let Err(err) = self.inner.scheduler.write().await.start(self.clone()).await {
+            error!(self, "Failed to start IO: {}", err)
         }
     }
 
@@ -210,7 +204,9 @@ impl Context {
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
 
-        self.inner.stop_io().await;
+        if let Err(err) = self.inner.stop_io().await {
+            warn!(self, "failed to stop IO: {}", err);
+        }
     }
 
     /// Returns a reference to the underlying SQL instance.
@@ -239,6 +235,24 @@ impl Context {
         });
     }
 
+    /// Emits a generic MsgsChanged event (without chat or message id)
+    pub fn emit_msgs_changed_without_ids(&self) {
+        self.emit_event(EventType::MsgsChanged {
+            chat_id: ChatId::new(0),
+            msg_id: MsgId::new(0),
+        });
+    }
+
+    /// Emits a MsgsChanged event with specified chat and message ids
+    pub fn emit_msgs_changed(&self, chat_id: ChatId, msg_id: MsgId) {
+        self.emit_event(EventType::MsgsChanged { chat_id, msg_id });
+    }
+
+    /// Emits an IncomingMsg event with specified chat and message ids
+    pub fn emit_incoming_msg(&self, chat_id: ChatId, msg_id: MsgId) {
+        self.emit_event(EventType::IncomingMsg { chat_id, msg_id });
+    }
+
     /// Returns a receiver for emitted events.
     ///
     /// Multiple emitters can be created, but note that in this case each emitted event will
@@ -314,8 +328,9 @@ impl Context {
 
     pub async fn get_info(&self) -> Result<BTreeMap<&'static str, String>> {
         let unset = "0";
-        let l = LoginParam::from_database(self, "").await?;
-        let l2 = LoginParam::from_database(self, "configured_").await?;
+        let l = LoginParam::load_candidate_params(self).await?;
+        let l2 = LoginParam::load_configured_params(self).await?;
+        let secondary_addrs = self.get_secondary_self_addrs().await?.join(", ");
         let displayname = self.get_config(Config::Displayname).await?;
         let chats = get_chat_cnt(self).await? as usize;
         let unblocked_msgs = message::get_unblocked_msg_cnt(self).await as usize;
@@ -400,6 +415,7 @@ impl Context {
         res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
+        res.insert("secondary_addrs", secondary_addrs);
         res.insert(
             "fetch_existing_msgs",
             self.get_config_int(Config::FetchExistingMsgs)
@@ -606,11 +622,6 @@ impl Context {
         Ok(mvbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_spam_folder(&self, folder_name: &str) -> Result<bool> {
-        let spam = self.get_config(Config::ConfiguredSpamFolder).await?;
-        Ok(spam.as_deref() == Some(folder_name))
-    }
-
     pub(crate) fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -627,25 +638,9 @@ impl Context {
 }
 
 impl InnerContext {
-    async fn is_io_running(&self) -> bool {
-        self.scheduler.read().await.is_running()
-    }
-
-    async fn stop_io(&self) {
-        if self.is_io_running().await {
-            let token = {
-                let lock = &*self.scheduler.read().await;
-                lock.pre_stop().await
-            };
-            {
-                let lock = &mut *self.scheduler.write().await;
-                lock.stop(token).await;
-            }
-        }
-
-        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
-            ephemeral_task.cancel().await;
-        }
+    async fn stop_io(&self) -> Result<()> {
+        self.scheduler.write().await.stop().await?;
+        Ok(())
     }
 }
 
@@ -670,7 +665,7 @@ mod tests {
     use crate::chat::{
         get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
     };
-    use crate::constants::DC_CONTACT_ID_SELF;
+    use crate::contact::ContactId;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::dc_tools::dc_create_outgoing_rfc724_mid;
     use crate::message::{Message, Viewtype};
@@ -716,9 +711,7 @@ mod tests {
             dc_create_outgoing_rfc724_mid(None, contact.get_addr())
         );
         println!("{}", msg);
-        dc_receive_imf(t, msg.as_bytes(), "INBOX", false)
-            .await
-            .unwrap();
+        dc_receive_imf(t, msg.as_bytes(), false).await.unwrap();
     }
 
     #[async_std::test]
@@ -956,7 +949,7 @@ mod tests {
     #[async_std::test]
     async fn test_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
-        let self_talk = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
+        let self_talk = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
         let chat = alice
             .create_chat_with_contact("Bob", "bob@example.org")
             .await;

src/dc_tools.rs

@@ -86,7 +86,7 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
 //   `last_smeared_timestamp` is again in sync with the normal time.
 // - however, we do not do all this for the far future,
 //   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
-const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
+pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 
 /// Returns the current smeared timestamp,
 ///
@@ -214,7 +214,10 @@ pub(crate) fn dc_create_id() -> String {
     rng.fill(&mut arr[..]);
 
     // Take 11 base64 characters containing 66 random bits.
-    base64::encode(&arr).chars().take(11).collect()
+    base64::encode_config(&arr, base64::URL_SAFE)
+        .chars()
+        .take(11)
+        .collect()
 }
 
 /// Function generates a Message-ID that can be used for a new outgoing message.
@@ -700,7 +703,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
     async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
-        dc_receive_imf(&t, raw, "INBOX", false).await.unwrap();
+        dc_receive_imf(&t, raw, false).await.unwrap();
         let msg = t.get_last_msg().await;
         let msg_info = get_msg_info(&t, msg.id).await.unwrap();
 
@@ -762,6 +765,15 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         assert_eq!(buf.len(), 11);
     }
 
+    #[test]
+    fn test_dc_create_id_invalid_chars() {
+        for _ in 1..1000 {
+            let buf = dc_create_id();
+            assert!(!buf.contains('/')); // `/` must not be used to be URL-safe
+            assert!(!buf.contains('.')); // `.` is used as a delimiter when extracting grpid from Message-ID
+        }
+    }
+
     #[test]
     fn test_dc_extract_grpid_from_rfc724_mid() {
         // Should return None if we pass invalid mid

src/download.rs

@@ -346,7 +346,6 @@ mod tests {
             &t,
             "Mr.12345678901@example.com",
             header.as_bytes(),
-            "INBOX",
             false,
             Some(100000),
             false,
@@ -364,7 +363,6 @@ mod tests {
             &t,
             "Mr.12345678901@example.com",
             format!("{}\n\n100k text...", header).as_bytes(),
-            "INBOX",
             false,
             None,
             false,
@@ -400,7 +398,6 @@ mod tests {
                     Message-ID: <first@example.org>\n\
                     Date: Sun, 14 Nov 2021 00:10:00 +0000\
                     Content-Type: text/plain",
-            "INBOX",
             false,
             Some(100000),
             false,

src/e2ee.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashSet;
 
-use anyhow::{bail, format_err, Context as _, Result};
+use anyhow::{format_err, Context as _, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
@@ -28,13 +28,7 @@ impl EncryptHelper {
         let prefer_encrypt =
             EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await?)
                 .unwrap_or_default();
-        let addr = match context.get_config(Config::ConfiguredAddr).await? {
-            None => {
-                bail!("addr not configured!");
-            }
-            Some(addr) => addr,
-        };
-
+        let addr = context.get_primary_self_addr().await?;
         let public_key = SignedPublicKey::load_self(context).await?;
 
         Ok(EncryptHelper {
@@ -387,13 +381,7 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 /// [Config::ConfiguredAddr] is configured, this address is returned.
 // TODO, remove this once deltachat::key::Key no longer exists.
 pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await?
-        .context(concat!(
-            "Failed to get self address, ",
-            "cannot ensure secret key if not configured."
-        ))?;
+    let self_addr = context.get_primary_self_addr().await?;
     SignedPublicKey::load_self(context).await?;
     Ok(self_addr)
 }

src/ephemeral.rs

@@ -48,9 +48,9 @@
 //!
 //! ## When messages are deleted
 //!
-//! Local deletion happens when the chatlist or chat is loaded. A
-//! `MsgsChanged` event is emitted when a message deletion is due, to
-//! make UI reload displayed messages and cause actual deletion.
+//! The `ephemeral_loop` task schedules the next due running of
+//! `delete_expired_messages` which in turn emits `MsgsChanged` events
+//! when deleting local messages to make UIs reload displayed messages.
 //!
 //! Server deletion happens by updating the `imap` table based on
 //! the database entries which are expired either according to their
@@ -62,18 +62,18 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{ensure, Context as _, Result};
-use async_std::task;
+use async_std::channel::Receiver;
+use async_std::future::timeout;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{send_msg, ChatId};
-use crate::constants::{
-    DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
-};
+use crate::constants::{DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH};
 use crate::contact::ContactId;
 use crate::context::Context;
-use crate::dc_tools::time;
+use crate::dc_tools::{duration_to_str, time};
 use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
+use crate::log::LogExt;
 use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::sql;
@@ -198,7 +198,7 @@ impl ChatId {
         }
         self.inner_set_ephemeral_timer(context, timer).await?;
         let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, ContactId::SELF).await);
         msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
         if let Err(err) = send_msg(context, self, &mut msg).await {
             error!(
@@ -293,7 +293,7 @@ impl MsgId {
                     paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
                 )
                 .await?;
-            schedule_ephemeral_task(context).await;
+            context.interrupt_ephemeral_task().await;
         }
         Ok(())
     }
@@ -315,7 +315,7 @@ pub(crate) async fn start_ephemeral_timers_msgids(
                 "UPDATE msgs SET ephemeral_timestamp = ? + ephemeral_timer
          WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ? + ephemeral_timer) AND ephemeral_timer > 0
          AND id IN ({})",
-                sql::repeat_vars(msg_ids.len())?
+                sql::repeat_vars(msg_ids.len())
             ),
             rusqlite::params_from_iter(
                 std::iter::once(&now as &dyn crate::ToSql)
@@ -325,7 +325,7 @@ pub(crate) async fn start_ephemeral_timers_msgids(
         )
         .await?;
     if count > 0 {
-        schedule_ephemeral_task(context).await;
+        context.interrupt_ephemeral_task().await;
     }
     Ok(())
 }
@@ -338,7 +338,7 @@ pub(crate) async fn start_ephemeral_timers_msgids(
 /// false. This function does not emit the MsgsChanged event itself,
 /// because it is also called when chatlist is reloaded, and emitting
 /// MsgsChanged there will cause infinite reload loop.
-pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool> {
+pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Result<()> {
     let mut updated = context
         .sql
         .execute(
@@ -354,21 +354,21 @@ WHERE
   AND ephemeral_timestamp <= ?
   AND chat_id != ?
 "#,
-            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
+            paramsv![DC_CHAT_ID_TRASH, now, DC_CHAT_ID_TRASH],
         )
         .await
         .context("update failed")?
         > 0;
 
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
             .await?
             .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
             .await?
             .unwrap_or_default();
 
-        let threshold_timestamp = time() - delete_device_after;
+        let threshold_timestamp = now.saturating_sub(delete_device_after);
 
         // Delete expired messages
         //
@@ -378,7 +378,8 @@ WHERE
             .sql
             .execute(
                 "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
+             SET chat_id = ?, txt = '', subject='', txt_raw='', \
+                 mime_headers='', from_id=0, to_id=0, param='' \
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
@@ -397,72 +398,116 @@ WHERE
         updated |= rows_modified > 0;
     }
 
-    schedule_ephemeral_task(context).await;
-    Ok(updated)
+    if updated {
+        context.emit_msgs_changed_without_ids();
+    }
+
+    Ok(())
 }
 
-/// Schedule a task to emit MsgsChanged event when the next local
-/// deletion happens. Existing task is cancelled to make sure at most
-/// one such task is scheduled at a time.
+/// Calculates the next timestamp when a message will be deleted due to
+/// `delete_device_after` setting being set.
+async fn next_delete_device_after_timestamp(context: &Context) -> Result<Option<i64>> {
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
+        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+            .await?
+            .unwrap_or_default();
+        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
+            .await?
+            .unwrap_or_default();
+
+        let oldest_message_timestamp: Option<i64> = context
+            .sql
+            .query_get_value(
+                r#"
+                SELECT min(timestamp)
+                FROM msgs
+                WHERE chat_id > ?
+                  AND chat_id != ?
+                  AND chat_id != ?;
+                "#,
+                paramsv![DC_CHAT_ID_TRASH, self_chat_id, device_chat_id],
+            )
+            .await?;
+
+        Ok(oldest_message_timestamp.map(|x| x.saturating_add(delete_device_after)))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Calculates next timestamp when expiration of some message will happen.
 ///
-/// UI is expected to reload the chatlist or the chat in response to
-/// MsgsChanged event, this will trigger actual deletion.
-///
-/// This takes into account only per-chat timeouts, because global device
-/// timeouts are at least one hour long and deletion is triggered often enough
-/// by user actions.
-pub async fn schedule_ephemeral_task(context: &Context) {
+/// Expiration can happen either because user has set `delete_device_after` setting or because the
+/// message itself has an ephemeral timer.
+async fn next_expiration_timestamp(context: &Context) -> Option<i64> {
     let ephemeral_timestamp: Option<i64> = match context
         .sql
         .query_get_value(
             r#"
-    SELECT ephemeral_timestamp
-    FROM msgs
-    WHERE ephemeral_timestamp != 0
-      AND chat_id != ?
-    ORDER BY ephemeral_timestamp ASC
-    LIMIT 1;
-    "#,
+            SELECT min(ephemeral_timestamp)
+            FROM msgs
+            WHERE ephemeral_timestamp != 0
+              AND chat_id != ?;
+            "#,
             paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
         )
         .await
     {
         Err(err) => {
             warn!(context, "Can't calculate next ephemeral timeout: {}", err);
-            return;
+            None
         }
         Ok(ephemeral_timestamp) => ephemeral_timestamp,
     };
 
-    // Cancel existing task, if any
-    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
-        ephemeral_task.cancel().await;
-    }
+    let delete_device_after_timestamp: Option<i64> =
+        match next_delete_device_after_timestamp(context).await {
+            Err(err) => {
+                warn!(
+                    context,
+                    "Can't calculate timestamp of the next message expiration: {}", err
+                );
+                None
+            }
+            Ok(timestamp) => timestamp,
+        };
+
+    ephemeral_timestamp
+        .into_iter()
+        .chain(delete_device_after_timestamp.into_iter())
+        .min()
+}
+
+pub(crate) async fn ephemeral_loop(context: &Context, interrupt_receiver: Receiver<()>) {
+    loop {
+        let ephemeral_timestamp = next_expiration_timestamp(context).await;
 
-    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
         let now = SystemTime::now();
-        let until = UNIX_EPOCH
-            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
-            + Duration::from_secs(1);
+        let until = if let Some(ephemeral_timestamp) = ephemeral_timestamp {
+            UNIX_EPOCH
+                + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+                + Duration::from_secs(1)
+        } else {
+            // no messages to be deleted for now, wait long for one to occur
+            now + Duration::from_secs(86400)
+        };
 
         if let Ok(duration) = until.duration_since(now) {
-            // Schedule a task, ephemeral_timestamp is in the future
-            let context1 = context.clone();
-            let ephemeral_task = task::spawn(async move {
-                async_std::task::sleep(duration).await;
-                context1.emit_event(EventType::MsgsChanged {
-                    chat_id: ChatId::new(0),
-                    msg_id: MsgId::new(0),
-                });
-            });
-            *context.ephemeral_task.write().await = Some(ephemeral_task);
-        } else {
-            // Emit event immediately
-            context.emit_event(EventType::MsgsChanged {
-                chat_id: ChatId::new(0),
-                msg_id: MsgId::new(0),
-            });
+            info!(
+                context,
+                "Ephemeral loop waiting for deletion in {} or interrupt",
+                duration_to_str(duration)
+            );
+            if timeout(duration, interrupt_receiver.recv()).await.is_ok() {
+                // received an interruption signal, recompute waiting time (if any)
+                continue;
+            }
         }
+
+        delete_expired_messages(context, time())
+            .await
+            .ok_or_log(context);
     }
 }
 
@@ -532,6 +577,7 @@ mod tests {
     use super::*;
     use crate::config::Config;
     use crate::dc_receive_imf::dc_receive_imf;
+    use crate::dc_tools::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
@@ -544,7 +590,7 @@ mod tests {
         let context = TestContext::new().await;
 
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, ContactId::SELF).await,
             "Message deletion timer is disabled by me."
         );
 
@@ -552,7 +598,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 1 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 s by me."
@@ -561,7 +607,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 30 s by me."
@@ -570,7 +616,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 minute by me."
@@ -579,7 +625,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 90 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1.5 minutes by me."
@@ -588,7 +634,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 * 60 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 30 minutes by me."
@@ -597,7 +643,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 * 60 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 hour by me."
@@ -606,7 +652,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 5400 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1.5 hours by me."
@@ -617,7 +663,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 2 hours by me."
@@ -628,7 +674,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 day by me."
@@ -639,7 +685,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 2 days by me."
@@ -650,7 +696,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 week by me."
@@ -661,7 +707,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 4 weeks by me."
@@ -820,31 +866,106 @@ mod tests {
     #[async_std::test]
     async fn test_ephemeral_delete_msgs() -> Result<()> {
         let t = TestContext::new_alice().await;
-        let chat = t.get_self_chat().await;
+        let self_chat = t.get_self_chat().await;
 
-        t.send_text(chat.id, "Saved message, which we delete manually")
+        assert_eq!(next_expiration_timestamp(&t).await, None);
+
+        t.send_text(self_chat.id, "Saved message, which we delete manually")
             .await;
-        let msg = t.get_last_msg_in(chat.id).await;
+        let msg = t.get_last_msg_in(self_chat.id).await;
         msg.id.delete_from_db(&t).await?;
-        check_msg_was_deleted(&t, &chat, msg.id).await;
+        check_msg_is_deleted(&t, &self_chat, msg.id).await;
 
-        chat.id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
+        self_chat
+            .id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 3600 })
             .await
             .unwrap();
-        let msg = t
-            .send_text(chat.id, "Saved message, disappearing after 1s")
-            .await;
 
-        async_std::task::sleep(Duration::from_millis(1100)).await;
+        // Send a saved message which will be deleted after 3600s
+        let now = time();
+        let msg = t.send_text(self_chat.id, "Message text").await;
 
-        // Check that the msg was deleted locally.
-        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
+        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3599, time() + 3601)
+            .await
+            .unwrap();
+
+        // Set DeleteDeviceAfter to 1800s. Thend send a saved message which will
+        // still be deleted after 3600s because DeleteDeviceAfter doesn't apply to saved messages.
+        t.set_config(Config::DeleteDeviceAfter, Some("1800"))
+            .await?;
+
+        let now = time();
+        let msg = t.send_text(self_chat.id, "Message text").await;
+
+        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3559, time() + 3601)
+            .await
+            .unwrap();
+
+        // Send a message to Bob which will be deleted after 1800s because of DeleteDeviceAfter.
+        let bob_chat = t.create_chat_with_contact("", "bob@example.net").await;
+        let now = time();
+        let msg = t.send_text(bob_chat.id, "Message text").await;
+
+        check_msg_will_be_deleted(
+            &t,
+            msg.sender_msg_id,
+            &bob_chat,
+            now + 1799,
+            // The message may appear to be sent MAX_SECONDS_TO_LEND_FROM_FUTURE later and
+            // therefore be deleted MAX_SECONDS_TO_LEND_FROM_FUTURE later.
+            time() + 1801 + MAX_SECONDS_TO_LEND_FROM_FUTURE,
+        )
+        .await
+        .unwrap();
+
+        // Enable ephemeral messages with Bob -> message will be deleted after 60s.
+        // This tests that the message is deleted at min(ephemeral deletion time, DeleteDeviceAfter deletion time).
+        bob_chat
+            .id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
+            .await?;
+
+        let now = time();
+        let msg = t.send_text(bob_chat.id, "Message text").await;
+
+        check_msg_will_be_deleted(&t, msg.sender_msg_id, &bob_chat, now + 59, time() + 61)
+            .await
+            .unwrap();
 
         Ok(())
     }
 
-    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+    async fn check_msg_will_be_deleted(
+        t: &TestContext,
+        msg_id: MsgId,
+        chat: &Chat,
+        not_deleted_at: i64,
+        deleted_at: i64,
+    ) -> Result<()> {
+        let next_expiration = next_expiration_timestamp(t).await.unwrap();
+
+        assert!(next_expiration > not_deleted_at);
+        delete_expired_messages(t, not_deleted_at).await?;
+
+        let loaded = Message::load_from_db(t, msg_id).await?;
+        assert_eq!(loaded.text.unwrap(), "Message text");
+        assert_eq!(loaded.chat_id, chat.id);
+
+        assert!(next_expiration < deleted_at);
+        delete_expired_messages(t, deleted_at).await?;
+
+        let loaded = Message::load_from_db(t, msg_id).await?;
+        assert_eq!(loaded.text.unwrap(), "");
+        assert_eq!(loaded.chat_id, DC_CHAT_ID_TRASH);
+
+        // Check that the msg was deleted locally.
+        check_msg_is_deleted(t, chat, msg_id).await;
+
+        Ok(())
+    }
+
+    async fn check_msg_is_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
         let chat_items = chat::get_chat_msgs(t, chat.id, 0, None).await.unwrap();
         // Check that the chat is empty except for possibly info messages:
         for item in &chat_items {
@@ -856,8 +977,8 @@ mod tests {
 
         // Check that if there is a message left, the text and metadata are gone
         if let Ok(msg) = Message::load_from_db(t, msg_id).await {
-            assert_eq!(msg.from_id, ContactId::new(0));
-            assert_eq!(msg.to_id, ContactId::new(0));
+            assert_eq!(msg.from_id, ContactId::UNDEFINED);
+            assert_eq!(msg.to_id, ContactId::UNDEFINED);
             assert!(msg.text.is_none_or_empty(), "{:?}", msg.text);
             let rawtxt: Option<String> = t
                 .sql
@@ -994,7 +1115,6 @@ mod tests {
                     Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
                     \n\
                     hello\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -1015,7 +1135,6 @@ mod tests {
                     Ephemeral-Timer: 60\n\
                     \n\
                     second message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -1052,7 +1171,6 @@ mod tests {
                     In-Reply-To: <first@example.com>\n\
                     \n\
                     > hello\n",
-            "INBOX",
             false,
         )
         .await?;

src/events.rs

@@ -10,6 +10,7 @@ use crate::chat::ChatId;
 use crate::contact::ContactId;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
+use crate::webxdc::StatusUpdateSerial;
 
 #[derive(Debug)]
 pub struct Events {
@@ -335,5 +336,8 @@ pub enum EventType {
     SelfavatarChanged,
 
     #[strum(props(id = "2120"))]
-    WebxdcStatusUpdate(MsgId),
+    WebxdcStatusUpdate {
+        msg_id: MsgId,
+        status_update_serial: StatusUpdateSerial,
+    },
 }

src/html.rs

@@ -279,7 +279,7 @@ mod tests {
     use crate::chat;
     use crate::chat::forward_msgs;
     use crate::config::Config;
-    use crate::constants::DC_CONTACT_ID_SELF;
+    use crate::contact::ContactId;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::message::{MessengerMessage, Viewtype};
     use crate::test_utils::TestContext;
@@ -440,9 +440,9 @@ test some special html-characters as < > and & but also " and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
+        dc_receive_imf(&alice, raw, false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_ne!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
         assert!(!msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -456,7 +456,7 @@ test some special html-characters as < > and & but also " and &#x
             .await
             .unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -469,7 +469,7 @@ test some special html-characters as < > and & but also " and &#x
         let chat = bob.create_chat_with_contact("", "alice@example.org").await;
         bob.recv_msg(&alice.pop_sent_msg().await).await;
         let msg = bob.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_ne!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -489,7 +489,7 @@ test some special html-characters as < > and & but also " and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
+        dc_receive_imf(&alice, raw, false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
 
         // forward the message to saved-messages,
@@ -506,7 +506,7 @@ test some special html-characters as < > and & but also " and &#x
         alice.recv_msg(&msg).await;
         let chat = alice.get_self_chat().await;
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.get_showpadlock());
         assert!(msg.is_forwarded());
@@ -555,7 +555,6 @@ test some special html-characters as < > and & but also " and &#x
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/cp1252-html.eml"),
-            "INBOX",
             false,
         )
         .await?;

src/imap.rs

@@ -7,6 +7,7 @@ use std::{
     cmp,
     cmp::max,
     collections::{BTreeMap, BTreeSet},
+    iter::Peekable,
 };
 
 use anyhow::{bail, format_err, Context as _, Result};
@@ -20,9 +21,10 @@ use num_traits::FromPrimitive;
 use crate::chat::{self, ChatId, ChatIdBlocked};
 use crate::config::Config;
 use crate::constants::{
-    Blocked, Chattype, ShowEmails, DC_CONTACT_ID_SELF, DC_FETCH_EXISTING_MSGS_COUNT,
-    DC_FOLDERS_CONFIGURED_VERSION, DC_LP_AUTH_OAUTH2,
+    Blocked, Chattype, ShowEmails, DC_FETCH_EXISTING_MSGS_COUNT, DC_FOLDERS_CONFIGURED_VERSION,
+    DC_LP_AUTH_OAUTH2,
 };
+use crate::contact::ContactId;
 use crate::context::Context;
 use crate::dc_receive_imf::{
     dc_receive_imf_inner, from_field_to_contact_id, get_prefetch_parent_message, ReceivedMsg,
@@ -30,14 +32,13 @@ use crate::dc_receive_imf::{
 use crate::dc_tools::dc_create_id;
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::job::{self, Action};
+use crate::job;
 use crate::login_param::{
     CertificateChecks, LoginParam, ServerAddress, ServerLoginParam, Socks5Config,
 };
 use crate::message::{self, Message, MessageState, MessengerMessage, MsgId, Viewtype};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
-use crate::param::Params;
 use crate::provider::Socket;
 use crate::scheduler::connectivity::ConnectivityStore;
 use crate::scheduler::InterruptInfo;
@@ -130,7 +131,7 @@ impl FolderMeaning {
     fn to_config(self) -> Option<Config> {
         match self {
             FolderMeaning::Unknown => None,
-            FolderMeaning::Spam => Some(Config::ConfiguredSpamFolder),
+            FolderMeaning::Spam => None,
             FolderMeaning::Sent => Some(Config::ConfiguredSentboxFolder),
             FolderMeaning::Drafts => None,
             FolderMeaning::Other => None,
@@ -164,6 +165,67 @@ struct ImapConfig {
     pub can_condstore: bool,
 }
 
+struct UidGrouper<T: Iterator<Item = (i64, u32, String)>> {
+    inner: Peekable<T>,
+}
+
+impl<T, I> From<I> for UidGrouper<T>
+where
+    T: Iterator<Item = (i64, u32, String)>,
+    I: IntoIterator<IntoIter = T>,
+{
+    fn from(inner: I) -> Self {
+        Self {
+            inner: inner.into_iter().peekable(),
+        }
+    }
+}
+
+impl<T: Iterator<Item = (i64, u32, String)>> Iterator for UidGrouper<T> {
+    // Tuple of folder, row IDs, and UID range as a string.
+    type Item = (String, Vec<i64>, String);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let (_, _, folder) = self.inner.peek().cloned()?;
+
+        let mut uid_set = String::new();
+        let mut rowid_set = Vec::new();
+
+        while uid_set.len() < 1000 {
+            // Construct a new range.
+            if let Some((start_rowid, start_uid, _)) = self
+                .inner
+                .next_if(|(_, _, start_folder)| start_folder == &folder)
+            {
+                rowid_set.push(start_rowid);
+                let mut end_uid = start_uid;
+
+                while let Some((next_rowid, next_uid, _)) =
+                    self.inner.next_if(|(_, next_uid, next_folder)| {
+                        next_folder == &folder && *next_uid == end_uid + 1
+                    })
+                {
+                    end_uid = next_uid;
+                    rowid_set.push(next_rowid);
+                }
+
+                let uid_range = UidRange {
+                    start: start_uid,
+                    end: end_uid,
+                };
+                if !uid_set.is_empty() {
+                    uid_set.push(',');
+                }
+                uid_set.push_str(&uid_range.to_string());
+            } else {
+                break;
+            }
+        }
+
+        Some((folder, rowid_set, uid_set))
+    }
+}
+
 impl Imap {
     /// Creates new disconnected IMAP client using the specific login parameters.
     ///
@@ -223,7 +285,7 @@ impl Imap {
             bail!("IMAP Connect without configured params");
         }
 
-        let param = LoginParam::from_database(context, "configured_").await?;
+        let param = LoginParam::load_configured_params(context).await?;
         // the trailing underscore is correct
 
         let imap = Self::new(
@@ -450,16 +512,29 @@ impl Imap {
     ///
     /// Prefetches headers and downloads new message from the folder, moves messages away from the
     /// folder and deletes messages in the folder.
-    pub async fn fetch_move_delete(&mut self, context: &Context, watch_folder: &str) -> Result<()> {
+    pub async fn fetch_move_delete(
+        &mut self,
+        context: &Context,
+        watch_folder: &str,
+        is_spam_folder: bool,
+    ) -> Result<()> {
         if !context.sql.is_open().await {
             // probably shutdown
             bail!("IMAP operation attempted while it is torn down");
         }
         self.prepare(context).await?;
 
-        self.fetch_new_messages(context, watch_folder, false)
+        let msgs_fetched = self
+            .fetch_new_messages(context, watch_folder, is_spam_folder, false)
             .await
             .context("fetch_new_messages")?;
+        if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
+            // New messages were fetched and shall be deleted later, restart ephemeral loop.
+            // Note that the `Config::DeleteDeviceAfter` timer starts as soon as the messages are
+            // fetched while the per-chat ephemeral timers start as soon as the messages are marked
+            // as noticed.
+            context.interrupt_ephemeral_task().await;
+        }
 
         self.move_delete_messages(context, watch_folder)
             .await
@@ -657,13 +732,17 @@ impl Imap {
         Ok(false)
     }
 
+    /// Fetches new messages.
+    ///
+    /// Returns true if at least one message was fetched.
     pub(crate) async fn fetch_new_messages(
         &mut self,
         context: &Context,
         folder: &str,
+        is_spam_folder: bool,
         fetch_existing_msgs: bool,
     ) -> Result<bool> {
-        if should_ignore_folder(context, folder).await? {
+        if should_ignore_folder(context, folder, is_spam_folder).await? {
             info!(context, "Not fetching from {}", folder);
             return Ok(false);
         }
@@ -706,7 +785,7 @@ impl Imap {
             // Get the Message-ID or generate a fake one to identify the message in the database.
             let message_id = prefetch_get_message_id(&headers).unwrap_or_else(dc_create_id);
 
-            let target = match target_folder(context, folder, &headers).await? {
+            let target = match target_folder(context, folder, is_spam_folder, &headers).await? {
                 Some(config) => match context.get_config(config).await? {
                     Some(target) => target,
                     None => folder.to_string(),
@@ -737,7 +816,7 @@ impl Imap {
                 // If the sender is known, the message will be moved to the Inbox or Mvbox
                 // and then we download the message from there.
                 // Also see `spam_target_folder()`.
-                && !context.is_spam_folder(folder).await?
+                && !is_spam_folder
                 && prefetch_should_download(
                     context,
                     &headers,
@@ -833,7 +912,7 @@ impl Imap {
             .execute(
                 format!(
                     "DELETE FROM imap WHERE id IN ({})",
-                    sql::repeat_vars(row_ids.len())?
+                    sql::repeat_vars(row_ids.len())
                 ),
                 rusqlite::params_from_iter(row_ids),
             )
@@ -870,7 +949,7 @@ impl Imap {
                         .execute(
                             format!(
                                 "DELETE FROM imap WHERE id IN ({})",
-                                sql::repeat_vars(row_ids.len())?
+                                sql::repeat_vars(row_ids.len())
                             ),
                             rusqlite::params_from_iter(row_ids),
                         )
@@ -912,7 +991,7 @@ impl Imap {
                     .execute(
                         format!(
                             "UPDATE imap SET target='' WHERE id IN ({})",
-                            sql::repeat_vars(row_ids.len())?
+                            sql::repeat_vars(row_ids.len())
                         ),
                         rusqlite::params_from_iter(row_ids),
                     )
@@ -932,7 +1011,7 @@ impl Imap {
     ///
     /// This is the only place where messages are moved or deleted on the IMAP server.
     async fn move_delete_messages(&mut self, context: &Context, folder: &str) -> Result<()> {
-        let mut rows = context
+        let rows = context
             .sql
             .query_map(
                 "SELECT id, uid, target FROM imap
@@ -948,48 +1027,12 @@ impl Imap {
                 },
                 |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
             )
-            .await?
-            .into_iter()
-            .peekable();
+            .await?;
 
         self.prepare(context).await?;
         self.select_folder(context, Some(folder)).await?;
 
-        while let Some((_, _, target)) = rows.peek().cloned() {
-            // Construct next request for the target folder.
-            let mut uid_set = String::new();
-            let mut rowid_set = Vec::new();
-
-            while uid_set.len() < 1000 {
-                // Construct a new range.
-                if let Some((start_rowid, start_uid, _)) =
-                    rows.next_if(|(_, _, start_target)| start_target == &target)
-                {
-                    rowid_set.push(start_rowid);
-                    let mut end_uid = start_uid;
-
-                    while let Some((next_rowid, next_uid, _)) =
-                        rows.next_if(|(_, next_uid, next_target)| {
-                            next_target == &target && *next_uid == end_uid + 1
-                        })
-                    {
-                        end_uid = next_uid;
-                        rowid_set.push(next_rowid);
-                    }
-
-                    let uid_range = UidRange {
-                        start: start_uid,
-                        end: end_uid,
-                    };
-                    if !uid_set.is_empty() {
-                        uid_set.push(',');
-                    }
-                    uid_set.push_str(&uid_range.to_string());
-                } else {
-                    break;
-                }
-            }
-
+        for (target, rowid_set, uid_set) in UidGrouper::from(rows) {
             // Empty target folder name means messages should be deleted.
             if target.is_empty() {
                 self.delete_message_batch(context, &uid_set, rowid_set)
@@ -1016,6 +1059,62 @@ impl Imap {
         Ok(())
     }
 
+    /// Stores pending `\Seen` flags for messages in `imap_markseen` table.
+    pub(crate) async fn store_seen_flags_on_imap(&mut self, context: &Context) -> Result<()> {
+        self.prepare(context).await?;
+
+        let rows = context
+            .sql
+            .query_map(
+                "SELECT imap.id, uid, folder FROM imap, imap_markseen
+                 WHERE imap.id = imap_markseen.id AND target = folder
+                 ORDER BY folder, uid",
+                [],
+                |row| {
+                    let rowid: i64 = row.get(0)?;
+                    let uid: u32 = row.get(1)?;
+                    let folder: String = row.get(2)?;
+                    Ok((rowid, uid, folder))
+                },
+                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+            )
+            .await?;
+
+        for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
+            self.select_folder(context, Some(&folder))
+                .await
+                .context("failed to select folder")?;
+
+            if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
+                warn!(
+                    context,
+                    "Cannot mark messages {} in folder {} as seen, will retry later: {}.",
+                    uid_set,
+                    folder,
+                    err
+                );
+            } else {
+                info!(
+                    context,
+                    "Marked messages {} in folder {} as seen.", uid_set, folder
+                );
+                context
+                    .sql
+                    .execute(
+                        format!(
+                            "DELETE FROM imap_markseen WHERE id IN ({})",
+                            sql::repeat_vars(rowid_set.len())
+                        ),
+                        rusqlite::params_from_iter(rowid_set),
+                    )
+                    .await
+                    .context("cannot remove messages marked as seen from imap_markseen table")?;
+            }
+        }
+
+        Ok(())
+    }
+
     /// Synchronizes `\Seen` flags using `CONDSTORE` extension.
     pub(crate) async fn sync_seen_flags(&mut self, context: &Context, folder: &str) -> Result<()> {
         if !self.config.can_condstore {
@@ -1107,14 +1206,9 @@ impl Imap {
             .session
             .as_mut()
             .context("IMAP No Connection established")?;
-        let self_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .context("not configured")?;
 
-        let search_command = format!("FROM \"{}\"", self_addr);
         let uids = session
-            .uid_search(search_command)
+            .uid_search(get_imap_self_sent_search_command(context).await?)
             .await?
             .into_iter()
             .collect();
@@ -1261,8 +1355,6 @@ impl Imap {
                 }
             };
 
-            let folder = folder.to_string();
-
             while let Some(Ok(msg)) = msgs.next().await {
                 let server_uid = msg.uid.unwrap_or_default();
 
@@ -1296,7 +1388,6 @@ impl Imap {
 
                 // XXX put flags into a set and pass them to dc_receive_imf
                 let context = context.clone();
-                let folder = folder.clone();
 
                 // safe, as we checked above that there is a body.
                 let body = body
@@ -1317,7 +1408,6 @@ impl Imap {
                     &context,
                     rfc724_mid,
                     body,
-                    &folder,
                     is_seen,
                     partial,
                     fetching_existing_messages,
@@ -1356,11 +1446,6 @@ impl Imap {
     /// the flag, or other imap-errors, returns true as well.
     ///
     /// Returning error means that the operation can be retried.
-    async fn add_flag_finalized(&mut self, server_uid: u32, flag: &str) -> Result<()> {
-        let s = server_uid.to_string();
-        self.add_flag_finalized_with_set(&s, flag).await
-    }
-
     async fn add_flag_finalized_with_set(&mut self, uid_set: &str, flag: &str) -> Result<()> {
         if self.should_reconnect() {
             bail!("Can't set flag, should reconnect");
@@ -1378,7 +1463,7 @@ impl Imap {
         Ok(())
     }
 
-    pub async fn prepare_imap_operation_on_msg(
+    pub(crate) async fn prepare_imap_operation_on_msg(
         &mut self,
         context: &Context,
         folder: &str,
@@ -1418,32 +1503,6 @@ impl Imap {
         }
     }
 
-    pub(crate) async fn set_seen(
-        &mut self,
-        context: &Context,
-        folder: &str,
-        uid: u32,
-    ) -> ImapActionResult {
-        if let Some(imapresult) = self
-            .prepare_imap_operation_on_msg(context, folder, uid)
-            .await
-        {
-            return imapresult;
-        }
-        // we are connected, and the folder is selected
-        info!(context, "Marking message {}/{} as seen...", folder, uid,);
-
-        if let Err(err) = self.add_flag_finalized(uid, "\\Seen").await {
-            warn!(
-                context,
-                "Cannot mark message {} in folder {} as seen, ignoring: {}.", uid, folder, err
-            );
-            ImapActionResult::Failed
-        } else {
-            ImapActionResult::Success
-        }
-    }
-
     pub async fn ensure_configured_folders(
         &mut self,
         context: &Context,
@@ -1639,7 +1698,7 @@ async fn should_move_out_of_spam(
             if chat_id_blocked.blocked != Blocked::Not {
                 return Ok(false);
             }
-        } else if from_id != DC_CONTACT_ID_SELF {
+        } else if from_id != ContactId::SELF {
             // No chat with this contact found.
             return Ok(false);
         }
@@ -1676,13 +1735,14 @@ async fn spam_target_folder(
 pub async fn target_folder(
     context: &Context,
     folder: &str,
+    is_spam_folder: bool,
     headers: &[mailparse::MailHeader<'_>],
 ) -> Result<Option<Config>> {
     if context.is_mvbox(folder).await? {
         return Ok(None);
     }
 
-    if context.is_spam_folder(folder).await? {
+    if is_spam_folder {
         spam_target_folder(context, headers).await
     } else if needs_move_to_mvbox(context, headers).await? {
         Ok(Some(Config::ConfiguredMvboxFolder))
@@ -1874,13 +1934,11 @@ pub(crate) async fn prefetch_should_download(
     mut flags: impl Iterator<Item = Flag<'_>>,
     show_emails: ShowEmails,
 ) -> Result<bool> {
-    if let Some(msg_id) = message::rfc724_mid_exists(context, message_id).await? {
-        // We know the Message-ID already, it must be a Bcc: to self.
-        job::add(
-            context,
-            job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
-        )
-        .await?;
+    if message::rfc724_mid_exists(context, message_id)
+        .await?
+        .is_some()
+    {
+        markseen_on_imap_table(context, message_id).await?;
         return Ok(false);
     }
 
@@ -2014,6 +2072,22 @@ async fn mark_seen_by_uid(
     }
 }
 
+/// Schedule marking the message as Seen on IMAP by adding all known IMAP messages corresponding to
+/// the given Message-ID to `imap_markseen` table.
+pub(crate) async fn markseen_on_imap_table(context: &Context, message_id: &str) -> Result<()> {
+    context
+        .sql
+        .execute(
+            "INSERT OR IGNORE INTO imap_markseen (id)
+             SELECT id FROM imap WHERE rfc724_mid=?",
+            paramsv![message_id],
+        )
+        .await?;
+    context.interrupt_inbox(InterruptInfo::new(false)).await;
+
+    Ok(())
+}
+
 /// uid_next is the next unique identifier value from the last time we fetched a folder
 /// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
 /// This function is used to update our uid_next after fetching messages.
@@ -2095,6 +2169,18 @@ async fn get_modseq(context: &Context, folder: &str) -> Result<u64> {
         .unwrap_or(0))
 }
 
+/// Compute the imap search expression for all self-sent mails (for all self addresses)
+pub(crate) async fn get_imap_self_sent_search_command(context: &Context) -> Result<String> {
+    // See https://www.rfc-editor.org/rfc/rfc3501#section-6.4.4 for syntax of SEARCH and OR
+    let mut search_command = format!("FROM \"{}\"", context.get_primary_self_addr().await?);
+
+    for item in context.get_secondary_self_addrs().await? {
+        search_command = format!("OR ({}) (FROM \"{}\")", search_command, item);
+    }
+
+    Ok(search_command)
+}
+
 /// Deprecated, use get_uid_next() and get_uidvalidity()
 pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result<(u32, u32)> {
     let key = format!("imap.mailbox.{}", folder);
@@ -2114,7 +2200,11 @@ pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result
 ///
 /// This caters for the [`Config::OnlyFetchMvbox`] setting which means mails from folders
 /// not explicitly watched should not be fetched.
-async fn should_ignore_folder(context: &Context, folder: &str) -> Result<bool> {
+async fn should_ignore_folder(
+    context: &Context,
+    folder: &str,
+    is_spam_folder: bool,
+) -> Result<bool> {
     if !context.get_config_bool(Config::OnlyFetchMvbox).await? {
         return Ok(false);
     }
@@ -2122,7 +2212,7 @@ async fn should_ignore_folder(context: &Context, folder: &str) -> Result<bool> {
         // Still respect the SentboxWatch setting.
         return Ok(!context.get_config_bool(Config::SentboxWatch).await?);
     }
-    Ok(!(context.is_mvbox(folder).await? || context.is_spam_folder(folder).await?))
+    Ok(!(context.is_mvbox(folder).await? || is_spam_folder))
 }
 
 /// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000
@@ -2298,9 +2388,6 @@ mod tests {
                                folder, mvbox_move, chat_msg, accepted_chat, outgoing, setupmessage);
 
         let t = TestContext::new_alice().await;
-        t.ctx
-            .set_config(Config::ConfiguredSpamFolder, Some("Spam"))
-            .await?;
         t.ctx
             .set_config(Config::ConfiguredMvboxFolder, Some("DeltaChat"))
             .await?;
@@ -2342,11 +2429,13 @@ mod tests {
 
         let (headers, _) = mailparse::parse_headers(bytes)?;
 
-        let actual = if let Some(config) = target_folder(&t, folder, &headers).await? {
-            t.get_config(config).await?
-        } else {
-            None
-        };
+        let is_spam_folder = folder == "Spam";
+        let actual =
+            if let Some(config) = target_folder(&t, folder, is_spam_folder, &headers).await? {
+                t.get_config(config).await?
+            } else {
+                None
+            };
 
         let expected = if expected_destination == folder {
             None
@@ -2460,4 +2549,27 @@ mod tests {
         }
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_get_imap_search_command() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        assert_eq!(
+            get_imap_self_sent_search_command(&t.ctx).await?,
+            r#"FROM "alice@example.org""#
+        );
+
+        t.ctx.set_primary_self_addr("alice@another.com").await?;
+        assert_eq!(
+            get_imap_self_sent_search_command(&t.ctx).await?,
+            r#"OR (FROM "alice@another.com") (FROM "alice@example.org")"#
+        );
+
+        t.ctx.set_primary_self_addr("alice@third.com").await?;
+        assert_eq!(
+            get_imap_self_sent_search_command(&t.ctx).await?,
+            r#"OR (OR (FROM "alice@third.com") (FROM "alice@another.com")) (FROM "alice@example.org")"#
+        );
+
+        Ok(())
+    }
 }

src/imap/idle.rs

@@ -157,7 +157,10 @@ impl Imap {
                     // in anything.  If so, we behave as if IDLE had data but
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
-                    match self.fetch_new_messages(context, &watch_folder, false).await {
+                    match self
+                        .fetch_new_messages(context, &watch_folder, false, false)
+                        .await
+                    {
                         Ok(res) => {
                             info!(context, "fetch_new_messages returned {:?}", res);
                             if res {

src/imap/scan_folders.rs

@@ -60,6 +60,9 @@ impl Imap {
             let is_drafts = folder_meaning == FolderMeaning::Drafts
                 || (folder_meaning == FolderMeaning::Unknown
                     && folder_name_meaning == FolderMeaning::Drafts);
+            let is_spam_folder = folder_meaning == FolderMeaning::Spam
+                || (folder_meaning == FolderMeaning::Unknown
+                    && folder_name_meaning == FolderMeaning::Spam);
 
             // Don't scan folders that are watched anyway
             if !watched_folders.contains(&folder.name().to_string()) && !is_drafts {
@@ -67,7 +70,7 @@ impl Imap {
                 self.server_sent_unsolicited_exists(context)?;
 
                 loop {
-                    self.fetch_move_delete(context, folder.name())
+                    self.fetch_move_delete(context, folder.name(), is_spam_folder)
                         .await
                         .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
 
@@ -79,16 +82,15 @@ impl Imap {
             }
         }
 
-        // We iterate over both folder meanings to make sure that if e.g. the "Sent" folder was deleted,
-        // `ConfiguredSentboxFolder` is set to `None`:
-        for config in &[
-            Config::ConfiguredSentboxFolder,
-            Config::ConfiguredSpamFolder,
-        ] {
-            context
-                .set_config(*config, folder_configs.get(config).map(|s| s.as_str()))
-                .await?;
-        }
+        // Set the `ConfiguredSentboxFolder` or set it to `None` if the folder was deleted.
+        context
+            .set_config(
+                Config::ConfiguredSentboxFolder,
+                folder_configs
+                    .get(&Config::ConfiguredSentboxFolder)
+                    .map(|s| s.as_str()),
+            )
+            .await?;
 
         last_scan.replace(Instant::now());
         Ok(true)

src/imex.rs

@@ -16,7 +16,7 @@ use rand::{thread_rng, Rng};
 use crate::blob::BlobObject;
 use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
 use crate::config::Config;
-use crate::constants::DC_CONTACT_ID_SELF;
+use crate::contact::ContactId;
 use crate::context::Context;
 use crate::dc_tools::{
     dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
@@ -176,7 +176,7 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     )
     .await?;
 
-    let chat_id = ChatId::create_for_contact(context, DC_CONTACT_ID_SELF).await?;
+    let chat_id = ChatId::create_for_contact(context, ContactId::SELF).await?;
     let mut msg = Message {
         viewtype: Viewtype::File,
         ..Default::default()
@@ -351,9 +351,8 @@ async fn set_self_key(
         }
     };
 
-    let self_addr = context.get_config(Config::ConfiguredAddr).await?;
-    ensure!(self_addr.is_some(), "Missing self addr");
-    let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
+    let self_addr = context.get_primary_self_addr().await?;
+    let addr = EmailAddress::new(&self_addr)?;
     let keypair = pgp::KeyPair {
         addr,
         public: public_key,

src/job.rs

@@ -4,7 +4,7 @@
 //! and job types.
 use std::fmt;
 
-use anyhow::{bail, format_err, Context as _, Error, Result};
+use anyhow::{format_err, Context as _, Result};
 use deltachat_derive::{FromSql, ToSql};
 use rand::{thread_rng, Rng};
 
@@ -13,14 +13,12 @@ use crate::contact::{normalize_name, Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::events::EventType;
-use crate::imap::{Imap, ImapActionResult};
-use crate::location;
-use crate::log::LogExt;
+use crate::imap::Imap;
 use crate::message::{Message, MsgId};
 use crate::mimefactory::MimeFactory;
 use crate::param::{Param, Params};
 use crate::scheduler::InterruptInfo;
-use crate::smtp::{smtp_send, Smtp};
+use crate::smtp::{smtp_send, SendResult, Smtp};
 use crate::sql;
 
 // results in ~3 weeks for the last backoff timespan
@@ -32,7 +30,6 @@ const JOB_RETRIES: u32 = 17;
 )]
 #[repr(u32)]
 pub(crate) enum Thread {
-    Unknown = 0,
     Imap = 100,
     Smtp = 5000,
 }
@@ -40,7 +37,7 @@ pub(crate) enum Thread {
 /// Job try result.
 #[derive(Debug, Display)]
 pub enum Status {
-    Finished(std::result::Result<(), Error>),
+    Finished(Result<()>),
     RetryNow,
     RetryLater,
 }
@@ -60,12 +57,6 @@ macro_rules! job_try {
     };
 }
 
-impl Default for Thread {
-    fn default() -> Self {
-        Thread::Unknown
-    }
-}
-
 #[derive(
     Debug,
     Display,
@@ -81,12 +72,8 @@ impl Default for Thread {
 )]
 #[repr(u32)]
 pub enum Action {
-    Unknown = 0,
-
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
-    Housekeeping = 105, // low priority ...
     FetchExistingMsgs = 110,
-    MarkseenMsgOnImap = 130,
 
     // this is user initiated so it should have a fairly high priority
     UpdateRecentQuota = 140,
@@ -102,33 +89,19 @@ pub enum Action {
     ResyncFolders = 300,
 
     // Jobs in the SMTP-thread, range from DC_SMTP_THREAD..DC_SMTP_THREAD+999
-    MaybeSendLocations = 5005, // low priority ...
-    MaybeSendLocationsEnded = 5007,
     SendMdn = 5010,
 }
 
-impl Default for Action {
-    fn default() -> Self {
-        Action::Unknown
-    }
-}
-
 impl From<Action> for Thread {
     fn from(action: Action) -> Thread {
         use Action::*;
 
         match action {
-            Unknown => Thread::Unknown,
-
-            Housekeeping => Thread::Imap,
             FetchExistingMsgs => Thread::Imap,
             ResyncFolders => Thread::Imap,
-            MarkseenMsgOnImap => Thread::Imap,
             UpdateRecentQuota => Thread::Imap,
             DownloadMsg => Thread::Imap,
 
-            MaybeSendLocations => Thread::Smtp,
-            MaybeSendLocationsEnded => Thread::Smtp,
             SendMdn => Thread::Smtp,
         }
     }
@@ -318,13 +291,18 @@ impl Job {
             return Status::RetryLater;
         }
 
-        let status = smtp_send(context, &recipients, &body, smtp, msg_id, 0).await;
-        if matches!(status, Status::Finished(Ok(_))) {
-            // Remove additional SendMdn jobs we have aggregated into this one.
-            job_try!(kill_ids(context, &additional_job_ids).await);
+        match smtp_send(context, &recipients, &body, smtp, msg_id, 0).await {
+            SendResult::Success => {
+                // Remove additional SendMdn jobs we have aggregated into this one.
+                job_try!(kill_ids(context, &additional_job_ids).await);
+                Status::Finished(Ok(()))
+            }
+            SendResult::Retry => {
+                info!(context, "Temporary SMTP failure while sending an MDN");
+                Status::RetryLater
+            }
+            SendResult::Failure(err) => Status::Finished(Err(err)),
         }
-
-        status
     }
 
     /// Read the recipients from old emails sent by the user and add them as contacts.
@@ -352,7 +330,7 @@ impl Job {
                 Config::ConfiguredSentboxFolder,
             ] {
                 if let Some(folder) = job_try!(context.get_config(*config).await) {
-                    if let Err(e) = imap.fetch_new_messages(context, &folder, true).await {
+                    if let Err(e) = imap.fetch_new_messages(context, &folder, false, true).await {
                         // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
                         warn!(context, "Could not fetch messages, retrying: {:#}", e);
                         return Status::RetryLater;
@@ -398,67 +376,6 @@ impl Job {
             Status::Finished(Ok(()))
         }
     }
-
-    async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.prepare(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let row = job_try!(
-            context
-                .sql
-                .query_row_optional(
-                    "SELECT uid, folder FROM imap
-                    WHERE rfc724_mid=? AND folder=target
-                    ORDER BY uid ASC
-                    LIMIT 1",
-                    paramsv![msg.rfc724_mid],
-                    |row| {
-                        let uid: u32 = row.get(0)?;
-                        let folder: String = row.get(1)?;
-                        Ok((uid, folder))
-                    }
-                )
-                .await
-        );
-        if let Some((server_uid, server_folder)) = row {
-            let result = imap.set_seen(context, &server_folder, server_uid).await;
-            match result {
-                ImapActionResult::RetryLater => return Status::RetryLater,
-                ImapActionResult::Success | ImapActionResult::Failed => {}
-            }
-        } else {
-            info!(
-                context,
-                "Can't mark the message {} as seen on IMAP because there is no known UID",
-                msg.rfc724_mid
-            );
-        }
-
-        // XXX we send MDN even in case of failure to mark the messages as seen, e.g. if it was
-        // already deleted on the server by another device. The job will not be retried so locally
-        // there is no risk of double-sending MDNs.
-        //
-        // Read receipts for system messages are never sent. These messages have no place to
-        // display received read receipt anyway.  And since their text is locally generated,
-        // quoting them is dangerous as it may contain contact names. E.g., for original message
-        // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
-        // be a display name stored in address book rather than the name sent in the From field by
-        // the user.
-        if msg.param.get_bool(Param::WantsMdn).unwrap_or_default() && !msg.is_system_message() {
-            let mdns_enabled = job_try!(context.get_config_bool(Config::MdnsEnabled).await);
-            if mdns_enabled {
-                if let Err(err) = send_mdn(context, &msg).await {
-                    warn!(context, "could not send out mdn for {}: {}", msg.id, err);
-                    return Status::Finished(Err(err));
-                }
-            }
-        }
-
-        Status::Finished(Ok(()))
-    }
 }
 
 /// Delete all pending jobs with the given action.
@@ -477,7 +394,7 @@ async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
     }
     let q = format!(
         "DELETE FROM jobs WHERE id IN({})",
-        sql::repeat_vars(job_ids.len())?
+        sql::repeat_vars(job_ids.len())
     );
     context
         .sql
@@ -648,19 +565,9 @@ async fn perform_job_action(
     );
 
     let try_res = match job.action {
-        Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))),
         Action::SendMdn => job.send_mdn(context, connection.smtp()).await,
-        Action::MaybeSendLocations => location::job_maybe_send_locations(context, job).await,
-        Action::MaybeSendLocationsEnded => {
-            location::job_maybe_send_locations_ended(context, job).await
-        }
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
-        Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
-        Action::Housekeeping => {
-            sql::housekeeping(context).await.ok_or_log(context);
-            Status::Finished(Ok(()))
-        }
         Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
             Ok(status) => status,
             Err(err) => Status::Finished(Err(err)),
@@ -693,13 +600,13 @@ fn get_backoff_time_offset(tries: u32, action: Action) -> i64 {
     }
 }
 
-async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
+pub(crate) async fn send_mdn(context: &Context, msg_id: MsgId, from_id: ContactId) -> Result<()> {
     let mut param = Params::new();
-    param.set(Param::MsgId, msg.id.to_u32().to_string());
+    param.set(Param::MsgId, msg_id.to_u32().to_string());
 
     add(
         context,
-        Job::new(Action::SendMdn, msg.from_id.to_u32(), param, 0),
+        Job::new(Action::SendMdn, from_id.to_u32(), param, 0),
     )
     .await?;
 
@@ -724,17 +631,14 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
 
     if delay_seconds == 0 {
         match action {
-            Action::Unknown => unreachable!(),
-            Action::Housekeeping
-            | Action::ResyncFolders
-            | Action::MarkseenMsgOnImap
+            Action::ResyncFolders
             | Action::FetchExistingMsgs
             | Action::UpdateRecentQuota
             | Action::DownloadMsg => {
                 info!(context, "interrupt: imap");
                 context.interrupt_inbox(InterruptInfo::new(false)).await;
             }
-            Action::MaybeSendLocations | Action::MaybeSendLocationsEnded | Action::SendMdn => {
+            Action::SendMdn => {
                 info!(context, "interrupt: smtp");
                 context.interrupt_smtp(InterruptInfo::new(false)).await;
             }
@@ -743,18 +647,6 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
     Ok(())
 }
 
-async fn load_housekeeping_job(context: &Context) -> Result<Option<Job>> {
-    let last_time = context.get_config_i64(Config::LastHousekeeping).await?;
-
-    let next_time = last_time + (60 * 60 * 24);
-    if next_time <= time() {
-        kill_action(context, Action::Housekeeping).await?;
-        Ok(Some(Job::new(Action::Housekeeping, 0, Params::new(), 0)))
-    } else {
-        Ok(None)
-    }
-}
-
 /// Load jobs from the database.
 ///
 /// Load jobs for this "[Thread]", i.e. either load SMTP jobs or load
@@ -798,7 +690,7 @@ LIMIT 1;
         params = paramsv![thread_i];
     };
 
-    let job = loop {
+    loop {
         let job_res = context
             .sql
             .query_row_optional(query, params.clone(), |row| {
@@ -817,7 +709,7 @@ LIMIT 1;
             .await;
 
         match job_res {
-            Ok(job) => break job,
+            Ok(job) => return Ok(job),
             Err(err) => {
                 // Remove invalid job from the DB
                 info!(context, "cleaning up job, because of {}", err);
@@ -835,20 +727,6 @@ LIMIT 1;
                     .with_context(|| format!("Failed to delete invalid job {}", id))?;
             }
         }
-    };
-
-    match thread {
-        Thread::Unknown => {
-            bail!("unknown thread for job")
-        }
-        Thread::Imap => {
-            if let Some(job) = job {
-                Ok(Some(job))
-            } else {
-                Ok(load_housekeeping_job(context).await?)
-            }
-        }
-        Thread::Smtp => Ok(job),
     }
 }
 
@@ -896,8 +774,7 @@ mod tests {
             &InterruptInfo::new(false),
         )
         .await?;
-        // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
-        assert_eq!(jobs.unwrap().action, Action::Housekeeping);
+        assert!(jobs.is_none());
 
         insert_job(&t, 1, true).await;
         let jobs = load_next(

src/key.rs

@@ -91,16 +91,17 @@ impl DcKey for SignedPublicKey {
     type KeyType = SignedPublicKey;
 
     async fn load_self(context: &Context) -> Result<Self::KeyType> {
+        let addr = context.get_primary_self_addr().await?;
         match context
             .sql
             .query_row_optional(
                 r#"
             SELECT public_key
               FROM keypairs
-             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
+             WHERE addr=?
                AND is_default=1;
             "#,
-                paramsv![],
+                paramsv![addr],
                 |row| {
                     let bytes: Vec<u8> = row.get(0)?;
                     Ok(bytes)
@@ -198,10 +199,7 @@ impl DcSecretKey for SignedSecretKey {
 }
 
 async fn generate_keypair(context: &Context) -> Result<KeyPair> {
-    let addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await?
-        .context("no address configured")?;
+    let addr = context.get_primary_self_addr().await?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 

src/location.rs

@@ -1,21 +1,20 @@
 //! Location handling.
 use std::convert::TryFrom;
 
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Context as _, Result};
+use async_std::channel::Receiver;
+use async_std::future::timeout;
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+use std::time::Duration;
 
 use crate::chat::{self, ChatId};
-use crate::config::Config;
-use crate::constants::DC_CONTACT_ID_SELF;
 use crate::contact::ContactId;
 use crate::context::Context;
-use crate::dc_tools::time;
+use crate::dc_tools::{duration_to_str, time};
 use crate::events::EventType;
-use crate::job::{self, Job};
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
-use crate::param::Params;
 use crate::stock_str;
 
 /// Location record
@@ -228,32 +227,11 @@ pub async fn send_locations_to_chat(
     }
     context.emit_event(EventType::ChatModified(chat_id));
     if 0 != seconds {
-        schedule_maybe_send_locations(context, false).await?;
-        job::add(
-            context,
-            job::Job::new(
-                job::Action::MaybeSendLocationsEnded,
-                chat_id.to_u32(),
-                Params::new(),
-                seconds + 1,
-            ),
-        )
-        .await?;
+        context.interrupt_location().await;
     }
     Ok(())
 }
 
-async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) -> Result<()> {
-    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await? {
-        job::add(
-            context,
-            job::Job::new(job::Action::MaybeSendLocations, 0, Params::new(), 60),
-        )
-        .await?;
-    };
-    Ok(())
-}
-
 /// Returns whether `chat_id` or any chat is sending locations.
 ///
 /// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
@@ -315,18 +293,18 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                         accuracy,
                         time(),
                         chat_id,
-                        DC_CONTACT_ID_SELF,
+                        ContactId::SELF,
                     ]
             ).await {
                 warn!(context, "failed to store location {:?}", err);
             } else {
+                info!(context, "stored location for chat {}", chat_id);
                 continue_streaming = true;
             }
         }
         if continue_streaming {
-            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+            context.emit_event(EventType::LocationChanged(Some(ContactId::SELF)));
         };
-        schedule_maybe_send_locations(context, false).await.ok();
     }
 
     continue_streaming
@@ -425,10 +403,7 @@ pub async fn delete_all(context: &Context) -> Result<()> {
 pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)> {
     let mut last_added_location_id = 0;
 
-    let self_addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await?
-        .unwrap_or_default();
+    let self_addr = context.get_primary_self_addr().await?;
 
     let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
         "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
@@ -463,10 +438,10 @@ pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)
              GROUP BY timestamp \
              ORDER BY timestamp;",
                 paramsv![
-                    DC_CONTACT_ID_SELF,
+                    ContactId::SELF,
                     locations_send_begin,
                     locations_last_sent,
-                    DC_CONTACT_ID_SELF
+                    ContactId::SELF
                 ],
                 |row| {
                     let location_id: i32 = row.get(0)?;
@@ -612,147 +587,140 @@ pub(crate) async fn save(
     Ok(newest_location_id)
 }
 
-pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> job::Status {
-    let now = time();
-    let mut continue_streaming = false;
-    info!(
-        context,
-        " ----------------- MAYBE_SEND_LOCATIONS -------------- ",
-    );
+pub(crate) async fn location_loop(context: &Context, interrupt_receiver: Receiver<()>) {
+    loop {
+        let next_event = match maybe_send_locations(context).await {
+            Err(err) => {
+                warn!(context, "maybe_send_locations failed: {}", err);
+                Some(60) // Retry one minute later.
+            }
+            Ok(next_event) => next_event,
+        };
 
+        let duration = if let Some(next_event) = next_event {
+            Duration::from_secs(next_event)
+        } else {
+            Duration::from_secs(86400)
+        };
+
+        info!(
+            context,
+            "Location loop is waiting for {} or interrupt",
+            duration_to_str(duration)
+        );
+        timeout(duration, interrupt_receiver.recv()).await.ok();
+    }
+}
+
+/// Returns number of seconds until the next time location streaming for some chat ends
+/// automatically.
+async fn maybe_send_locations(context: &Context) -> Result<Option<u64>> {
+    let mut next_event: Option<u64> = None;
+
+    let now = time();
     let rows = context
         .sql
         .query_map(
-            "SELECT id, locations_send_begin, locations_last_sent \
-         FROM chats \
-         WHERE locations_send_until>?;",
-            paramsv![now],
+            "SELECT id, locations_send_begin, locations_send_until, locations_last_sent
+             FROM chats
+             WHERE locations_send_until>0",
+            [],
             |row| {
                 let chat_id: ChatId = row.get(0)?;
                 let locations_send_begin: i64 = row.get(1)?;
-                let locations_last_sent: i64 = row.get(2)?;
-                continue_streaming = true;
-
-                // be a bit tolerant as the timer may not align exactly with time(NULL)
-                if now - locations_last_sent < (60 - 3) {
-                    Ok(None)
-                } else {
-                    Ok(Some((chat_id, locations_send_begin, locations_last_sent)))
-                }
+                let locations_send_until: i64 = row.get(2)?;
+                let locations_last_sent: i64 = row.get(3)?;
+                Ok((
+                    chat_id,
+                    locations_send_begin,
+                    locations_send_until,
+                    locations_last_sent,
+                ))
             },
             |rows| {
-                rows.filter_map(|v| v.transpose())
-                    .collect::<std::result::Result<Vec<_>, _>>()
+                rows.collect::<std::result::Result<Vec<_>, _>>()
                     .map_err(Into::into)
             },
         )
-        .await;
+        .await
+        .context("failed to query location streaming chats")?;
 
-    if let Ok(rows) = rows {
-        let mut msgs = Vec::new();
-
-        {
-            let conn = job_try!(context.sql.get_conn().await);
-
-            let mut stmt_locations = job_try!(conn.prepare_cached(
-                "SELECT id \
+    for (chat_id, locations_send_begin, locations_send_until, locations_last_sent) in rows {
+        if locations_send_begin > 0 && locations_send_until > now {
+            let can_send = now > locations_last_sent + 60;
+            let has_locations = context
+                .sql
+                .exists(
+                    "SELECT COUNT(id) \
      FROM locations \
      WHERE from_id=? \
      AND timestamp>=? \
      AND timestamp>? \
-     AND independent=0 \
-             ORDER BY timestamp;",
-            ));
+     AND independent=0",
+                    paramsv![ContactId::SELF, locations_send_begin, locations_last_sent,],
+                )
+                .await?;
 
-            for (chat_id, locations_send_begin, locations_last_sent) in &rows {
-                if !stmt_locations
-                    .exists(paramsv![
-                        DC_CONTACT_ID_SELF,
-                        *locations_send_begin,
-                        *locations_last_sent,
-                    ])
-                    .unwrap_or_default()
-                {
-                    // if there is no new location, there's nothing to send.
-                    // however, maybe we want to bypass this test eg. 15 minutes
-                } else {
-                    // pending locations are attached automatically to every message,
+            next_event = next_event
+                .into_iter()
+                .chain(u64::try_from(locations_send_until - now).into_iter())
+                .min();
+
+            if has_locations {
+                if can_send {
+                    // Send location-only message.
+                    // Pending locations are attached automatically to every message,
                     // so also to this empty text message.
-                    // DC_CMD_LOCATION is only needed to create a nicer subject.
-                    //
-                    // for optimisation and to avoid flooding the sending queue,
-                    // we could sending these messages only if we're really online.
-                    // the easiest way to determine this, is to check for an empty message queue.
-                    // (might not be 100%, however, as positions are sent combined later
-                    // and dc_set_location() is typically called periodically, this is ok)
+                    info!(
+                        context,
+                        "Chat {} has pending locations, sending them.", chat_id
+                    );
                     let mut msg = Message::new(Viewtype::Text);
                     msg.hidden = true;
                     msg.param.set_cmd(SystemMessage::LocationOnly);
-                    msgs.push((*chat_id, msg));
+                    chat::send_msg(context, chat_id, &mut msg).await?;
+                } else {
+                    // Wait until pending locations can be sent.
+                    info!(
+                        context,
+                        "Chat {} has pending locations, but they can't be sent yet.", chat_id
+                    );
+                    next_event = next_event
+                        .into_iter()
+                        .chain(u64::try_from(locations_last_sent + 61 - now).into_iter())
+                        .min();
                 }
+            } else {
+                info!(
+                    context,
+                    "Chat {} has location streaming enabled, but no pending locations.", chat_id
+                );
             }
-        }
-
-        for (chat_id, mut msg) in msgs.into_iter() {
-            // TODO: better error handling
-            chat::send_msg(context, chat_id, &mut msg)
-                .await
-                .unwrap_or_default();
-        }
-    }
-
-    if continue_streaming {
-        job_try!(schedule_maybe_send_locations(context, true).await);
-    }
-    job::Status::Finished(Ok(()))
-}
-
-pub(crate) async fn job_maybe_send_locations_ended(
-    context: &Context,
-    job: &mut Job,
-) -> job::Status {
-    // this function is called when location-streaming _might_ have ended for a chat.
-    // the function checks, if location-streaming is really ended;
-    // if so, a device-message is added if not yet done.
-
-    let chat_id = ChatId::new(job.foreign_id);
-
-    let (send_begin, send_until) = job_try!(
-        context
-            .sql
-            .query_row(
-                "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
-                paramsv![chat_id],
-                |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
-            )
-            .await
-    );
-
-    let now = time();
-    if !(send_begin != 0 && now <= send_until) {
-        // still streaming -
-        // may happen as several calls to dc_send_locations_to_chat()
-        // do not un-schedule pending DC_MAYBE_SEND_LOC_ENDED jobs
-        if !(send_begin == 0 && send_until == 0) {
-            // not streaming, device-message already sent
-            job_try!(
-                context
-                    .sql
-                    .execute(
-                        "UPDATE chats \
-                             SET locations_send_begin=0, locations_send_until=0 \
- WHERE id=?",
-                        paramsv![chat_id],
-                    )
-                    .await
+        } else {
+            // Location streaming was either explicitly disabled (locations_send_begin = 0) or
+            // locations_send_until is in the past.
+            info!(
+                context,
+                "Disabling location streaming for chat {}.", chat_id
             );
+            context
+                .sql
+                .execute(
+                    "UPDATE chats \
+                         SET locations_send_begin=0, locations_send_until=0 \
+                         WHERE id=?",
+                    paramsv![chat_id],
+                )
+                .await
+                .context("failed to disable location streaming")?;
 
             let stock_str = stock_str::msg_location_disabled(context).await;
-            job_try!(chat::add_info_msg(context, chat_id, &stock_str, now).await);
+            chat::add_info_msg(context, chat_id, &stock_str, now).await?;
             context.emit_event(EventType::ChatModified(chat_id));
         }
     }
-    job::Status::Finished(Ok(()))
+
+    Ok(next_event)
 }
 
 #[cfg(test)]
@@ -760,6 +728,7 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
+    use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils::TestContext;
 
     #[async_std::test]
@@ -820,4 +789,68 @@ mod tests {
         assert!(!is_marker(" "));
         assert!(!is_marker("\t"));
     }
+
+    /// Tests that location.kml is hidden.
+    #[async_std::test]
+    async fn receive_location_kml() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        dc_receive_imf(
+            &alice,
+            br#"Subject: Hello
+Message-ID: hello@example.net
+To: Alice <alice@example.org>
+From: Bob <bob@example.net>
+Date: Mon, 20 Dec 2021 00:00:00 +0000
+Chat-Version: 1.0
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+Text message."#,
+            false,
+        )
+        .await?;
+        let received_msg = alice.get_last_msg().await;
+        assert_eq!(received_msg.text.unwrap(), "Text message.");
+
+        dc_receive_imf(
+            &alice,
+            br#"Subject: locations
+MIME-Version: 1.0
+To: <alice@example.org>
+From: <bob@example.net>
+Date: Tue, 21 Dec 2021 00:00:00 +0000
+Chat-Version: 1.0
+Message-ID: <foobar@example.net>
+Content-Type: multipart/mixed; boundary="U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF"
+
+
+--U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+
+
+--U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
+Content-Type: application/vnd.google-earth.kml+xml
+Content-Disposition: attachment; filename="location.kml"
+
+<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2">
+<Document addr="bob@example.net">
+<Placemark><Timestamp><when>2021-11-21T00:00:00Z</when></Timestamp><Point><coordinates accuracy="1.0000000000000000">10.00000000000000,20.00000000000000</coordinates></Point></Placemark>
+</Document>
+</kml>
+
+--U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF--"#,
+            false,
+        )
+        .await?;
+
+        // Received location message is not visible, last message stays the same.
+        let received_msg2 = alice.get_last_msg().await;
+        assert_eq!(received_msg2.id, received_msg.id);
+
+        let locations = get_range(&alice, None, None, 0, 0).await?;
+        assert_eq!(locations.len(), 1);
+        Ok(())
+    }
 }

src/login_param.rs

@@ -139,8 +139,18 @@ pub struct LoginParam {
 }
 
 impl LoginParam {
+    /// Load entered (candidate) account settings
+    pub async fn load_candidate_params(context: &Context) -> Result<Self> {
+        LoginParam::from_database(context, "").await
+    }
+
+    /// Load configured (working) account settings
+    pub async fn load_configured_params(context: &Context) -> Result<Self> {
+        LoginParam::from_database(context, "configured_").await
+    }
+
     /// Read the login parameters from the database.
-    pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
+    async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
@@ -242,12 +252,11 @@ impl LoginParam {
     }
 
     /// Save this loginparam to the database.
-    pub async fn save_to_database(&self, context: &Context, prefix: impl AsRef<str>) -> Result<()> {
-        let prefix = prefix.as_ref();
+    pub async fn save_as_configured_params(&self, context: &Context) -> Result<()> {
+        let prefix = "configured_";
         let sql = &context.sql;
 
-        let key = format!("{}addr", prefix);
-        sql.set_raw_config(key, Some(&self.addr)).await?;
+        context.set_primary_self_addr(&self.addr).await?;
 
         let key = format!("{}mail_server", prefix);
         sql.set_raw_config(key, Some(&self.imap.server)).await?;
@@ -438,8 +447,8 @@ mod tests {
             socks5_config: None,
         };
 
-        param.save_to_database(&t, "foobar_").await?;
-        let loaded = LoginParam::from_database(&t, "foobar_").await?;
+        param.save_as_configured_params(&t).await?;
+        let loaded = LoginParam::load_configured_params(&t).await?;
 
         assert_eq!(param, loaded);
         Ok(())

src/message.rs

@@ -9,9 +9,9 @@ use rusqlite::types::ValueRef;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
+use crate::config::Config;
 use crate::constants::{
-    Blocked, Chattype, VideochatType, DC_CHAT_ID_TRASH, DC_CONTACT_ID_INFO, DC_CONTACT_ID_SELF,
-    DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
+    Blocked, Chattype, VideochatType, DC_CHAT_ID_TRASH, DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
 };
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
@@ -22,7 +22,8 @@ use crate::dc_tools::{
 use crate::download::DownloadState;
 use crate::ephemeral::{start_ephemeral_timers_msgids, Timer as EphemeralTimer};
 use crate::events::EventType;
-use crate::job::{self, Action};
+use crate::imap::markseen_on_imap_table;
+use crate::job;
 use crate::log::LogExt;
 use crate::mimeparser::{parse_message_id, FailureReport, SystemMessage};
 use crate::param::{Param, Params};
@@ -431,7 +432,7 @@ impl Message {
     /// this is done by dc_set_location() and dc_send_locations_to_chat().
     ///
     /// Typically results in the event #DC_EVENT_LOCATION_CHANGED with
-    /// contact_id set to DC_CONTACT_ID_SELF.
+    /// contact_id set to ContactId::SELF.
     ///
     /// @param latitude North-south position of the location.
     /// @param longitude East-west position of the location.
@@ -544,7 +545,7 @@ impl Message {
             &chat_loaded
         };
 
-        let contact = if self.from_id != DC_CONTACT_ID_SELF {
+        let contact = if self.from_id != ContactId::SELF {
             match chat.typ {
                 Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
                     Some(Contact::get_by_id(context, self.from_id).await?)
@@ -597,8 +598,8 @@ impl Message {
 
     pub fn is_info(&self) -> bool {
         let cmd = self.param.get_cmd();
-        self.from_id == DC_CONTACT_ID_INFO
-            || self.to_id == DC_CONTACT_ID_INFO
+        self.from_id == ContactId::INFO
+            || self.to_id == ContactId::INFO
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
@@ -1017,7 +1018,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
     ret += &format!(" by {}", name);
     ret += "\n";
 
-    if msg.from_id != DC_CONTACT_ID_SELF {
+    if msg.from_id != ContactId::SELF {
         let s = dc_timestamp_to_str(if 0 != msg.timestamp_rcvd {
             msg.timestamp_rcvd
         } else {
@@ -1038,7 +1039,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
         );
     }
 
-    if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
+    if msg.from_id == ContactId::INFO || msg.to_id == ContactId::INFO {
         // device-internal message, no further details needed
         return Ok(ret);
     }
@@ -1253,19 +1254,13 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     }
 
     if !msg_ids.is_empty() {
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
-        job::kill_action(context, Action::Housekeeping).await?;
-        job::add(
-            context,
-            job::Job::new(Action::Housekeeping, 0, Params::new(), 10),
-        )
-        .await?;
+        context.emit_msgs_changed_without_ids();
+
+        // Run housekeeping to delete unused blobs.
+        context.set_config(Config::LastHousekeeping, None).await?;
     }
 
-    // Interrupt Inbox loop to start message deletion.
+    // Interrupt Inbox loop to start message deletion and run housekeeping.
     context.interrupt_inbox(InterruptInfo::new(false)).await;
     Ok(())
 }
@@ -1295,22 +1290,31 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
                     m.chat_id AS chat_id,
                     m.state AS state,
                     m.ephemeral_timer AS ephemeral_timer,
+                    m.param AS param,
+                    m.from_id AS from_id,
+                    m.rfc724_mid AS rfc724_mid,
                     c.blocked AS blocked
                  FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id
                  WHERE m.id IN ({}) AND m.chat_id>9",
-                sql::repeat_vars(msg_ids.len())?
+                sql::repeat_vars(msg_ids.len())
             ),
             rusqlite::params_from_iter(&msg_ids),
             |row| {
                 let id: MsgId = row.get("id")?;
                 let chat_id: ChatId = row.get("chat_id")?;
                 let state: MessageState = row.get("state")?;
+                let param: Params = row.get::<_, String>("param")?.parse().unwrap_or_default();
+                let from_id: ContactId = row.get("from_id")?;
+                let rfc724_mid: String = row.get("rfc724_mid")?;
                 let blocked: Option<Blocked> = row.get("blocked")?;
                 let ephemeral_timer: EphemeralTimer = row.get("ephemeral_timer")?;
                 Ok((
                     id,
                     chat_id,
                     state,
+                    param,
+                    from_id,
+                    rfc724_mid,
                     blocked.unwrap_or_default(),
                     ephemeral_timer,
                 ))
@@ -1319,30 +1323,52 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
         )
         .await?;
 
-    if msgs
-        .iter()
-        .any(|(_id, _chat_id, _state, _blocked, ephemeral_timer)| {
+    if msgs.iter().any(
+        |(_id, _chat_id, _state, _param, _from_id, _rfc724_mid, _blocked, ephemeral_timer)| {
             *ephemeral_timer != EphemeralTimer::Disabled
-        })
-    {
+        },
+    ) {
         start_ephemeral_timers_msgids(context, &msg_ids)
             .await
             .context("failed to start ephemeral timers")?;
     }
 
     let mut updated_chat_ids = BTreeSet::new();
-    for (id, curr_chat_id, curr_state, curr_blocked, _curr_ephemeral_timer) in msgs.into_iter() {
+    for (
+        id,
+        curr_chat_id,
+        curr_state,
+        curr_param,
+        curr_from_id,
+        curr_rfc724_mid,
+        curr_blocked,
+        _curr_ephemeral_timer,
+    ) in msgs.into_iter()
+    {
         if curr_blocked == Blocked::Not
             && (curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed)
         {
             update_msg_state(context, id, MessageState::InSeen).await?;
             info!(context, "Seen message {}.", id);
 
-            job::add(
-                context,
-                job::Job::new(Action::MarkseenMsgOnImap, id.to_u32(), Params::new(), 0),
-            )
-            .await?;
+            markseen_on_imap_table(context, &curr_rfc724_mid).await?;
+
+            // Read receipts for system messages are never sent. These messages have no place to
+            // display received read receipt anyway.  And since their text is locally generated,
+            // quoting them is dangerous as it may contain contact names. E.g., for original message
+            // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
+            // be a display name stored in address book rather than the name sent in the From field by
+            // the user.
+            if curr_param.get_bool(Param::WantsMdn).unwrap_or_default()
+                && curr_param.get_cmd() == SystemMessage::Unknown
+            {
+                let mdns_enabled = context.get_config_bool(Config::MdnsEnabled).await?;
+                if mdns_enabled {
+                    if let Err(err) = job::send_mdn(context, id, curr_from_id).await {
+                        warn!(context, "could not send out mdn for {}: {}", id, err);
+                    }
+                }
+            }
             updated_chat_ids.insert(curr_chat_id);
         }
     }
@@ -1432,7 +1458,7 @@ pub async fn handle_mdn(
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Result<Option<(ChatId, MsgId)>> {
-    if from_id == DC_CONTACT_ID_SELF {
+    if from_id == ContactId::SELF {
         warn!(
             context,
             "ignoring MDN sent to self, this is a bug on the sender device"
@@ -1637,7 +1663,7 @@ pub async fn estimate_deletion_cnt(
     from_server: bool,
     seconds: i64,
 ) -> Result<usize> {
-    let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+    let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
         .await?
         .unwrap_or_default();
     let threshold_timestamp = time() - seconds;
@@ -1805,7 +1831,6 @@ mod tests {
 
     use crate::chat::{marknoticed_chat, ChatItem};
     use crate::chatlist::Chatlist;
-    use crate::constants::DC_CONTACT_ID_DEVICE;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils as test;
     use crate::test_utils::TestContext;
@@ -1945,7 +1970,7 @@ mod tests {
         // test that get_width() and get_height() are returning some dimensions for images;
         // (as the device-chat contains a welcome-images, we check that)
         t.update_device_chats().await.ok();
-        let device_chat_id = ChatId::get_for_contact(&t, DC_CONTACT_ID_DEVICE)
+        let device_chat_id = ChatId::get_for_contact(&t, ContactId::DEVICE)
             .await
             .unwrap();
 
@@ -2019,7 +2044,6 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello\n",
-            "INBOX",
             false,
         )
         .await
@@ -2228,7 +2252,6 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -2246,7 +2269,6 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello again\n",
-            "INBOX",
             false,
         )
         .await?;

src/mimefactory.rs

@@ -133,11 +133,7 @@ impl<'a> MimeFactory<'a> {
     ) -> Result<MimeFactory<'a>> {
         let chat = Chat::load_from_db(context, msg.chat_id).await?;
 
-        let from_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
-
+        let from_addr = context.get_primary_self_addr().await?;
         let config_displayname = context
             .get_config(Config::Displayname)
             .await?
@@ -237,10 +233,7 @@ impl<'a> MimeFactory<'a> {
         ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
         let contact = Contact::load_from_db(context, msg.from_id).await?;
-        let from_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
+        let from_addr = context.get_primary_self_addr().await?;
         let from_displayname = context
             .get_config(Config::Displayname)
             .await?
@@ -278,10 +271,7 @@ impl<'a> MimeFactory<'a> {
         &self,
         context: &Context,
     ) -> Result<Vec<(Option<Peerstate>, &str)>> {
-        let self_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .context("not configured")?;
+        let self_addr = context.get_primary_self_addr().await?;
 
         let mut res = Vec::new();
         for (_, addr) in self
@@ -860,9 +850,12 @@ impl<'a> MimeFactory<'a> {
         }
 
         if chat.typ == Chattype::Group {
-            headers
-                .protected
-                .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
+            // Send group ID unless it is an ad hoc group that has no ID.
+            if !chat.grpid.is_empty() {
+                headers
+                    .protected
+                    .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
+            }
 
             let encoded = encode_words(&chat.name);
             headers
@@ -1672,7 +1665,6 @@ mod tests {
             Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
             \n\
             hello\n",
-            "INBOX",
             false,
         )
         .await
@@ -1765,7 +1757,6 @@ mod tests {
                 t.get_last_msg().await.rfc724_mid
             )
             .as_bytes(),
-            "INBOX",
             false,
         )
         .await?;
@@ -1876,7 +1867,6 @@ mod tests {
                     Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                     \n\
                     Some other, completely unrelated content\n",
-                "INBOX",
                 false,
             )
             .await
@@ -1901,9 +1891,7 @@ mod tests {
             .await
             .unwrap();
 
-        dc_receive_imf(context, imf_raw, "INBOX", false)
-            .await
-            .unwrap();
+        dc_receive_imf(context, imf_raw, false).await.unwrap();
 
         let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
 

src/mimeparser.rs

@@ -2915,7 +2915,6 @@ On 2020-10-25, Bob wrote:
         dc_receive_imf(
             &t.ctx,
             include_bytes!("../test-data/message/subj_with_multimedia_msg.eml"),
-            "INBOX",
             false,
         )
         .await
@@ -3064,7 +3063,7 @@ Subject: ...
 
 Some quote.
 "###;
-        dc_receive_imf(&t, raw, "INBOX", false).await?;
+        dc_receive_imf(&t, raw, false).await?;
 
         // Delta Chat generates In-Reply-To with a starting tab when Message-ID is too long.
         let raw = br###"In-Reply-To:
@@ -3081,7 +3080,7 @@ Subject: ...
 Some reply
 "###;
 
-        dc_receive_imf(&t, raw, "INBOX", false).await?;
+        dc_receive_imf(&t, raw, false).await?;
 
         let msg = t.get_last_msg().await;
         assert_eq!(msg.get_text().unwrap(), "Some reply");
@@ -3109,13 +3108,13 @@ Message.
 "###;
 
         // Bob receives message.
-        dc_receive_imf(&bob, raw, "INBOX", false).await?;
+        dc_receive_imf(&bob, raw, false).await?;
         let msg = bob.get_last_msg().await;
         // Message is incoming.
         assert!(msg.param.get_bool(Param::WantsMdn).unwrap());
 
         // Alice receives copy-to-self.
-        dc_receive_imf(&alice, raw, "INBOX", false).await?;
+        dc_receive_imf(&alice, raw, false).await?;
         let msg = alice.get_last_msg().await;
         // Message is outgoing, don't send read receipt to self.
         assert!(msg.param.get_bool(Param::WantsMdn).is_none());
@@ -3141,7 +3140,6 @@ Message.
                  \n\
                  hello\n"
                 .as_bytes(),
-            "INBOX",
             false,
         )
         .await?;
@@ -3179,7 +3177,6 @@ Message.
                  \n\
                  --SNIPP--"
             .as_bytes(),
-            "INBOX",
             false,
         )
         .await?;
@@ -3202,7 +3199,7 @@ Message.
 
         let original =
             include_bytes!("../test-data/message/ms_exchange_report_original_message.eml");
-        dc_receive_imf(&t, original, "INBOX", false).await?;
+        dc_receive_imf(&t, original, false).await?;
         let original_msg_id = t.get_last_msg().await.id;
 
         // 1. Test mimeparser directly
@@ -3217,7 +3214,7 @@ Message.
         assert!(mimeparser.mdn_reports[0].additional_message_ids.is_empty());
 
         // 2. Test that marking the original msg as read works
-        dc_receive_imf(&t, mdn, "INBOX", false).await?;
+        dc_receive_imf(&t, mdn, false).await?;
 
         assert_eq!(
             original_msg_id.get_state(&t).await?,

src/peerstate.rs

@@ -4,11 +4,13 @@ use std::collections::HashSet;
 use std::fmt;
 
 use crate::aheader::{Aheader, EncryptPreference};
-use crate::chat::{self, ChatIdBlocked};
-use crate::constants::Blocked;
+use crate::chat::{self};
+use crate::chatlist::Chatlist;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
+use crate::message::Message;
+use crate::mimeparser::SystemMessage;
 use crate::sql::Sql;
 use crate::stock_str;
 use anyhow::{bail, Result};
@@ -271,14 +273,34 @@ impl Peerstate {
                 .query_get_value("SELECT id FROM contacts WHERE addr=?;", paramsv![self.addr])
                 .await?
             {
-                let chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Request)
-                    .await?
-                    .id;
-
+                let chats = Chatlist::try_load(context, 0, None, contact_id).await?;
                 let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
-
-                chat::add_info_msg(context, chat_id, &msg, timestamp).await?;
-                context.emit_event(EventType::ChatModified(chat_id));
+                for (chat_id, msg_id) in chats.iter() {
+                    let timestamp_sort = if let Some(msg_id) = msg_id {
+                        let lastmsg = Message::load_from_db(context, *msg_id).await?;
+                        lastmsg.timestamp_sort
+                    } else {
+                        context
+                            .sql
+                            .query_get_value(
+                                "SELECT created_timestamp FROM chats WHERE id=?;",
+                                paramsv![chat_id],
+                            )
+                            .await?
+                            .unwrap_or(0)
+                    };
+                    chat::add_info_msg_with_cmd(
+                        context,
+                        *chat_id,
+                        &msg,
+                        SystemMessage::Unknown,
+                        timestamp_sort,
+                        Some(timestamp),
+                        None,
+                    )
+                    .await?;
+                    context.emit_event(EventType::ChatModified(*chat_id));
+                }
             } else {
                 bail!("contact with peerstate.addr {:?} not found", &self.addr);
             }

src/qr.rs

@@ -711,7 +711,7 @@ mod tests {
             ..
         } = qr
         {
-            assert_ne!(contact_id, ContactId::new(0));
+            assert_ne!(contact_id, ContactId::UNDEFINED);
             assert_eq!(grpname, "test ? test !");
         } else {
             bail!("Wrong QR code type");
@@ -729,7 +729,7 @@ mod tests {
             ..
         } = qr
         {
-            assert_ne!(contact_id, ContactId::new(0));
+            assert_ne!(contact_id, ContactId::UNDEFINED);
             assert_eq!(grpname, "test ? test !");
 
             let contact = Contact::get_by_id(&ctx.ctx, contact_id).await?;
@@ -751,7 +751,7 @@ mod tests {
         ).await?;
 
         if let Qr::AskVerifyContact { contact_id, .. } = qr {
-            assert_ne!(contact_id, ContactId::new(0));
+            assert_ne!(contact_id, ContactId::UNDEFINED);
         } else {
             bail!("Wrong QR code type");
         }

src/qr_code_generator.rs

@@ -6,8 +6,7 @@ use crate::{
     chat::{Chat, ChatId},
     color::color_int_to_hex_string,
     config::Config,
-    constants::DC_CONTACT_ID_SELF,
-    contact::Contact,
+    contact::{Contact, ContactId},
     context::Context,
     securejoin, stock_str,
 };
@@ -41,7 +40,7 @@ async fn generate_join_group_qr_code(context: &Context, chat_id: ChatId) -> Resu
 }
 
 async fn generate_verification_qr(context: &Context) -> Result<String> {
-    let contact = Contact::get_by_id(context, DC_CONTACT_ID_SELF).await?;
+    let contact = Contact::get_by_id(context, ContactId::SELF).await?;
 
     let avatar = match contact.get_profile_image(context).await? {
         Some(path) => {

src/scheduler.rs

@@ -8,18 +8,19 @@ use async_std::{
 use crate::config::Config;
 use crate::context::Context;
 use crate::dc_tools::maybe_add_time_based_warnings;
-use crate::ephemeral::delete_expired_imap_messages;
+use crate::dc_tools::time;
+use crate::ephemeral::{self, delete_expired_imap_messages};
 use crate::imap::Imap;
 use crate::job::{self, Thread};
+use crate::location;
 use crate::log::LogExt;
 use crate::smtp::{send_smtp_messages, Smtp};
+use crate::sql;
 
 use self::connectivity::ConnectivityStore;
 
 pub(crate) mod connectivity;
 
-pub(crate) struct StopToken;
-
 /// Job and connection scheduler.
 #[derive(Debug)]
 #[allow(clippy::large_enum_variant)]
@@ -34,6 +35,10 @@ pub(crate) enum Scheduler {
         sentbox_handle: Option<task::JoinHandle<()>>,
         smtp: SmtpConnectionState,
         smtp_handle: Option<task::JoinHandle<()>>,
+        ephemeral_handle: Option<task::JoinHandle<()>>,
+        ephemeral_interrupt_send: Sender<()>,
+        location_handle: Option<task::JoinHandle<()>>,
+        location_interrupt_send: Sender<()>,
     },
 }
 
@@ -59,6 +64,14 @@ impl Context {
     pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
         self.scheduler.read().await.interrupt_smtp(info).await;
     }
+
+    pub(crate) async fn interrupt_ephemeral_task(&self) {
+        self.scheduler.read().await.interrupt_ephemeral_task().await;
+    }
+
+    pub(crate) async fn interrupt_location(&self) {
+        self.scheduler.read().await.interrupt_location().await;
+    }
 }
 
 async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConnectionHandlers) {
@@ -68,19 +81,16 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
     let ImapConnectionHandlers {
         mut connection,
         stop_receiver,
-        shutdown_sender,
     } = inbox_handlers;
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started
-            .send(())
-            .await
-            .expect("inbox loop, missing started receiver");
         let ctx = ctx1;
+        if let Err(err) = started.send(()).await {
+            warn!(ctx, "inbox loop, missing started receiver: {}", err);
+            return;
+        };
 
-        // track number of continously executed jobs
-        let mut jobs_loaded = 0;
         let mut info = InterruptInfo::default();
         loop {
             let job = match job::load_next(&ctx, Thread::Imap, &info).await {
@@ -92,22 +102,26 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
             };
 
             match job {
-                Some(job) if jobs_loaded <= 20 => {
-                    jobs_loaded += 1;
+                Some(job) => {
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
                     info = Default::default();
                 }
-                Some(job) => {
-                    // Let the fetch run, but return back to the job afterwards.
-                    jobs_loaded = 0;
-                    info!(ctx, "postponing imap-job {} to run fetch...", job);
-                    fetch(&ctx, &mut connection).await;
-                }
                 None => {
-                    jobs_loaded = 0;
-
                     maybe_add_time_based_warnings(&ctx).await;
 
+                    match ctx.get_config_i64(Config::LastHousekeeping).await {
+                        Ok(last_housekeeping_time) => {
+                            let next_housekeeping_time =
+                                last_housekeeping_time.saturating_add(60 * 60 * 24);
+                            if next_housekeeping_time <= time() {
+                                sql::housekeeping(&ctx).await.ok_or_log(&ctx);
+                            }
+                        }
+                        Err(err) => {
+                            warn!(ctx, "Failed to get last housekeeping time: {}", err);
+                        }
+                    };
+
                     info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await;
                 }
             }
@@ -121,36 +135,6 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         })
         .race(fut)
         .await;
-    shutdown_sender
-        .send(())
-        .await
-        .expect("inbox loop, missing shutdown receiver");
-}
-
-async fn fetch(ctx: &Context, connection: &mut Imap) {
-    match ctx.get_config(Config::ConfiguredInboxFolder).await {
-        Ok(Some(watch_folder)) => {
-            if let Err(err) = connection.prepare(ctx).await {
-                warn!(ctx, "Could not connect: {}", err);
-                return;
-            }
-
-            // fetch
-            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
-                connection.trigger_reconnect(ctx).await;
-                warn!(ctx, "{:#}", err);
-            }
-        }
-        Ok(None) => {
-            info!(ctx, "Can not fetch inbox folder, not set");
-        }
-        Err(err) => {
-            warn!(
-                ctx,
-                "Can not fetch inbox folder, failed to get config: {:?}", err
-            );
-        }
-    }
 }
 
 async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
@@ -162,7 +146,30 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 return connection.fake_idle(ctx, Some(watch_folder)).await;
             }
 
-            // Mark expired messages for deletion.
+            if folder == Config::ConfiguredInboxFolder {
+                if let Err(err) = connection
+                    .store_seen_flags_on_imap(ctx)
+                    .await
+                    .context("store_seen_flags_on_imap failed")
+                {
+                    warn!(ctx, "{:#}", err);
+                }
+            }
+
+            // Fetch the watched folder.
+            if let Err(err) = connection
+                .fetch_move_delete(ctx, &watch_folder, false)
+                .await
+            {
+                connection.trigger_reconnect(ctx).await;
+                warn!(ctx, "{:#}", err);
+                return InterruptInfo::new(false);
+            }
+
+            // Mark expired messages for deletion. Marked messages will be deleted from the server
+            // on the next iteration of `fetch_move_delete`. `delete_expired_imap_messages` is not
+            // called right before `fetch_move_delete` because it is not well optimized and would
+            // otherwise slow down message fetching.
             if let Err(err) = delete_expired_imap_messages(ctx)
                 .await
                 .context("delete_expired_imap_messages failed")
@@ -170,13 +177,6 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 warn!(ctx, "{:#}", err);
             }
 
-            // Fetch the watched folder.
-            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
-                connection.trigger_reconnect(ctx).await;
-                warn!(ctx, "{:#}", err);
-                return InterruptInfo::new(false);
-            }
-
             // Scan additional folders only after finishing fetching the watched folder.
             //
             // On iOS the application has strictly limited time to work in background, so we may not
@@ -196,7 +196,10 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                         // In most cases this will select the watched folder and return because there are
                         // no new messages. We want to select the watched folder anyway before going IDLE
                         // there, so this does not take additional protocol round-trip.
-                        if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
+                        if let Err(err) = connection
+                            .fetch_move_delete(ctx, &watch_folder, false)
+                            .await
+                        {
                             connection.trigger_reconnect(ctx).await;
                             warn!(ctx, "{:#}", err);
                             return InterruptInfo::new(false);
@@ -256,17 +259,16 @@ async fn simple_imap_loop(
     let ImapConnectionHandlers {
         mut connection,
         stop_receiver,
-        shutdown_sender,
     } = inbox_handlers;
 
     let ctx1 = ctx.clone();
 
     let fut = async move {
-        started
-            .send(())
-            .await
-            .expect("simple imap loop, missing started receive");
         let ctx = ctx1;
+        if let Err(err) = started.send(()).await {
+            warn!(&ctx, "simple imap loop, missing started receiver: {}", err);
+            return;
+        }
 
         loop {
             fetch_idle(&ctx, &mut connection, folder).await;
@@ -280,10 +282,6 @@ async fn simple_imap_loop(
         })
         .race(fut)
         .await;
-    shutdown_sender
-        .send(())
-        .await
-        .expect("simple imap loop, missing shutdown receiver");
 }
 
 async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnectionHandlers) {
@@ -293,18 +291,18 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
     let SmtpConnectionHandlers {
         mut connection,
         stop_receiver,
-        shutdown_sender,
         idle_interrupt_receiver,
     } = smtp_handlers;
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started
-            .send(())
-            .await
-            .expect("smtp loop, missing started receiver");
         let ctx = ctx1;
+        if let Err(err) = started.send(()).await {
+            warn!(&ctx, "smtp loop, missing started receiver: {}", err);
+            return;
+        }
 
+        let mut timeout = None;
         let mut interrupt_info = Default::default();
         loop {
             let job = match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
@@ -322,9 +320,16 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
                     interrupt_info = Default::default();
                 }
                 None => {
-                    if let Err(err) = send_smtp_messages(&ctx, &mut connection).await {
+                    let res = send_smtp_messages(&ctx, &mut connection).await;
+                    if let Err(err) = &res {
                         warn!(ctx, "send_smtp_messages failed: {:#}", err);
                     }
+                    let success = res.unwrap_or(false);
+                    timeout = if success {
+                        None
+                    } else {
+                        Some(timeout.map_or(30, |timeout: u64| timeout.saturating_mul(3)))
+                    };
 
                     // Fake Idle
                     info!(ctx, "smtp fake idle - started");
@@ -333,7 +338,27 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
                         Some(err) => connection.connectivity.set_err(&ctx, err).await,
                     }
 
-                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    // If send_smtp_messages() failed, we set a timeout for the fake-idle so that
+                    // sending is retried (at the latest) after the timeout. If sending fails
+                    // again, we increase the timeout exponentially, in order not to do lots of
+                    // unnecessary retries.
+                    if let Some(timeout) = timeout {
+                        info!(
+                            ctx,
+                            "smtp has messages to retry, planning to retry {} seconds later",
+                            timeout
+                        );
+                        let duration = std::time::Duration::from_secs(timeout);
+                        interrupt_info = async_std::future::timeout(duration, async {
+                            idle_interrupt_receiver.recv().await.unwrap_or_default()
+                        })
+                        .await
+                        .unwrap_or_default();
+                    } else {
+                        info!(ctx, "smtp has no messages to retry, waiting for interrupt");
+                        interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
+                    };
+
                     info!(ctx, "smtp fake idle - interrupted")
                 }
             }
@@ -347,15 +372,15 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
         })
         .race(fut)
         .await;
-    shutdown_sender
-        .send(())
-        .await
-        .expect("smtp loop, missing shutdown receiver");
 }
 
 impl Scheduler {
-    /// Start the scheduler, panics if it is already running.
+    /// Start the scheduler, returns error if it is already running.
     pub async fn start(&mut self, ctx: Context) -> Result<()> {
+        if self.is_running() {
+            bail!("scheduler is already stopped");
+        }
+
         let (mvbox, mvbox_handlers) = ImapConnectionState::new(&ctx).await?;
         let (sentbox, sentbox_handlers) = ImapConnectionState::new(&ctx).await?;
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
@@ -367,6 +392,8 @@ impl Scheduler {
         let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1);
         let mut sentbox_handle = None;
         let (smtp_start_send, smtp_start_recv) = channel::bounded(1);
+        let (ephemeral_interrupt_send, ephemeral_interrupt_recv) = channel::bounded(1);
+        let (location_interrupt_send, location_interrupt_recv) = channel::bounded(1);
 
         let inbox_handle = {
             let ctx = ctx.clone();
@@ -390,7 +417,7 @@ impl Scheduler {
             mvbox_start_send
                 .send(())
                 .await
-                .expect("mvbox start send, missing receiver");
+                .context("mvbox start send, missing receiver")?;
             mvbox_handlers
                 .connection
                 .connectivity
@@ -413,7 +440,7 @@ impl Scheduler {
             sentbox_start_send
                 .send(())
                 .await
-                .expect("sentbox start send, missing receiver");
+                .context("sentbox start send, missing receiver")?;
             sentbox_handlers
                 .connection
                 .connectivity
@@ -428,6 +455,20 @@ impl Scheduler {
             }))
         };
 
+        let ephemeral_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                ephemeral::ephemeral_loop(&ctx, ephemeral_interrupt_recv).await;
+            }))
+        };
+
+        let location_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                location::location_loop(&ctx, location_interrupt_recv).await;
+            }))
+        };
+
         *self = Scheduler::Running {
             inbox,
             mvbox,
@@ -437,6 +478,10 @@ impl Scheduler {
             mvbox_handle,
             sentbox_handle,
             smtp_handle,
+            ephemeral_handle,
+            ephemeral_interrupt_send,
+            location_handle,
+            location_interrupt_send,
         };
 
         // wait for all loops to be started
@@ -502,11 +547,31 @@ impl Scheduler {
         }
     }
 
-    /// Halts the scheduler, must be called first, and then `stop`.
-    pub(crate) async fn pre_stop(&self) -> StopToken {
+    async fn interrupt_ephemeral_task(&self) {
+        if let Scheduler::Running {
+            ref ephemeral_interrupt_send,
+            ..
+        } = self
+        {
+            ephemeral_interrupt_send.try_send(()).ok();
+        }
+    }
+
+    async fn interrupt_location(&self) {
+        if let Scheduler::Running {
+            ref location_interrupt_send,
+            ..
+        } = self
+        {
+            location_interrupt_send.try_send(()).ok();
+        }
+    }
+
+    /// Halt the scheduler.
+    pub(crate) async fn stop(&mut self) -> Result<()> {
         match self {
             Scheduler::Stopped => {
-                panic!("WARN: already stopped");
+                bail!("scheduler is already stopped");
             }
             Scheduler::Running {
                 inbox,
@@ -517,39 +582,23 @@ impl Scheduler {
                 sentbox_handle,
                 smtp,
                 smtp_handle,
+                ephemeral_handle,
+                location_handle,
                 ..
             } => {
                 if inbox_handle.is_some() {
-                    inbox.stop().await;
+                    inbox.stop().await?;
                 }
                 if mvbox_handle.is_some() {
-                    mvbox.stop().await;
+                    mvbox.stop().await?;
                 }
                 if sentbox_handle.is_some() {
-                    sentbox.stop().await;
+                    sentbox.stop().await?;
                 }
                 if smtp_handle.is_some() {
-                    smtp.stop().await;
+                    smtp.stop().await?;
                 }
 
-                StopToken
-            }
-        }
-    }
-
-    /// Halt the scheduler, must only be called after pre_stop.
-    pub(crate) async fn stop(&mut self, _t: StopToken) {
-        match self {
-            Scheduler::Stopped => {
-                panic!("WARN: already stopped");
-            }
-            Scheduler::Running {
-                inbox_handle,
-                mvbox_handle,
-                sentbox_handle,
-                smtp_handle,
-                ..
-            } => {
                 if let Some(handle) = inbox_handle.take() {
                     handle.await;
                 }
@@ -562,8 +611,15 @@ impl Scheduler {
                 if let Some(handle) = smtp_handle.take() {
                     handle.await;
                 }
+                if let Some(handle) = ephemeral_handle.take() {
+                    handle.cancel().await;
+                }
+                if let Some(handle) = location_handle.take() {
+                    handle.cancel().await;
+                }
 
                 *self = Scheduler::Stopped;
+                Ok(())
             }
         }
     }
@@ -577,8 +633,6 @@ impl Scheduler {
 /// Connection state logic shared between imap and smtp connections.
 #[derive(Debug)]
 struct ConnectionState {
-    /// Channel to notify that shutdown has completed.
-    shutdown_receiver: Receiver<()>,
     /// Channel to interrupt the whole connection.
     stop_sender: Sender<()>,
     /// Channel to interrupt idle.
@@ -589,14 +643,13 @@ struct ConnectionState {
 
 impl ConnectionState {
     /// Shutdown this connection completely.
-    async fn stop(&self) {
+    async fn stop(&self) -> Result<()> {
         // Trigger shutdown of the run loop.
         self.stop_sender
             .send(())
             .await
-            .expect("stop, missing receiver");
-        // Wait for a notification that the run loop has been shutdown.
-        self.shutdown_receiver.recv().await.ok();
+            .context("failed to stop, missing receiver")?;
+        Ok(())
     }
 
     async fn interrupt(&self, info: InterruptInfo) {
@@ -613,18 +666,15 @@ pub(crate) struct SmtpConnectionState {
 impl SmtpConnectionState {
     fn new() -> (Self, SmtpConnectionHandlers) {
         let (stop_sender, stop_receiver) = channel::bounded(1);
-        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
         let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = SmtpConnectionHandlers {
             connection: Smtp::new(),
             stop_receiver,
-            shutdown_sender,
             idle_interrupt_receiver,
         };
 
         let state = ConnectionState {
-            shutdown_receiver,
             stop_sender,
             idle_interrupt_sender,
             connectivity: handlers.connection.connectivity.clone(),
@@ -641,15 +691,15 @@ impl SmtpConnectionState {
     }
 
     /// Shutdown this connection completely.
-    async fn stop(&self) {
-        self.state.stop().await;
+    async fn stop(&self) -> Result<()> {
+        self.state.stop().await?;
+        Ok(())
     }
 }
 
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,
-    shutdown_sender: Sender<()>,
     idle_interrupt_receiver: Receiver<InterruptInfo>,
 }
 
@@ -662,17 +712,14 @@ impl ImapConnectionState {
     /// Construct a new connection.
     async fn new(context: &Context) -> Result<(Self, ImapConnectionHandlers)> {
         let (stop_sender, stop_receiver) = channel::bounded(1);
-        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
         let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = ImapConnectionHandlers {
             connection: Imap::new_configured(context, idle_interrupt_receiver).await?,
             stop_receiver,
-            shutdown_sender,
         };
 
         let state = ConnectionState {
-            shutdown_receiver,
             stop_sender,
             idle_interrupt_sender,
             connectivity: handlers.connection.connectivity.clone(),
@@ -689,8 +736,9 @@ impl ImapConnectionState {
     }
 
     /// Shutdown this connection completely.
-    async fn stop(&self) {
-        self.state.stop().await;
+    async fn stop(&self) -> Result<()> {
+        self.state.stop().await?;
+        Ok(())
     }
 }
 
@@ -698,7 +746,6 @@ impl ImapConnectionState {
 struct ImapConnectionHandlers {
     connection: Imap,
     stop_receiver: Receiver<()>,
-    shutdown_sender: Sender<()>,
 }
 
 #[derive(Default, Debug)]

src/scheduler/connectivity.rs

@@ -304,7 +304,7 @@ impl Context {
             <html>
             <head>
                 <meta charset="UTF-8" />
-                <meta name="viewport" content="initial-scale=1.0" />
+                <meta name="viewport" content="initial-scale=1.0; user-scalable=no" />
                 <style>
                     ul {
                         list-style-type: none;
@@ -449,13 +449,7 @@ impl Context {
         //                                [======67%=====       ]
         // =============================================================================================
 
-        let domain = dc_tools::EmailAddress::new(
-            &self
-                .get_config(Config::ConfiguredAddr)
-                .await?
-                .unwrap_or_default(),
-        )?
-        .domain;
+        let domain = dc_tools::EmailAddress::new(&self.get_primary_self_addr().await?)?.domain;
         ret += &format!(
             "<h3>{}</h3><ul>",
             stock_str::storage_on_domain(self, domain).await
@@ -544,7 +538,7 @@ impl Context {
                 self.schedule_quota_update().await?;
             }
         } else {
-            ret += &format!("<li>{}</li>", stock_str::one_moment(self).await);
+            ret += &format!("<li>{}</li>", stock_str::not_connected(self).await);
             self.schedule_quota_update().await?;
         }
         ret += "</ul>";

src/securejoin.rs

@@ -8,7 +8,7 @@ use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat, ChatId, ChatIdBlocked};
 use crate::config::Config;
-use crate::constants::{Blocked, DC_CONTACT_ID_LAST_SPECIAL};
+use crate::constants::Blocked;
 use crate::contact::{Contact, ContactId, Origin, VerifiedStatus};
 use crate::context::Context;
 use crate::dc_tools::time;
@@ -66,19 +66,7 @@ pub async fn dc_get_securejoin_qr(context: &Context, group: Option<ChatId>) -> R
         .is_none();
     let invitenumber = token::lookup_or_new(context, Namespace::InviteNumber, group).await;
     let auth = token::lookup_or_new(context, Namespace::Auth, group).await;
-    let self_addr = match context.get_config(Config::ConfiguredAddr).await {
-        Ok(Some(addr)) => addr,
-        Ok(None) => {
-            bail!("Not configured, cannot generate QR code.");
-        }
-        Err(err) => {
-            bail!(
-                "Unable to retrieve configuration, cannot generate QR code: {:?}",
-                err
-            );
-        }
-    };
-
+    let self_addr = context.get_primary_self_addr().await?;
     let self_name = context
         .get_config(Config::Displayname)
         .await?
@@ -99,6 +87,12 @@ pub async fn dc_get_securejoin_qr(context: &Context, group: Option<ChatId>) -> R
     let qr = if let Some(group) = group {
         // parameters used: a=g=x=i=s=
         let chat = Chat::load_from_db(context, group).await?;
+        if chat.grpid.is_empty() {
+            bail!(
+                "can't generate securejoin QR code for ad-hoc group {}",
+                group
+            );
+        }
         let group_name = chat.get_name();
         let group_name_urlencoded = utf8_percent_encode(group_name, NON_ALPHANUMERIC).to_string();
         if sync_token {
@@ -310,7 +304,7 @@ pub(crate) async fn handle_securejoin_handshake(
     mime_message: &MimeMessage,
     contact_id: ContactId,
 ) -> Result<HandshakeMessage> {
-    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+    if contact_id.is_special() {
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
@@ -573,7 +567,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
     mime_message: &MimeMessage,
     contact_id: ContactId,
 ) -> Result<HandshakeMessage> {
-    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+    if contact_id.is_special() {
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
@@ -722,6 +716,7 @@ mod tests {
     use crate::chat::ProtectionStatus;
     use crate::chatlist::Chatlist;
     use crate::constants::Chattype;
+    use crate::dc_receive_imf::dc_receive_imf;
     use crate::peerstate::Peerstate;
     use crate::test_utils::{TestContext, TestContextManager};
 
@@ -1320,4 +1315,25 @@ mod tests {
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_adhoc_group_no_qr() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("2")).await?;
+
+        let mime = br#"Subject: First thread
+Message-ID: first@example.org
+To: Alice <alice@example.org>, Bob <bob@example.net>
+From: Claire <claire@example.org>
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+First thread."#;
+
+        dc_receive_imf(&alice, mime, false).await?;
+        let msg = alice.get_last_msg().await;
+        let chat_id = msg.chat_id;
+
+        assert!(dc_get_securejoin_qr(&alice, Some(chat_id)).await.is_err());
+        Ok(())
+    }
 }

src/smtp.rs

@@ -4,14 +4,14 @@ pub mod send;
 
 use std::time::{Duration, SystemTime};
 
-use anyhow::{bail, format_err, Context as _, Result};
+use anyhow::{bail, format_err, Context as _, Error, Result};
 use async_smtp::smtp::client::net::ClientTlsParameters;
 use async_smtp::smtp::response::{Category, Code, Detail};
 use async_smtp::{smtp, EmailAddress, ServerAddress};
+use async_std::task;
 
 use crate::constants::DC_LP_AUTH_OAUTH2;
 use crate::events::EventType;
-use crate::job::Status;
 use crate::login_param::{
     dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam, Socks5Config,
 };
@@ -50,7 +50,10 @@ impl Smtp {
     /// Disconnect the SMTP transport and drop it entirely.
     pub async fn disconnect(&mut self) {
         if let Some(mut transport) = self.transport.take() {
-            transport.close().await.ok();
+            // Closing connection with a QUIT command may take some time, especially if it's a
+            // stale connection and an attempt to send the command times out. Send a command in a
+            // separate task to avoid waiting for reply or timeout.
+            task::spawn(async move { transport.close().await });
         }
         self.last_success = None;
     }
@@ -84,7 +87,7 @@ impl Smtp {
         }
 
         self.connectivity.set_connecting(context).await;
-        let lp = LoginParam::from_database(context, "configured_").await?;
+        let lp = LoginParam::load_configured_params(context).await?;
         self.connect(
             context,
             &lp.smtp,
@@ -196,12 +199,18 @@ impl Smtp {
     }
 }
 
+pub(crate) enum SendResult {
+    /// Message was sent successfully.
+    Success,
+
+    /// Permanent error, message sending has failed.
+    Failure(Error),
+
+    /// Temporary error, the message should be retried later.
+    Retry,
+}
+
 /// Tries to send a message.
-///
-/// Returns Status::Finished if sending the message should not be retried anymore,
-/// Status::RetryLater if sending should be postponed and Status::RetryNow if it is suspected that
-/// temporary failure is caused by stale connection, in which case a second attempt to send the
-/// same message may be done immediately.
 pub(crate) async fn smtp_send(
     context: &Context,
     recipients: &[async_smtp::EmailAddress],
@@ -209,7 +218,7 @@ pub(crate) async fn smtp_send(
     smtp: &mut Smtp,
     msg_id: MsgId,
     rowid: i64,
-) -> Status {
+) -> SendResult {
     if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
         info!(context, "smtp-sending out mime message:");
         println!("{}", message);
@@ -217,6 +226,20 @@ pub(crate) async fn smtp_send(
 
     smtp.connectivity.set_working(context).await;
 
+    if smtp.has_maybe_stale_connection().await {
+        info!(context, "Closing stale connection");
+        smtp.disconnect().await;
+
+        if let Err(err) = smtp
+            .connect_configured(context)
+            .await
+            .context("failed to reopen stale SMTP connection")
+        {
+            smtp.last_send_error = Some(format!("{:#}", err));
+            return SendResult::Retry;
+        }
+    }
+
     let send_result = smtp
         .send(context, recipients, message.as_bytes(), rowid)
         .await;
@@ -253,14 +276,14 @@ pub(crate) async fn smtp_send(
 
                     if maybe_transient {
                         info!(context, "Permanent error that is likely to actually be transient, postponing retry for later");
-                        Status::RetryLater
+                        SendResult::Retry
                     } else {
                         info!(context, "Permanent error, message sending failed");
                         // If we do not retry, add an info message to the chat.
                         // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
                         // should definitely go here, because user has to open the link to
                         // resume message sending.
-                        Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                        SendResult::Failure(format_err!("Permanent SMTP error: {}", err))
                     }
                 }
                 async_smtp::smtp::error::Error::Transient(ref response) => {
@@ -277,35 +300,29 @@ pub(crate) async fn smtp_send(
                             // receive as a transient error are misconfigurations of the smtp server.
                             // See <https://tools.ietf.org/html/rfc3463#section-3.2>
                             info!(context, "Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", first_word);
-                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            SendResult::Failure(format_err!("Permanent SMTP error: {}", err))
                         } else {
                             info!(
                                 context,
                                 "Transient error with status code {}, postponing retry for later",
                                 first_word
                             );
-                            Status::RetryLater
+                            SendResult::Retry
                         }
                     } else {
                         info!(
                             context,
                             "Transient error without status code, postponing retry for later"
                         );
-                        Status::RetryLater
+                        SendResult::Retry
                     }
                 }
                 _ => {
                     info!(
                         context,
-                        "Message sending failed without error returned by the server"
+                        "Message sending failed without error returned by the server, retry later"
                     );
-                    if smtp.has_maybe_stale_connection().await {
-                        info!(context, "Connection is probably stale, retry immediately");
-                        Status::RetryNow
-                    } else {
-                        info!(context, "Connection is not stale, retry later");
-                        Status::RetryLater
-                    }
+                    SendResult::Retry
                 }
             };
 
@@ -319,24 +336,24 @@ pub(crate) async fn smtp_send(
             // Local error, job is invalid, do not retry.
             smtp.disconnect().await;
             warn!(context, "SMTP job is invalid: {}", err);
-            Status::Finished(Err(err.into()))
+            SendResult::Failure(err.into())
         }
         Err(crate::smtp::send::Error::NoTransport) => {
             // Should never happen.
             // It does not even make sense to disconnect here.
             error!(context, "SMTP job failed because SMTP has no transport");
-            Status::Finished(Err(format_err!("SMTP has not transport")))
+            SendResult::Failure(format_err!("SMTP has not transport"))
         }
         Err(crate::smtp::send::Error::Other(err)) => {
             // Local error, job is invalid, do not retry.
             smtp.disconnect().await;
             warn!(context, "unable to load job: {}", err);
-            Status::Finished(Err(err))
+            SendResult::Failure(err)
         }
-        Ok(()) => Status::Finished(Ok(())),
+        Ok(()) => SendResult::Success,
     };
 
-    if let Status::Finished(Err(err)) = &status {
+    if let SendResult::Failure(err) = &status {
         // We couldn't send the message, so mark it as failed
         message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
     }
@@ -432,7 +449,7 @@ pub(crate) async fn send_msg_to_smtp(
         return Ok(());
     }
 
-    let status = match smtp_send(
+    let status = smtp_send(
         context,
         &recipients_list,
         body.as_str(),
@@ -440,47 +457,25 @@ pub(crate) async fn send_msg_to_smtp(
         msg_id,
         rowid,
     )
-    .await
-    {
-        Status::RetryNow => {
-            // Do a single retry immediately without increasing retry counter in case of stale
-            // connection.
-            info!(context, "Doing immediate retry to send message.");
+    .await;
 
-            // smtp_send just closed stale SMTP connection, reconnect and try again.
-            if let Err(err) = smtp
-                .connect_configured(context)
-                .await
-                .context("failed to reopen stale SMTP connection")
-            {
-                smtp.last_send_error = Some(format!("{:#}", err));
-                return Err(err);
-            }
-
-            smtp_send(
-                context,
-                &recipients_list,
-                body.as_str(),
-                smtp,
-                msg_id,
-                rowid,
-            )
-            .await
-        }
-        status => status,
-    };
     match status {
-        Status::Finished(res) => {
+        SendResult::Retry => {}
+        SendResult::Success | SendResult::Failure(_) => {
             context
                 .sql
                 .execute("DELETE FROM smtp WHERE id=?", paramsv![rowid])
                 .await?;
-            if res.is_ok() {
-                msg_id.set_delivered(context).await?;
-            }
-            res
         }
-        Status::RetryNow | Status::RetryLater => Err(format_err!("Retry")),
+    };
+
+    match status {
+        SendResult::Retry => Err(format_err!("Retry")),
+        SendResult::Success => {
+            msg_id.set_delivered(context).await?;
+            Ok(())
+        }
+        SendResult::Failure(err) => Err(format_err!("{}", err)),
     }
 }
 
@@ -488,10 +483,9 @@ pub(crate) async fn send_msg_to_smtp(
 ///
 /// Logs and ignores SMTP errors to ensure that a single SMTP message constantly failing to be sent
 /// does not block other messages in the queue from being sent.
-pub(crate) async fn send_smtp_messages(
-    context: &Context,
-    connection: &mut Smtp,
-) -> anyhow::Result<()> {
+///
+/// Returns true if all messages were sent successfully, false otherwise.
+pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp) -> Result<bool> {
     context.send_sync_msg().await?; // Add sync message to the end of the queue if needed.
     let rowids = context
         .sql
@@ -509,10 +503,12 @@ pub(crate) async fn send_smtp_messages(
             },
         )
         .await?;
+    let mut success = true;
     for rowid in rowids {
         if let Err(err) = send_msg_to_smtp(context, connection, rowid).await {
             info!(context, "Failed to send message over SMTP: {:#}.", err);
+            success = false;
         }
     }
-    Ok(())
+    Ok(success)
 }

src/sql.rs

@@ -34,6 +34,17 @@ macro_rules! paramsv {
     };
 }
 
+#[macro_export]
+macro_rules! params_iterv {
+    ($($param:expr),+ $(,)?) => {
+        vec![$(&$param as &dyn $crate::ToSql),+]
+    };
+}
+
+pub(crate) fn params_iter(iter: &[impl crate::ToSql]) -> impl Iterator<Item = &dyn crate::ToSql> {
+    iter.iter().map(|item| item as &dyn crate::ToSql)
+}
+
 mod migrations;
 
 /// A wrapper around the underlying Sqlite3 object.
@@ -181,6 +192,7 @@ impl Sql {
                      PRAGMA secure_delete=on;
                      PRAGMA busy_timeout = {};
                      PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
+                     PRAGMA foreign_keys=on;
                      ",
                     Duration::from_secs(10).as_millis()
                 ))?;
@@ -599,10 +611,55 @@ impl Sql {
 }
 
 pub async fn housekeeping(context: &Context) -> Result<()> {
-    if let Err(err) = crate::ephemeral::delete_expired_messages(context).await {
-        warn!(context, "Failed to delete expired messages: {}", err);
+    if let Err(err) = remove_unused_files(context).await {
+        warn!(
+            context,
+            "Housekeeping: cannot remove unusued files: {}", err
+        );
     }
 
+    if let Err(err) = start_ephemeral_timers(context).await {
+        warn!(
+            context,
+            "Housekeeping: cannot start ephemeral timers: {}", err
+        );
+    }
+
+    if let Err(err) = prune_tombstones(&context.sql).await {
+        warn!(
+            context,
+            "Housekeeping: Cannot prune message tombstones: {}", err
+        );
+    }
+
+    if let Err(err) = deduplicate_peerstates(&context.sql).await {
+        warn!(context, "Failed to deduplicate peerstates: {}", err)
+    }
+
+    context.schedule_quota_update().await?;
+
+    // Try to clear the freelist to free some space on the disk. This
+    // only works if auto_vacuum is enabled.
+    if let Err(err) = context
+        .sql
+        .execute("PRAGMA incremental_vacuum", paramsv![])
+        .await
+    {
+        warn!(context, "Failed to run incremental vacuum: {}", err);
+    }
+
+    if let Err(e) = context
+        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
+        .await
+    {
+        warn!(context, "Can't set config: {}", e);
+    }
+
+    info!(context, "Housekeeping done.");
+    Ok(())
+}
+
+pub async fn remove_unused_files(context: &Context) -> Result<()> {
     let mut files_in_use = HashSet::new();
     let mut unreferenced_count = 0;
 
@@ -717,44 +774,6 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         }
     }
 
-    if let Err(err) = start_ephemeral_timers(context).await {
-        warn!(
-            context,
-            "Housekeeping: cannot start ephemeral timers: {}", err
-        );
-    }
-
-    if let Err(err) = prune_tombstones(&context.sql).await {
-        warn!(
-            context,
-            "Housekeeping: Cannot prune message tombstones: {}", err
-        );
-    }
-
-    if let Err(err) = deduplicate_peerstates(&context.sql).await {
-        warn!(context, "Failed to deduplicate peerstates: {}", err)
-    }
-
-    context.schedule_quota_update().await?;
-
-    // Try to clear the freelist to free some space on the disk. This
-    // only works if auto_vacuum is enabled.
-    if let Err(err) = context
-        .sql
-        .execute("PRAGMA incremental_vacuum", paramsv![])
-        .await
-    {
-        warn!(context, "Failed to run incremental vacuum: {}", err);
-    }
-
-    if let Err(e) = context
-        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
-        .await
-    {
-        warn!(context, "Can't set config: {}", e);
-    }
-
-    info!(context, "Housekeeping done.");
     Ok(())
 }
 
@@ -824,13 +843,10 @@ async fn prune_tombstones(sql: &Sql) -> Result<()> {
 ///
 /// Use this together with [`rusqlite::ParamsFromIter`] to use dynamically generated
 /// parameter lists.
-pub fn repeat_vars(count: usize) -> Result<String> {
-    if count == 0 {
-        bail!("Must have at least one repeat variable");
-    }
+pub fn repeat_vars(count: usize) -> String {
     let mut s = "?,".repeat(count);
     s.pop(); // Remove trailing comma
-    Ok(s)
+    s
 }
 
 #[cfg(test)]

src/sql/migrations.rs

@@ -395,7 +395,7 @@ UPDATE chats SET protected=1, type=120 WHERE type=130;"#,
 
     if dbversion < 71 {
         info!(context, "[migration] v71");
-        if let Some(addr) = context.get_config(Config::ConfiguredAddr).await? {
+        if let Ok(addr) = context.get_primary_self_addr().await {
             if let Ok(domain) = addr.parse::<EmailAddress>().map(|email| email.domain) {
                 context
                     .set_config(
@@ -608,6 +608,22 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid);
         )
         .await?;
     }
+    if dbversion < 88 {
+        info!(context, "[migration] v88");
+        sql.execute_migration("DROP TABLE IF EXISTS backup_blobs;", 88)
+            .await?;
+    }
+    if dbversion < 89 {
+        info!(context, "[migration] v89");
+        sql.execute_migration(
+            r#"CREATE TABLE imap_markseen (
+              id INTEGER,
+              FOREIGN KEY(id) REFERENCES imap(id) ON DELETE CASCADE
+            );"#,
+            89,
+        )
+        .await?;
+    }
 
     Ok((
         recalc_fingerprints,

src/stock_str.rs

@@ -10,7 +10,6 @@ use strum_macros::EnumProperty;
 use crate::blob::BlobObject;
 use crate::chat::{self, Chat, ChatId, ProtectionStatus};
 use crate::config::Config;
-use crate::constants::DC_CONTACT_ID_SELF;
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::dc_tools::dc_timestamp_to_str;
@@ -288,9 +287,6 @@ pub enum StockMessage {
     #[strum(props(fallback = "Storage on %1$s"))]
     StorageOnDomain = 105,
 
-    #[strum(props(fallback = "One moment…"))]
-    OneMoment = 106,
-
     #[strum(props(fallback = "Connected"))]
     Connected = 107,
 
@@ -333,6 +329,9 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Scan to join group %1$s"))]
     SecureJoinGroupQRDescription = 120,
+
+    #[strum(props(fallback = "Not connected"))]
+    NotConnected = 121,
 }
 
 impl StockMessage {
@@ -397,7 +396,7 @@ trait StockStringMods: AsRef<str> + Sized {
         Box::pin(async move {
             let message = self.as_ref().trim_end_matches('.');
             match contact_id {
-                DC_CONTACT_ID_SELF => msg_action_by_me(context, message).await,
+                ContactId::SELF => msg_action_by_me(context, message).await,
                 _ => {
                     let displayname = Contact::get_by_id(context, contact_id)
                         .await
@@ -1010,9 +1009,9 @@ pub(crate) async fn storage_on_domain(context: &Context, domain: impl AsRef<str>
         .replace1(domain)
 }
 
-/// Stock string: `One moment…`.
-pub(crate) async fn one_moment(context: &Context) -> String {
-    translated(context, StockMessage::OneMoment).await
+/// Stock string: `Not connected`.
+pub(crate) async fn not_connected(context: &Context) -> String {
+    translated(context, StockMessage::NotConnected).await
 }
 
 /// Stock string: `Connected`.
@@ -1129,7 +1128,7 @@ impl Context {
             self.sql
                 .set_raw_config_bool("self-chat-added", true)
                 .await?;
-            ChatId::create_for_contact(self, DC_CONTACT_ID_SELF).await?;
+            ChatId::create_for_contact(self, ContactId::SELF).await?;
         }
 
         // add welcome-messages. by the label, this is done only once,
@@ -1149,14 +1148,13 @@ impl Context {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use crate::test_utils::TestContext;
-
-    use crate::constants::DC_CONTACT_ID_SELF;
+    use num_traits::ToPrimitive;
 
     use crate::chat::Chat;
     use crate::chatlist::Chatlist;
-    use num_traits::ToPrimitive;
+    use crate::test_utils::TestContext;
+
+    use super::*;
 
     #[test]
     fn test_enum_mapping() {
@@ -1233,7 +1231,7 @@ mod tests {
     async fn test_stock_system_msg_add_member_by_me() {
         let t = TestContext::new().await;
         assert_eq!(
-            msg_add_member(&t, "alice@example.org", DC_CONTACT_ID_SELF).await,
+            msg_add_member(&t, "alice@example.org", ContactId::SELF).await,
             "Member alice@example.org added by me."
         )
     }
@@ -1245,7 +1243,7 @@ mod tests {
             .await
             .expect("failed to create contact");
         assert_eq!(
-            msg_add_member(&t, "alice@example.org", DC_CONTACT_ID_SELF).await,
+            msg_add_member(&t, "alice@example.org", ContactId::SELF).await,
             "Member Alice (alice@example.org) added by me."
         );
     }

src/summary.rs

@@ -1,8 +1,8 @@
 //! # Message summary for chatlist.
 
 use crate::chat::Chat;
-use crate::constants::{Chattype, DC_CONTACT_ID_SELF};
-use crate::contact::Contact;
+use crate::constants::Chattype;
+use crate::contact::{Contact, ContactId};
 use crate::context::Context;
 use crate::dc_tools::dc_truncate;
 use crate::message::{Message, MessageState, Viewtype};
@@ -60,7 +60,7 @@ impl Summary {
     ) -> Self {
         let prefix = if msg.state == MessageState::OutDraft {
             Some(SummaryPrefix::Draft(stock_str::draft(context).await))
-        } else if msg.from_id == DC_CONTACT_ID_SELF {
+        } else if msg.from_id == ContactId::SELF {
             if msg.is_info() || chat.is_self_talk() {
                 None
             } else {

src/sync.rs

@@ -2,7 +2,8 @@
 
 use crate::chat::{Chat, ChatId};
 use crate::config::Config;
-use crate::constants::{Blocked, DC_CONTACT_ID_SELF};
+use crate::constants::Blocked;
+use crate::contact::ContactId;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::message::{Message, MsgId, Viewtype};
@@ -126,7 +127,7 @@ impl Context {
     pub async fn send_sync_msg(&self) -> Result<Option<MsgId>> {
         if let Some((json, ids)) = self.build_sync_json().await? {
             let chat_id =
-                ChatId::create_for_contact_with_blocked(self, DC_CONTACT_ID_SELF, Blocked::Yes)
+                ChatId::create_for_contact_with_blocked(self, ContactId::SELF, Blocked::Yes)
                     .await?;
             let mut msg = Message {
                 chat_id,
@@ -483,7 +484,7 @@ mod tests {
         // check that the used self-talk is not visible to the user
         // but that creation will still work (in this case, the chat is empty)
         assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 0);
-        let chat_id = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
+        let chat_id = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
         let chat = Chat::load_from_db(&alice, chat_id).await?;
         assert!(chat.is_self_talk());
         assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 1);

src/test_utils.rs

@@ -22,8 +22,8 @@ use crate::chat::{self, Chat, ChatId};
 use crate::chatlist::Chatlist;
 use crate::config::Config;
 use crate::constants::Chattype;
-use crate::constants::{DC_CONTACT_ID_SELF, DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
-use crate::contact::{Contact, Modifier, Origin};
+use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
+use crate::contact::{Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_receive_imf::dc_receive_imf;
 use crate::dc_tools::EmailAddress;
@@ -88,6 +88,13 @@ impl TestContextBuilder {
         self.with_key_pair(bob_keypair())
     }
 
+    /// Configures as fiona@example.net with fixed secret key.
+    ///
+    /// This is a shortcut for `.with_key_pair(bob_keypair()).
+    pub fn configure_fiona(self) -> Self {
+        self.with_key_pair(fiona_keypair())
+    }
+
     /// Configures the new [`TestContext`] with the provided [`KeyPair`].
     ///
     /// This will extract the email address from the key and configure the context with the
@@ -183,6 +190,13 @@ impl TestContext {
         Self::builder().configure_bob().build().await
     }
 
+    /// Creates a new configured [`TestContext`].
+    ///
+    /// This is a shortcut which configures fiona@example.net with a fixed key.
+    pub async fn new_fiona() -> Self {
+        Self::builder().configure_fiona().build().await
+    }
+
     /// Internal constructor.
     ///
     /// `name` is used to identify this context in e.g. log output.  This is useful mostly
@@ -364,7 +378,7 @@ impl TestContext {
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n"
                 .to_owned()
                 + msg.payload();
-        dc_receive_imf(&self.ctx, received_msg.as_bytes(), "INBOX", false)
+        dc_receive_imf(&self.ctx, received_msg.as_bytes(), false)
             .await
             .unwrap();
     }
@@ -407,12 +421,7 @@ impl TestContext {
             .await
             .unwrap_or_default()
             .unwrap_or_default();
-        let addr = other
-            .ctx
-            .get_config(Config::ConfiguredAddr)
-            .await
-            .unwrap()
-            .unwrap();
+        let addr = other.ctx.get_primary_self_addr().await.unwrap();
         // MailinglistAddress is the lowest allowed origin, we'd prefer to not modify the
         // origin when creating this contact.
         let (contact_id, modified) =
@@ -466,7 +475,7 @@ impl TestContext {
 
     /// Retrieves the "self" chat.
     pub async fn get_self_chat(&self) -> Chat {
-        let chat_id = ChatId::create_for_contact(self, DC_CONTACT_ID_SELF)
+        let chat_id = ChatId::create_for_contact(self, ContactId::SELF)
             .await
             .unwrap();
         Chat::load_from_db(self, chat_id).await.unwrap()
@@ -696,6 +705,24 @@ pub fn bob_keypair() -> KeyPair {
     }
 }
 
+/// Load a pre-generated keypair for fiona@example.net from disk.
+///
+/// Like [alice_keypair] but a different key and identity.
+pub fn fiona_keypair() -> key::KeyPair {
+    let addr = EmailAddress::new("fiona@example.net").unwrap();
+    let public = key::SignedPublicKey::from_asc(include_str!("../test-data/key/fiona-public.asc"))
+        .unwrap()
+        .0;
+    let secret = key::SignedSecretKey::from_asc(include_str!("../test-data/key/fiona-secret.asc"))
+        .unwrap()
+        .0;
+    key::KeyPair {
+        addr,
+        public,
+        secret,
+    }
+}
+
 /// Utility to help wait for and retrieve events.
 ///
 /// This buffers the events in order they are emitted.  This allows consuming events in
@@ -851,7 +878,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.get_from_id() == DC_CONTACT_ID_SELF {
+        if msg.get_from_id() == ContactId::SELF {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"

src/update_helper.rs

@@ -99,7 +99,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 23:37:57 +0000\n\
                  \n\
                  second message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -113,7 +112,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
                  \n\
                  first message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -143,7 +141,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 01:00:00 +0000\n\
                  \n\
                  first message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -163,7 +160,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 03:00:00 +0000\n\
                  \n\
                  third message\n",
-            "INBOX",
             false,
         )
         .await?;
@@ -179,7 +175,6 @@ mod tests {
                  Date: Sun, 22 Mar 2021 02:00:00 +0000\n\
                  \n\
                  second message\n",
-            "INBOX",
             false,
         )
         .await?;

src/webxdc.rs

@@ -19,6 +19,12 @@ use std::fs::File;
 use std::io::{Read, Seek, SeekFrom};
 use zip::ZipArchive;
 
+/// The current API version.
+/// If `min_api` in manifest.toml is set to a larger value,
+/// the Webxdc's index.html is replaced by an error message.
+/// In the future, that may be useful to avoid new Webxdc being loaded on old Delta Chats.
+const WEBXDC_API_VERSION: u32 = 1;
+
 pub const WEBXDC_SUFFIX: &str = "xdc";
 const WEBXDC_DEFAULT_ICON: &str = "__webxdc__/default-icon.png";
 
@@ -44,6 +50,7 @@ const WEBXDC_RECEIVING_LIMIT: u64 = 4194304;
 #[non_exhaustive]
 struct WebxdcManifest {
     name: Option<String>,
+    min_api: Option<u32>,
 }
 
 /// Parsed information from WebxdcManifest and fallbacks.
@@ -219,6 +226,7 @@ impl Context {
                 info.as_str(),
                 SystemMessage::Unknown,
                 timestamp,
+                None,
                 Some(instance),
             )
             .await?;
@@ -231,10 +239,7 @@ impl Context {
             {
                 instance.param.set(Param::WebxdcSummary, summary);
                 instance.update_param(self).await;
-                self.emit_event(EventType::MsgsChanged {
-                    chat_id: instance.chat_id,
-                    msg_id: instance.id,
-                });
+                self.emit_msgs_changed(instance.chat_id, instance.id);
             }
         }
 
@@ -246,9 +251,14 @@ impl Context {
             )
             .await?;
 
-        self.emit_event(EventType::WebxdcStatusUpdate(instance.id));
+        let status_update_serial = StatusUpdateSerial(u32::try_from(rowid)?);
 
-        Ok(StatusUpdateSerial(u32::try_from(rowid)?))
+        self.emit_event(EventType::WebxdcStatusUpdate {
+            msg_id: instance.id,
+            status_update_serial,
+        });
+
+        Ok(status_update_serial)
     }
 
     /// Sends a status update for an webxdc instance.
@@ -498,6 +508,21 @@ impl Message {
         };
 
         let mut archive = self.get_webxdc_archive(context).await?;
+
+        if name == "index.html" {
+            if let Ok(bytes) = get_blob(&mut archive, "manifest.toml").await {
+                if let Ok(manifest) = parse_webxdc_manifest(&bytes).await {
+                    if let Some(min_api) = manifest.min_api {
+                        if min_api > WEBXDC_API_VERSION {
+                            return Ok(Vec::from(
+                                "<!DOCTYPE html>This Webxdc requires a newer Delta Chat version.",
+                            ));
+                        }
+                    }
+                }
+            }
+        }
+
         get_blob(&mut archive, name).await
     }
 
@@ -510,10 +535,16 @@ impl Message {
             if let Ok(manifest) = parse_webxdc_manifest(&bytes).await {
                 manifest
             } else {
-                WebxdcManifest { name: None }
+                WebxdcManifest {
+                    name: None,
+                    min_api: None,
+                }
             }
         } else {
-            WebxdcManifest { name: None }
+            WebxdcManifest {
+                name: None,
+                min_api: None,
+            }
         };
 
         if let Some(ref name) = manifest.name {
@@ -740,7 +771,6 @@ mod tests {
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/webxdc_good_extension.eml"),
-            "INBOX",
             false,
         )
         .await?;
@@ -751,7 +781,6 @@ mod tests {
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/webxdc_bad_extension.eml"),
-            "INBOX",
             false,
         )
         .await?;
@@ -987,7 +1016,10 @@ mod tests {
             .get_matching(|evt| matches!(evt, EventType::WebxdcStatusUpdate { .. }))
             .await;
         match event {
-            EventType::WebxdcStatusUpdate(msg_id) => {
+            EventType::WebxdcStatusUpdate {
+                msg_id,
+                status_update_serial: _,
+            } => {
                 assert_eq!(msg_id, instance_id);
             }
             _ => unreachable!(),
@@ -1294,6 +1326,38 @@ sth_for_the = "future""#
         )
         .await?;
         assert_eq!(manifest.name, Some("foz".to_string()));
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_parse_webxdc_manifest_min_api() -> Result<()> {
+        let manifest = parse_webxdc_manifest(r#"min_api = 3"#.as_bytes()).await?;
+        assert_eq!(manifest.min_api, Some(3));
+
+        let result = parse_webxdc_manifest(r#"min_api = "1""#.as_bytes()).await;
+        assert!(result.is_err());
+
+        let result = parse_webxdc_manifest(r#"min_api = 1.2"#.as_bytes()).await;
+        assert!(result.is_err());
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_webxdc_min_api_too_large() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "chat").await?;
+        let mut instance = create_webxdc_instance(
+            &t,
+            "with-min-api-1001.xdc",
+            include_bytes!("../test-data/webxdc/with-min-api-1001.xdc"),
+        )
+        .await?;
+        send_msg(&t, chat_id, &mut instance).await?;
+
+        let instance = t.get_last_msg().await;
+        let html = instance.get_webxdc_blob(&t, "index.html").await?;
+        assert!(String::from_utf8_lossy(&*html).contains("requires a newer Delta Chat version"));
 
         Ok(())
     }