fix speed issues with ORDER BY clause

.github/dependabot.yml

@@ -3,7 +3,7 @@ updates:
   - package-ecosystem: "cargo"
     directory: "/"
     schedule:
-      interval: "monthly"
+      interval: "daily"
     commit-message:
       prefix: "cargo"
-    open-pull-requests-limit: 50
+    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples --benches
+          args: --workspace --tests --examples
 
   docs:
     name: Rust doc comments
@@ -77,10 +77,10 @@ jobs:
         include:
           # Currently used Rust version, same as in `rust-toolchain` file.
           - os: ubuntu-latest
-            rust: 1.60.0
+            rust: 1.59.0
             python: 3.9
           - os: windows-latest
-            rust: 1.60.0
+            rust: 1.59.0
             python: false # Python bindings compilation on Windows is not supported.
 
           # Minimum Supported Rust Version = 1.56.0

.github/workflows/dependabot.yml

@@ -1,21 +0,0 @@
-name: Dependabot auto-approve
-on: pull_request
-
-permissions:
-  pull-requests: write
-
-jobs:
-  dependabot:
-    runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
-    steps:
-      - name: Dependabot metadata
-        id: metadata
-        uses: dependabot/fetch-metadata@v1.1.1
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Approve a PR
-        run: gh pr review --approve "$PR_URL"
-        env:
-          PR_URL: ${{github.event.pull_request.html_url}}
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.gitignore

@@ -29,7 +29,3 @@ deltachat-ffi/xml
 
 coverage/
 .DS_Store
-.vscode/launch.json
-python/accounts.txt
-python/all-testaccounts.txt
-tmp/

CHANGELOG.md

@@ -1,44 +1,19 @@
 # Changelog
 
-## 1.78.0
-
-### API-Changes
-- replaced stock string `DC_STR_ONE_MOMENT` by `DC_STR_NOT_CONNECTED` #3222
-- add `dc_resend_msgs()` #3238
-- `dc_provider_new_from_email()` does no longer do an DNS lookup for checking custom domains,
-  this is done by `dc_provider_new_from_email_with_dns()` now #3256
-
-### Changes
-- introduce multiple self addresses with the "configured" address always being the primary one #2896
-- Further improve finding the correct server after logging in #3208
-- `get_connectivity_html()` returns HTML as non-scalable #3213
-- add update-serial to `DC_EVENT_WEBXDC_STATUS_UPDATE` #3215
-- Speed up message receiving via IMAP a bit #3225
-- mark messages as seen on IMAP in batches #3223
-- remove Received: based draft detection heuristic #3230
-- Use pkgconfig for building Python package #2590
-- don't start io on unconfigured context #2664
-- do not assign group IDs to ad-hoc groups #2798
-- dynamic libraries use dylib extension on Darwin #3226
-- refactorings #3217 #3219 #3224 #3235 #3239 #3244 #3254
-- improve documentation #3214 #3220 #3237
-- improve tests and ci #3212 #3233 #3241 #3242 #3252 #3250 #3255 #3260
-
-### Fixes
-- Take `delete_device_after` into account when calculating ephemeral loop timeout #3211 #3221
-- Fix a bug where a blocked contact could send a contact request #3218
-- Make sure, videochat-room-names are always URL-safe #3231
-- Try removing account folder multiple times in case of failure #3229
-- Ignore messages from all spam folders if there are many #3246
-- Hide location-only messages instead of displaying empty bubbles #3248
-
-
-## 1.77.0
+## Unreleased
 
 ### API changes
 - change semantics of `dc_get_webxdc_status_updates()` second parameter
   and remove update-id from `DC_EVENT_WEBXDC_STATUS_UPDATE` #3081
 
+### Fixes
+- Hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
+- do not delete messages without Message-IDs as duplicates #3095
+- Assign replies from a different email address to the correct chat #3119
+- start ephemeral timer when seen status is synchronized via IMAP #3122
+- do not delete duplicate messages on IMAP immediately to accidentally deleting
+  the last copy #3138
+
 ### Changes
 - add more SMTP logging #3093
 - place common headers like `From:` before the large `Autocrypt:` header #3079
@@ -47,32 +22,12 @@
 - improve speed by caching config values #3131 #3145
 - optimize `markseen_msgs` #3141
 - automatically accept chats with outgoing messages #3143
-- `dc_receive_imf` refactorings #3154 #3156 #3159
+- `dc_receive_imf` refactorings #3154 #3156
 - add index to speedup deletion of expired ephemeral messages #3155
-- muted chats stay archived on new messages #3184
-- support `min_api` from Webxdc manifests #3206
-- do not read whole webxdc file into memory #3109
-- improve tests, refactorings #3073 #3096 #3102 #3108 #3139 #3128 #3133 #3142 #3153 #3151 #3174 #3170 #3148 #3179 #3185
-- improve documentation #2983 #3112 #3103 #3118 #3120
+
 
 ### Fixes
-- speed up loading of chat messages by a factor of 20 #3171 #3194 #3173
-- fix an issue where the app crashes when trying to export a backup #3195
-- hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
-- do not delete messages without Message-IDs as duplicates #3095
-- assign replies from a different email address to the correct chat #3119
-- assing outgoing private replies to the correct chat #3177
-- start ephemeral timer when seen status is synchronized via IMAP #3122
-- do not create empty contact requests with "setup changed" messages;
-  instead, send a "setup changed" message into all chats we share with the peer #3187
-- do not delete duplicate messages on IMAP immediately to accidentally deleting
-  the last copy #3138
-- clear more columns when message expires due to `delete_device_after` setting #3181
-- do not try to use stale SMTP connections #3180
-- slightly improve finding the correct server after logging in #3207
-- retry message sending automatically if loop is not interrupted #3183
-- fix a bug where sometimes the file extension of a long filename containing a dot was cropped #3098
-
+- Fix a bug where sometimes the file extension of a long filename containing a dot was cropped #3098
 
 ## 1.76.0
 

CMakeLists.txt

@@ -4,18 +4,10 @@ include(GNUInstallDirs)
 
 find_program(CARGO cargo)
 
-if(APPLE)
-       set(DYNAMIC_EXT "dylib")
-elseif(UNIX)
-       set(DYNAMIC_EXT "so")
-else()
-       set(DYNAMIC_EXT "dll")
-endif()
-
 add_custom_command(
 	OUTPUT
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
+	"target/release/libdeltachat.so"
 	"target/release/pkgconfig/deltachat.pc"
         COMMAND
         PREFIX=${CMAKE_INSTALL_PREFIX}
@@ -40,11 +32,11 @@ add_custom_target(
 	ALL
 	DEPENDS
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
+	"target/release/libdeltachat.so"
 	"target/release/pkgconfig/deltachat.pc"
 )
 
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.78.0"
+version = "1.76.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -19,7 +19,7 @@ ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
 async-imap = { git = "https://github.com/async-email/async-imap" }
 async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", default-features=false, features = ["smtp-transport", "socks5"] }
+async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
 async-std-resolver = "0.21"
 async-std = { version = "1" }
 async-tar = { version = "0.4", default-features=false }
@@ -62,6 +62,7 @@ serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
 smallvec = "1"
+stop-token = "0.7"
 strum = "0.24"
 strum_macros = "0.24"
 surf = { version = "2.3", default-features = false, features = ["h1-client"] }
@@ -74,7 +75,7 @@ humansize = "1"
 qrcodegen = "1.7.0"
 tagger = "4.3.3"
 textwrap = "0.15.0"
-zip = { version = "0.6.2", default-features = false, features = ["deflate"] }
+zip = { version = "0.6.0", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
@@ -123,10 +124,6 @@ harness = false
 name = "get_chat_msgs"
 harness = false
 
-[[bench]]
-name = "get_chatlist"
-harness = false
-
 [features]
 default = ["vendored"]
 internals = []

benches/get_chat_msgs.rs

@@ -27,12 +27,10 @@ fn criterion_benchmark(c: &mut Criterion) {
             (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
         });
 
-        c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
+        c.bench_function("Load all chats", |b| {
             b.to_async(AsyncStdExecutor)
-                .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
+                .iter(|| get_chat_msgs_benchmark(black_box(&path.as_ref()), black_box(&chats)))
         });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
     }
 }
 

benches/get_chatlist.rs

@@ -1,27 +0,0 @@
-use criterion::async_executor::AsyncStdExecutor;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use deltachat::chatlist::Chatlist;
-use deltachat::context::Context;
-
-async fn get_chat_list_benchmark(context: &Context) {
-    Chatlist::try_load(context, 0, None, None).await.unwrap();
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let context =
-            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
-        c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
-            b.to_async(AsyncStdExecutor)
-                .iter(|| get_chat_list_benchmark(black_box(&context)))
-        });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/receive_emails.rs

@@ -31,7 +31,7 @@ Hello {i}",
             i = i,
             i_dec = i - 1,
         );
-        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), false)
+        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), "INBOX", false)
             .await
             .unwrap();
     }

benches/search_msgs.rs

@@ -20,8 +20,6 @@ fn criterion_benchmark(c: &mut Criterion) {
         c.bench_function("search hello", |b| {
             b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
         });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
     }
 }
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.78.0"
+version = "1.76.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -1801,25 +1801,6 @@ void            dc_delete_msgs               (dc_context_t* context, const uint3
 void            dc_forward_msgs              (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt, uint32_t chat_id);
 
 
-/**
- * Resend messages and make information available for newly added chat members.
- * Resending sends out the original message, however, recipients and webxdc-status may differ.
- * Clients that already have the original message can still ignore the resent message as
- * they have tracked the state by dedicated updates.
- *
- * Some messages cannot be resent, eg. info-messages, drafts, already pending messages or messages that are not sent by SELF.
- * In this case, the return value indicates an error and the error string from dc_get_last_error() should be shown to the user.
- *
- * @memberof dc_context_t
- * @param context The context object.
- * @param msg_ids An array of uint32_t containing all message IDs that should be resend.
- *     All messages must belong to the same chat.
- * @param msg_cnt The number of messages IDs in the msg_ids array.
- * @return 1=all messages are queued for resending, 0=error
- */
-int             dc_resend_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
-
-
 /**
  * Mark messages as presented to the user.
  * Typically, UIs call this function on scrolling through the message list,
@@ -4648,10 +4629,9 @@ int             dc_contact_is_verified       (dc_contact_t* contact);
 
 
 /**
- * Create a provider struct for the given e-mail address by local lookup.
+ * Create a provider struct for the given e-mail address.
  *
- * Lookup is done from the local database by extracting the domain from the e-mail address.
- * Therefore the provider for custom domains cannot be identified.
+ * The provider is extracted from the e-mail address and it's information is returned.
  *
  * @memberof dc_provider_t
  * @param context The context object.
@@ -4663,23 +4643,6 @@ int             dc_contact_is_verified       (dc_contact_t* contact);
 dc_provider_t*  dc_provider_new_from_email            (const dc_context_t* context, const char* email);
 
 
-/**
- * Create a provider struct for the given e-mail address by local and DNS lookup.
- *
- * First lookup is done from the local database as of dc_provider_new_from_email().
- * If the first lookup fails, an additional DNS lookup is done,
- * trying to figure out the provider belonging to custom domains.
- *
- * @memberof dc_provider_t
- * @param context The context object.
- * @param email The user's e-mail address to extract the provider info form.
- * @return A dc_provider_t struct which can be used with the dc_provider_get_*
- *     accessor functions. If no provider info is found, NULL will be
- *     returned.
- */
-dc_provider_t*  dc_provider_new_from_email_with_dns    (const dc_context_t* context, const char* email);
-
-
 /**
  * URL of the overview page.
  *
@@ -5643,16 +5606,11 @@ void dc_event_unref(dc_event_t* event);
 /**
  * webxdc status update received.
  * To get the received status update, use dc_get_webxdc_status_updates() with
- * `serial` set to the last known update
- * (in case of special bots, `status_update_serial` from `data2`
- * may help to calculate the last known update for dc_get_webxdc_status_updates();
- * UIs must not peek at this parameter to avoid races in the status replication
- * eg. when events arrive while initial updates are played back).
- *
+ * `serial` set to the last known update.
  * To send status updates, use dc_send_webxdc_status_update().
  *
  * @param data1 (int) msg_id
- * @param data2 (int) status_update_serial - must not be used by UI implementations.
+ * @param data2 (int) 0
  */
 #define DC_EVENT_WEBXDC_STATUS_UPDATE                2120
 
@@ -6260,7 +6218,9 @@ void dc_event_unref(dc_event_t* event);
 /// `%1$s` will be replaced by the domain of the configured e-mail address.
 #define DC_STR_STORAGE_ON_DOMAIN          105
 
-/// @deprecated Deprecated 2022-04-16, this string is no longer needed.
+/// "One moment…"
+///
+/// Used in the connectivity view when some information are not yet there.
 #define DC_STR_ONE_MOMENT                 106
 
 /// "Connected"
@@ -6349,11 +6309,6 @@ void dc_event_unref(dc_event_t* event);
 /// `%1$s` will be replaced with the group name.
 #define DC_STR_SECURE_JOIN_GROUP_QR_DESC  120
 
-/// "Not connected"
-///
-/// Used as status in the connectivity view.
-#define DC_STR_NOT_CONNECTED              121
-
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -503,7 +503,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::SecurejoinJoinerProgress { contact_id, .. } => {
             contact_id.to_u32() as libc::c_int
         }
-        EventType::WebxdcStatusUpdate { msg_id, .. } => msg_id.to_u32() as libc::c_int,
+        EventType::WebxdcStatusUpdate(msg_id) => msg_id.to_u32() as libc::c_int,
     }
 }
 
@@ -535,8 +535,9 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ImexFileWritten(_)
         | EventType::MsgsNoticed(_)
         | EventType::ConnectivityChanged
-        | EventType::SelfavatarChanged => 0,
-        EventType::ChatModified(_) => 0,
+        | EventType::SelfavatarChanged
+        | EventType::WebxdcStatusUpdate(_)
+        | EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
         | EventType::MsgDelivered { msg_id, .. }
@@ -545,10 +546,6 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         EventType::SecurejoinInviterProgress { progress, .. }
         | EventType::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
         EventType::ChatEphemeralTimerModified { timer, .. } => timer.to_u32() as libc::c_int,
-        EventType::WebxdcStatusUpdate {
-            status_update_serial,
-            ..
-        } => status_update_serial.to_u32() as libc::c_int,
     }
 }
 
@@ -590,7 +587,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SecurejoinJoinerProgress { .. }
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
-        | EventType::WebxdcStatusUpdate { .. }
+        | EventType::WebxdcStatusUpdate(_)
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -1751,27 +1748,6 @@ pub unsafe extern "C" fn dc_forward_msgs(
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_resend_msgs(
-    context: *mut dc_context_t,
-    msg_ids: *const u32,
-    msg_cnt: libc::c_int,
-) -> libc::c_int {
-    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
-        eprintln!("ignoring careless call to dc_resend_msgs()");
-        return 0;
-    }
-    let ctx = &*context;
-    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
-
-    if let Err(err) = block_on(chat::resend_msgs(ctx, &msg_ids)) {
-        error!(ctx, "Resending failed: {}", err);
-        0
-    } else {
-        1
-    }
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_markseen_msgs(
     context: *mut dc_context_t,
@@ -1973,19 +1949,18 @@ pub unsafe extern "C" fn dc_block_contact(
     contact_id: u32,
     block: libc::c_int,
 ) {
-    let contact_id = ContactId::new(contact_id);
-    if context.is_null() || contact_id.is_special() {
+    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL.to_u32() {
         eprintln!("ignoring careless call to dc_block_contact()");
         return;
     }
     let ctx = &*context;
     block_on(async move {
         if block == 0 {
-            Contact::unblock(ctx, contact_id)
+            Contact::unblock(ctx, ContactId::new(contact_id))
                 .await
                 .ok_or_log_msg(ctx, "Can't unblock contact");
         } else {
-            Contact::block(ctx, contact_id)
+            Contact::block(ctx, ContactId::new(contact_id))
                 .await
                 .ok_or_log_msg(ctx, "Can't block contact");
         }
@@ -2019,15 +1994,14 @@ pub unsafe extern "C" fn dc_delete_contact(
     context: *mut dc_context_t,
     contact_id: u32,
 ) -> libc::c_int {
-    let contact_id = ContactId::new(contact_id);
-    if context.is_null() || contact_id.is_special() {
+    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL.to_u32() {
         eprintln!("ignoring careless call to dc_delete_contact()");
         return 0;
     }
     let ctx = &*context;
 
     block_on(async move {
-        match Contact::delete(ctx, contact_id).await {
+        match Contact::delete(ctx, ContactId::new(contact_id)).await {
             Ok(_) => 1,
             Err(_) => 0,
         }
@@ -3973,25 +3947,6 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
     }
     let addr = to_string_lossy(addr);
 
-    let ctx = &*context;
-
-    match block_on(provider::get_provider_info(ctx, addr.as_str(), true)) {
-        Some(provider) => provider,
-        None => ptr::null_mut(),
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_new_from_email_with_dns(
-    context: *const dc_context_t,
-    addr: *const libc::c_char,
-) -> *const dc_provider_t {
-    if context.is_null() || addr.is_null() {
-        eprintln!("ignoring careless call to dc_provider_new_from_email_with_dns()");
-        return ptr::null();
-    }
-    let addr = to_string_lossy(addr);
-
     let ctx = &*context;
     let socks5_enabled = block_on(async move {
         ctx.get_config_bool(config::Config::Socks5Enabled)

draft/webxdc-dev-reference.md

@@ -54,13 +54,12 @@ and you won't get the update by `setUpdateListener()`.
 ### setUpdateListener()
 
 ```js
-let promise = window.webxdc.setUpdateListener((update) => {}, serial);
+window.webxdc.setUpdateListener((update) => {}, serial);
 ```
 
 With `setUpdateListener()` you define a callback that receives the updates
 sent by `sendUpdate()`. The callback is called for updates sent by you or other peers.
 The `serial` specifies the last serial that you know about (defaults to 0). 
-The returned promise resolves when the listener has processed all the update messages known at the time when  `setUpdateListener` was called. 
 
 Each `update` which is passed to the callback comes with the following properties: 
 
@@ -177,9 +176,6 @@ just clone and start adapting things to your need.
 
 - older devices might not have the newest js features in their webview,
   you may want to transpile your code down to an older js version eg. with https://babeljs.io
-- viewport and scaling features are implementation specific,
-  if you want to have an explicit behavior, you can add eg.
-  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
 - there are tons of ideas for enhancements of the API and the file format,
   eg. in the future, we will may define icon- and manifest-files,
   allow to aggregate the state or add metadata.

examples/repl/cmdline.rs

@@ -21,6 +21,7 @@ use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
+use deltachat::EventType;
 use deltachat::{config, provider};
 use std::fs;
 use std::time::{Duration, SystemTime};
@@ -92,13 +93,16 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_msgs_changed_without_ids();
+    context.emit_event(EventType::MsgsChanged {
+        chat_id: ChatId::new(0),
+        msg_id: MsgId::new(0),
+    });
 }
 
 async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
     let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, false).await {
+    if let Err(err) = dc_receive_imf(context, &data, "import", false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -160,7 +164,10 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_msgs_changed_without_ids();
+        context.emit_event(EventType::MsgsChanged {
+            chat_id: ChatId::new(0),
+            msg_id: MsgId::new(0),
+        });
     }
     true
 }
@@ -203,7 +210,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         contact_id,
         msgtext.unwrap_or_default(),
         if msg.has_html() { "[HAS-HTML]️" } else { "" },
-        if msg.get_from_id() == ContactId::SELF {
+        if msg.get_from_id() == DC_CONTACT_ID_SELF {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -290,7 +297,7 @@ async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()
         let peerstate = Peerstate::from_addr(context, addr)
             .await
             .expect("peerstate error");
-        if peerstate.is_some() && *contact_id != ContactId::SELF {
+        if peerstate.is_some() && *contact_id != DC_CONTACT_ID_SELF {
             line2 = format!(
                 ", prefer-encrypt={}",
                 peerstate.as_ref().unwrap().prefer_encrypt
@@ -399,7 +406,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
-                 resend <msg-id>\n\
                  markseen <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
@@ -926,24 +932,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
-            let query = format!("{} {}", arg1, arg2).trim().to_string();
             let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
             let time_start = std::time::SystemTime::now();
-            let msglist = context.search_msgs(chat, &query).await?;
+            let msglist = context.search_msgs(chat, arg1).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
-            println!(
-                "{}{} messages for {}search of \"{}\"",
-                msglist.len(),
-                if msglist.len() == 1000 { "+" } else { "" },
-                if chat.is_none() {
-                    "global "
-                } else {
-                    "in-chat-"
-                },
-                query,
-            );
+            println!("{} messages.", msglist.len());
             println!("{:?} to create this list", time_needed);
         }
         "draft" => {
@@ -1100,13 +1095,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(&context, &msg_ids, chat_id).await?;
         }
-        "resend" => {
-            ensure!(!arg1.is_empty(), "Arguments <msg-id> expected");
-
-            let mut msg_ids = [MsgId::new(0); 1];
-            msg_ids[0] = MsgId::new(arg1.parse()?);
-            chat::resend_msgs(&context, &msg_ids).await?;
-        }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];

examples/repl/main.rs

@@ -207,12 +207,11 @@ const CHAT_COMMANDS: [&str; 36] = [
     "accept",
     "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&str; 7] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
-    "resend",
     "markseen",
     "delmsg",
     "download",

python/install_python_bindings.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
 
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
         subprocess.check_call([

python/mypy.ini

@@ -17,7 +17,3 @@ ignore_missing_imports = True
 
 [mypy-_pytest.*]
 ignore_missing_imports = True
-
-[mypy-imap_tools.*]
-ignore_missing_imports = True
-

python/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
+requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]

python/setup.py

@@ -11,11 +11,8 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
-        setup_requires=[
-            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
-            'pkgconfig',
-        ],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
+        setup_requires=['setuptools_scm'], # required for compatibility with `python3 setup.py sdist`
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/_build.py

@@ -8,9 +8,9 @@ import shutil
 import subprocess
 import tempfile
 import textwrap
+import types
 
 import cffi
-import pkgconfig  # type: ignore
 
 
 def local_build_flags(projdir, target):
@@ -19,31 +19,36 @@ def local_build_flags(projdir, target):
     :param projdir: The root directory of the deltachat-core-rust project.
     :param target: The rust build target, `debug` or `release`.
     """
-    flags = {}
+    flags = types.SimpleNamespace()
     if platform.system() == 'Darwin':
-        flags['libraries'] = ['resolv', 'dl']
-        flags['extra_link_args'] = [
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
                 '-framework', 'CoreFoundation',
                 '-framework', 'CoreServices',
                 '-framework', 'Security',
         ]
     elif platform.system() == 'Linux':
-        flags['libraries'] = ['rt', 'dl', 'm']
-        flags['extra_link_args'] = []
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
         raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
     target_dir = os.environ.get("CARGO_TARGET_DIR")
     if target_dir is None:
         target_dir = os.path.join(projdir, 'target')
-    flags['extra_objects'] = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags['extra_objects'][0]), flags['extra_objects']
-    flags['include_dirs'] = [os.path.join(projdir, 'deltachat-ffi')]
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
     return flags
 
 
 def system_build_flags():
     """Construct build flags for building against an installed libdeltachat."""
-    return pkgconfig.parse('deltachat')
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+    return flags
 
 
 def extract_functions(flags):
@@ -64,7 +69,7 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags['include_dirs'],
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
             return dst_fp.read()
@@ -100,7 +105,7 @@ def find_header(flags):
         try:
             os.chdir(tmpdir)
             cc.compile(sources=["where.c"],
-                       include_dirs=flags['include_dirs'],
+                       include_dirs=flags.incs,
                        macros=[("PY_CFFI_INC", "1")])
         finally:
             os.chdir(cwd)
@@ -178,7 +183,10 @@ def ffibuilder():
                 return DC_EVENT_DATA2_IS_STRING(e);
             }
         """,
-        **flags,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
     )
     builder.cdef("""
         typedef int... time_t;

python/src/deltachat/chat.py

@@ -5,7 +5,7 @@ import calendar
 import json
 from datetime import datetime, timezone
 import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
@@ -517,7 +517,7 @@ class Chat(object):
                     lib.dc_array_get_timestamp(dc_array, i),
                     timezone.utc
                 ),
-                marker=from_optional_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]

python/src/deltachat/direct_imap.py

@@ -4,22 +4,25 @@ and for cleaning up inbox/mvbox for each test function run.
 """
 
 import io
+import email
 import ssl
 import pathlib
-from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
 import imaplib
 import deltachat
 from deltachat import const, Account
-from typing import List
 
 
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
 FLAGS = b'FLAGS'
 FETCH = b'FETCH'
 ALL = "1:*"
 
 
 @deltachat.global_hookimpl
-def dc_account_extra_configure(account: Account):
+def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
@@ -33,10 +36,12 @@ def dc_account_extra_configure(account: Account):
                     assert imap.select_folder(folder)
                     imap.delete(ALL, expunge=True)
                 else:
-                    imap.conn.folder.delete(folder)
+                    imap.conn.delete_folder(folder)
                     # We just deleted the folder, so we have to make DC forget about it, too
                     if account.get_config("configured_sentbox_folder") == folder:
                         account.set_config("configured_sentbox_folder", None)
+                    if account.get_config("configured_spam_folder") == folder:
+                        account.set_config("configured_spam_folder", None)
 
             setattr(account, "direct_imap", imap)
 
@@ -83,34 +88,37 @@ class DirectImap:
             ssl_context.verify_mode = ssl.CERT_NONE
 
         if security == const.DC_SOCKET_STARTTLS:
-            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
-        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
-            self.conn = MailBox(host, port, ssl_context=ssl_context)
+            self.conn = IMAPClient(host, port, ssl=False)
+            self.conn.starttls(ssl_context)
+        elif security == const.DC_SOCKET_PLAIN:
+            self.conn = IMAPClient(host, port, ssl=False)
+        elif security == const.DC_SOCKET_SSL:
+            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
 
     def shutdown(self):
         try:
-            self.idle_done()
-        except (OSError, imaplib.IMAP4.abort):
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
             pass
         try:
             self.conn.logout()
-        except (OSError, imaplib.IMAP4.abort):
+        except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
     def create_folder(self, foldername):
         try:
-            self.conn.folder.create(foldername)
-        except errors.MailboxFolderCreateError as e:
+            self.conn.create_folder(foldername)
+        except imaplib.IMAP4.error as e:
             print("Can't create", foldername, "probably it already exists:", str(e))
 
-    def select_folder(self, foldername: str) -> tuple:
+    def select_folder(self, foldername):
         assert not self._idling
-        return self.conn.folder.set(foldername)
+        return self.conn.select_folder(foldername)
 
-    def select_config_folder(self, config_name: str):
+    def select_config_folder(self, config_name):
         """ Return info about selected folder if it is
         configured, otherwise None. """
         if "_" not in config_name:
@@ -119,36 +127,50 @@ class DirectImap:
         if foldername:
             return self.select_folder(foldername)
 
-    def list_folders(self) -> List[str]:
+    def list_folders(self):
         """ return list of all existing folder names"""
         assert not self._idling
-        return [folder.name for folder in self.conn.folder.list()]
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
 
-    def delete(self, uid_list: str, expunge=True):
+    def delete(self, range, expunge=True):
         """ delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
+        self.conn.set_flags(range, [DELETED])
         if expunge:
             self.conn.expunge()
 
-    def get_all_messages(self) -> List[MailMessage]:
+    def get_all_messages(self):
         assert not self._idling
-        return [mail for mail in self.conn.fetch()]
 
-    def get_unread_messages(self) -> List[str]:
+        # Flush unsolicited responses. IMAPClient has problems
+        # dealing with them: https://github.com/mjs/imapclient/issues/334
+        # When this NOOP was introduced, next FETCH returned empty
+        # result instead of a single message, even though IMAP server
+        # can only return more untagged responses than required, not
+        # less.
+        self.conn.noop()
+
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
         assert not self._idling
-        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
         if messages:
-            res = self.conn.flag(messages, MailMessageFlags.SEEN, True)
+            res = self.conn.set_flags(messages, [SEEN])
             print("marked seen:", messages, res)
 
-    def get_unread_cnt(self) -> int:
+    def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
     def dump_imap_structures(self, dir, logfile):
@@ -170,20 +192,21 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            for msg in self.conn.fetch(mark_seen=False):
-                body = getattr(msg.obj, "text", None)
-                if not body:
-                    body = getattr(msg.obj, "html", None)
-                if not body:
-                    log("Message", msg.uid, "has empty body")
+            requested = [b'BODY.PEEK[]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[]']
+                if not body_bytes:
+                    log("Message", uid, "has empty body")
                     continue
 
+                flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(msg.uid))
-                fn.write_bytes(body)
-                log("Message", msg.uid, fn)
-                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
@@ -193,58 +216,49 @@ class DirectImap:
     def idle_start(self):
         """ switch this connection to idle mode. non-blocking. """
         assert not self._idling
-        res = self.conn.idle.start()
+        res = self.conn.idle()
         self._idling = True
         return res
 
-    def idle_check(self, terminate=False, timeout=None) -> List[bytes]:
+    def idle_check(self, terminate=False):
         """ (blocking) wait for next idle message from server. """
         assert self._idling
         self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle.poll(timeout=timeout)
+        res = self.conn.idle_check()
         if terminate:
             self.idle_done()
         self.account.log("imap-direct: idle_check returned {!r}".format(res))
         return res
 
-    def idle_wait_for_new_message(self, terminate=False, timeout=None) -> bytes:
-        while 1:
-            for item in self.idle_check(timeout=timeout):
-                if b'EXISTS' in item or b'RECENT' in item:
-                    if terminate:
-                        self.idle_done()
-                    return item
-
-    def idle_wait_for_seen(self, terminate=False, timeout=None) -> int:
-        """ Return first message with SEEN flag from a running idle-stream.
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
         """
         while 1:
-            for item in self.idle_check(timeout=timeout):
-                if FETCH in item:
-                    self.account.log(str(item))
-                    if FLAGS in item and rb'\Seen' in item:
-                        if terminate:
-                            self.idle_done()
-                        return int(item.split(b' ')[1])
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
 
     def idle_done(self):
         """ send idle-done to server if we are currently in idle mode. """
         if self._idling:
-            res = self.conn.idle.stop()
+            res = self.conn.idle_done()
             self._idling = False
             return res
 
-    def append(self, folder: str, msg: str):
+    def append(self, folder, msg):
         """Upload a message to *folder*.
         Trailing whitespace or a linebreak at the beginning will be removed automatically.
         """
         if msg.startswith("\n"):
             msg = msg[1:]
         msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(bytes(msg, encoding='ascii'), folder)
+        self.conn.append(folder, msg)
 
-    def get_uid_by_message_id(self, message_id) -> str:
-        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
+    def get_uid_by_message_id(self, message_id):
+        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
         if len(msgs) == 0:
             raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
         return msgs[0]

python/tests/test_account.py

@@ -11,7 +11,6 @@ from deltachat.hookspec import account_hookimpl
 from deltachat.capi import ffi, lib
 from deltachat.cutil import iter_array
 from datetime import datetime, timedelta, timezone
-from imap_tools import AND, U
 
 
 @pytest.mark.parametrize("msgtext,res", [
@@ -1014,7 +1013,7 @@ class TestOnlineAccount:
         assert ev.data1 == msg2.chat.id
         assert ev.data2 == 0
 
-        ac2.direct_imap.idle_wait_for_new_message(terminate=True)
+        ac2.direct_imap.idle_check(terminate=True)
         lp.step("1")
         for i in range(2):
             ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
@@ -1045,7 +1044,8 @@ class TestOnlineAccount:
 
         ac1.create_chat(ac2).send_text("Hello!")
 
-        ac2.direct_imap.idle_wait_for_new_message(terminate=True)
+        # Wait for the message to arrive.
+        ac2.direct_imap.idle_check(terminate=True)
 
         # Emulate moving of the message to DeltaChat folder by Sieve rule.
         # mailcow server contains this rule by default.
@@ -1059,9 +1059,13 @@ class TestOnlineAccount:
         # Accept the contact request.
         msg.chat.accept()
         ac2.mark_seen_messages([msg])
-        uid = ac2.direct_imap.idle_wait_for_seen(terminate=True)
+        ac2.direct_imap.idle_wait_for_seen()
+        ac2.direct_imap.idle_done()
 
-        assert len([a for a in ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*")))]) == 1
+        fetch = list(ac2.direct_imap.conn.fetch("*", b'FLAGS').values())
+        flags = fetch[-1][b'FLAGS']
+        is_seen = b'\\Seen' in flags
+        assert is_seen
 
     def test_multidevice_sync_seen(self, acfactory, lp):
         """Test that message marked as seen on one device is marked as seen on another."""
@@ -1433,7 +1437,6 @@ class TestOnlineAccount:
         ac1.direct_imap.create_folder("Drafts")
         ac1.direct_imap.create_folder("Sent")
         ac1.direct_imap.create_folder("Spam")
-        ac1.direct_imap.create_folder("Junk")
 
         acfactory.wait_configure_and_start_io()
         # Wait until each folder was selected once and we are IDLEing again:
@@ -1467,15 +1470,6 @@ class TestOnlineAccount:
 
             Unknown message in Spam
         """.format(ac1.get_config("configured_addr")))
-        ac1.direct_imap.append("Junk", """
-            From: unknown.address@junk.org
-            Subject: subj
-            To: {}
-            Message-ID: <spam.message@junk.org>
-            Content-Type: text/plain; charset=utf-8
-
-            Unknown message in Junk
-        """.format(ac1.get_config("configured_addr")))
 
         ac1.set_config("scan_all_folders_debounce_secs", "0")
         lp.sec("All prepared, now let DC find the message")
@@ -2217,7 +2211,7 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.create_chat(ac1).send_text("Hi")
 
-        ac1.direct_imap.idle_wait_for_new_message(terminate=False)
+        ac1.direct_imap.idle_check(terminate=False)
         ac1.maybe_network()
 
         ac1._evtracker.wait_for_all_work_done()
@@ -2229,7 +2223,7 @@ class TestOnlineAccount:
 
         ac2.create_chat(ac1).send_text("Hi 2")
 
-        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
+        ac1.direct_imap.idle_check(terminate=True)
         ac1.maybe_network()
         ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTED, const.DC_CONNECTIVITY_WORKING)
         ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_WORKING, const.DC_CONNECTIVITY_CONNECTED)
@@ -2254,7 +2248,7 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.create_chat(ac1).send_text("Hi")
 
-        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
+        ac1.direct_imap.idle_check(terminate=True)
         ac1.maybe_network()
 
         while 1:
@@ -2444,23 +2438,25 @@ class TestOnlineAccount:
 
         acfactory.wait_configure_and_start_io()
 
+        imap2 = ac2.direct_imap
+        imap2.idle_start()
+
         lp.sec("ac1: create chat with ac2")
         chat1 = ac1.create_chat(ac2)
         ac2.create_chat(ac1)
 
-        lp.sec("ac1: send message to ac2")
         sent_msg = chat1.send_text("hello")
+        imap2.idle_check(terminate=False)
 
         msg = ac2._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        lp.sec("ac2: wait for close/expunge on autodelete")
+        imap2.idle_check(terminate=True)
         ac2._evtracker.get_info_contains("close/expunge succeeded")
 
-        lp.sec("ac2: check that message was autodeleted on server")
-        assert len(ac2.direct_imap.get_all_messages()) == 0
+        assert len(imap2.get_all_messages()) == 0
 
-        lp.sec("ac2: Mark deleted message as seen and check that read receipt arrives")
+        # Mark deleted message as seen and check that read receipt arrives
         msg.mark_seen()
         ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
         assert ev.data1 == chat1.id
@@ -2548,9 +2544,10 @@ class TestOnlineAccount:
 
         ac2._evtracker.get_info_contains("close/expunge succeeded")
 
-        lp.sec("ac2: test that only one message is left")
-        ac2.direct_imap.select_config_folder("inbox")
-        assert len(ac2.direct_imap.get_all_messages()) == 1
+        lp.sec("imap2: test that only one message is left")
+        imap2 = ac2.direct_imap
+
+        assert len(imap2.get_all_messages()) == 1
 
     def test_configure_error_msgs(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2720,7 +2717,13 @@ class TestOnlineAccount:
         ac1.direct_imap.select_config_folder("inbox")
         ac1.direct_imap.idle_start()
         acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
-        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
+        while True:
+            if len(ac1.direct_imap.idle_check(terminate=True)) > 1:
+                # If length is 1, it's [(b'OK', b'Still here')]
+                # Could happen on very slow network.
+                #
+                # More is usually [(1, b'EXISTS'), (1, b'RECENT')]
+                break
         ac1.direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
 
         lp.sec("Everything prepared, now see if DeltaChat finds the message (" + variant + ")")
@@ -2769,7 +2772,10 @@ class TestOnlineAccount:
         acfactory.wait_configure_and_start_io()
         assert_folders_configured(ac1)
 
-        assert ac1.direct_imap.select_config_folder("mvbox" if mvbox_move else "inbox")
+        if mvbox_move:
+            ac1.direct_imap.select_config_folder("mvbox")
+        else:
+            ac1.direct_imap.select_folder("INBOX")
         ac1.direct_imap.idle_start()
 
         lp.sec("send out message with bcc to ourselves")
@@ -2778,24 +2784,22 @@ class TestOnlineAccount:
         chat.send_text("message text")
         assert_folders_configured(ac1)
 
-        lp.sec("wait until the bcc_self message arrives in correct folder and is marked seen")
+        # now wait until the bcc_self message arrives
+        # Also test that bcc_self messages moved to the mvbox are marked as read.
         assert ac1.direct_imap.idle_wait_for_seen()
         assert_folders_configured(ac1)
 
-        lp.sec("create a cloned ac1 and fetch contact history during configure")
         ac1_clone = acfactory.clone_online_account(ac1)
         ac1_clone.set_config("fetch_existing_msgs", "1")
         ac1_clone._configtracker.wait_finish()
         ac1_clone.start_io()
         assert_folders_configured(ac1_clone)
 
-        lp.sec("check that ac2 contact was fetchted during configure")
         ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
         ac2_addr = ac2.get_config("addr")
         assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
         assert_folders_configured(ac1_clone)
 
-        lp.sec("check that messages changed events arrive for the correct message")
         msg = ac1_clone._evtracker.wait_next_messages_changed()
         assert msg.text == "message text"
         assert_folders_configured(ac1)
@@ -2854,15 +2858,15 @@ class TestOnlineAccount:
         ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure(ac1)
 
-        ac1.direct_imap.conn.folder.delete("DeltaChat")
-        assert "DeltaChat" not in ac1.direct_imap.list_folders()
+        ac1.direct_imap.conn.delete_folder("DeltaChat")
+        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 0
         acfactory.wait_configure_and_start_io()
 
         ac2.create_chat(ac1).send_text("hello")
         msg = ac1._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        assert "DeltaChat" in ac1.direct_imap.list_folders()
+        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 1
 
 
 class TestGroupStressTests:

python/tox.ini

@@ -8,7 +8,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n1 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 

rust-toolchain

@@ -1 +1 @@
-1.60.0
+1.59.0

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.60.0
+RUST_VERSION=1.59.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.60.0
+RUST_VERSION=1.59.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

src/accounts.rs

@@ -54,11 +54,7 @@ impl Accounts {
         ensure!(dir.exists().await, "directory does not exist");
 
         let config_file = dir.join(CONFIG_NAME);
-        ensure!(
-            config_file.exists().await,
-            "{:?} does not exist",
-            config_file
-        );
+        ensure!(config_file.exists().await, "accounts.toml does not exist");
 
         let config = Config::from_file(config_file)
             .await
@@ -148,27 +144,9 @@ impl Accounts {
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id).await {
-            // Spend up to 1 minute trying to remove the files.
-            // Files may remain locked up to 30 seconds due to r2d2 bug:
-            // https://github.com/sfackler/r2d2/issues/99
-            let mut counter = 0;
-            loop {
-                counter += 1;
-
-                if let Err(err) = fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
-                    .await
-                    .context("failed to remove account data")
-                {
-                    if counter > 60 {
-                        return Err(err);
-                    }
-
-                    // Wait 1 second and try again.
-                    async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
-                } else {
-                    break;
-                }
-            }
+            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                .await
+                .context("failed to remove account data")?;
         }
         self.config.remove_account(id).await?;
 

src/blob.rs

@@ -940,7 +940,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_recode_image_1() {
+    async fn test_recode_image() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
         send_image_check_mediaquality(Some("0"), bytes, 1000, 1000, 0, 1000, 1000)
@@ -957,10 +957,7 @@ mod tests {
         )
         .await
         .unwrap();
-    }
 
-    #[async_std::test]
-    async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(
@@ -981,24 +978,18 @@ mod tests {
             .write_to(&mut buf, image::ImageFormat::Jpeg)
             .unwrap();
         let bytes = buf.into_inner();
-
-        // Do this in parallel to speed up the test a bit
-        // (it still takes very long though)
-        let bytes2 = bytes.clone();
-        let join_handle = async_std::task::spawn(async move {
-            let img_rotated = send_image_check_mediaquality(
-                Some("0"),
-                &bytes2,
-                BALANCED_IMAGE_SIZE * 1800 / 2000,
-                BALANCED_IMAGE_SIZE,
-                0,
-                BALANCED_IMAGE_SIZE * 1800 / 2000,
-                BALANCED_IMAGE_SIZE,
-            )
-            .await
-            .unwrap();
-            assert_correct_rotation(&img_rotated);
-        });
+        let img_rotated = send_image_check_mediaquality(
+            Some("0"),
+            &bytes,
+            BALANCED_IMAGE_SIZE * 1800 / 2000,
+            BALANCED_IMAGE_SIZE,
+            0,
+            BALANCED_IMAGE_SIZE * 1800 / 2000,
+            BALANCED_IMAGE_SIZE,
+        )
+        .await
+        .unwrap();
+        assert_correct_rotation(&img_rotated);
 
         let img_rotated = send_image_check_mediaquality(
             Some("1"),
@@ -1013,11 +1004,6 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
-        join_handle.await;
-    }
-
-    #[async_std::test]
-    async fn test_recode_image_3() {
         let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
             .await

src/chatlist.rs

@@ -4,11 +4,13 @@ use anyhow::{ensure, Context as _, Result};
 
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
 use crate::constants::{
-    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_GCL_ADD_ALLDONE_HINT,
-    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CONTACT_ID_DEVICE,
+    DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT, DC_GCL_ARCHIVED_ONLY,
+    DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
 };
 use crate::contact::{Contact, ContactId};
 use crate::context::Context;
+use crate::ephemeral::delete_expired_messages;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock_str;
 use crate::summary::Summary;
@@ -90,6 +92,12 @@ impl Chatlist {
         let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
         let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
 
+        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
+        // messages get deleted to avoid reloading the same chatlist.
+        if let Err(err) = delete_expired_messages(context).await {
+            warn!(context, "Failed to hide expired messages: {}", err);
+        }
+
         let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
@@ -104,7 +112,7 @@ impl Chatlist {
         };
 
         let skip_id = if flag_for_forwarding {
-            ChatId::lookup_by_contact(context, ContactId::DEVICE)
+            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
                 .await?
                 .unwrap_or_default()
         } else {
@@ -208,7 +216,7 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                ChatId::lookup_by_contact(context, ContactId::SELF)
+                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
                     .await?
                     .unwrap_or_default()
             } else {
@@ -318,7 +326,7 @@ impl Chatlist {
 
         let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
             let lastmsg = Message::load_from_db(context, lastmsg_id).await?;
-            if lastmsg.from_id == ContactId::SELF {
+            if lastmsg.from_id == DC_CONTACT_ID_SELF {
                 (Some(lastmsg), None)
             } else {
                 match chat.typ {
@@ -335,7 +343,7 @@ impl Chatlist {
 
         if chat.id.is_archived_link() {
             Ok(Default::default())
-        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != ContactId::UNDEFINED) {
+        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED) {
             Ok(Summary::new(context, &lastmsg, chat, lastcontact.as_ref()).await)
         } else {
             Ok(Summary {
@@ -348,10 +356,6 @@ impl Chatlist {
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
         self.ids.iter().position(|(chat_id, _)| chat_id == &id)
     }
-
-    pub fn iter(&self) -> impl Iterator<Item = &(ChatId, Option<MsgId>)> {
-        self.ids.iter()
-    }
 }
 
 /// Returns the number of archived chats
@@ -503,6 +507,7 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
                  \n\
                  hello foo\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -563,6 +568,7 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:38:57 +0000\n\
                  \n\
                  hello foo\n",
+            "INBOX",
             false,
         )
         .await?;

src/config.rs

@@ -1,15 +1,16 @@
 //! # Key-value configuration management.
 
-use anyhow::{ensure, Context as _, Result};
+use anyhow::{ensure, Result};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
+use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
-use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
+use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
 
@@ -112,7 +113,6 @@ pub enum Config {
     DeleteDeviceAfter,
 
     SaveMimeHeaders,
-    /// The primary email address. Also see `SecondaryAddrs`.
     ConfiguredAddr,
     ConfiguredMailServer,
     ConfiguredMailUser,
@@ -131,14 +131,11 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
+    ConfiguredSpamFolder,
     ConfiguredTimestamp,
     ConfiguredProvider,
     Configured,
 
-    /// All secondary self addresses separated by spaces
-    /// (`addr1@example.org addr2@exapmle.org addr3@example.org`)
-    SecondaryAddrs,
-
     #[strum(serialize = "sys.version")]
     SysVersion,
 
@@ -296,8 +293,11 @@ impl Context {
             }
             Config::DeleteDeviceAfter => {
                 let ret = self.sql.set_raw_config(key, value).await;
-                // Interrupt ephemeral loop to delete old messages immediately.
-                self.interrupt_ephemeral_task().await;
+                // Force chatlist reload to delete old messages immediately.
+                self.emit_event(EventType::MsgsChanged {
+                    msg_id: MsgId::new(0),
+                    chat_id: ChatId::new(0),
+                });
                 ret?
             }
             Config::Displayname => {
@@ -333,73 +333,6 @@ impl Context {
     }
 }
 
-// Separate impl block for self address handling
-impl Context {
-    /// Determine whether the specified addr maps to the/a self addr.
-    /// Returns `false` if no addresses are configured.
-    pub(crate) async fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        Ok(self
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .iter()
-            .any(|a| addr_cmp(addr, a))
-            || self
-                .get_secondary_self_addrs()
-                .await?
-                .iter()
-                .any(|a| addr_cmp(addr, a)))
-    }
-
-    /// Sets `primary_new` as the new primary self address and saves the old
-    /// primary address (if exists) as a secondary address.
-    ///
-    /// This should only be used by test code and during configure.
-    pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
-        // add old primary address (if exists) to secondary addresses
-        let mut secondary_addrs = self.get_all_self_addrs().await?;
-        // never store a primary address also as a secondary
-        secondary_addrs.retain(|a| !addr_cmp(a, primary_new));
-        self.set_config(
-            Config::SecondaryAddrs,
-            Some(secondary_addrs.join(" ").as_str()),
-        )
-        .await?;
-
-        self.set_config(Config::ConfiguredAddr, Some(primary_new))
-            .await?;
-
-        Ok(())
-    }
-
-    /// Returns all primary and secondary self addresses.
-    pub(crate) async fn get_all_self_addrs(&self) -> Result<Vec<String>> {
-        let primary_addrs = self.get_config(Config::ConfiguredAddr).await?.into_iter();
-        let secondary_addrs = self.get_secondary_self_addrs().await?.into_iter();
-
-        Ok(primary_addrs.chain(secondary_addrs).collect())
-    }
-
-    /// Returns all secondary self addresses.
-    pub(crate) async fn get_secondary_self_addrs(&self) -> Result<Vec<String>> {
-        let secondary_addrs = self
-            .get_config(Config::SecondaryAddrs)
-            .await?
-            .unwrap_or_default();
-        Ok(secondary_addrs
-            .split_ascii_whitespace()
-            .map(|s| s.to_string())
-            .collect())
-    }
-
-    /// Returns the primary self address.
-    /// Returns an error if no self addr is configured.
-    pub async fn get_primary_self_addr(&self) -> Result<String> {
-        self.get_config(Config::ConfiguredAddr)
-            .await?
-            .context("No self addr configured")
-    }
-}
-
 /// Returns all available configuration keys concated together.
 fn get_config_keys_string() -> String {
     let keys = Config::iter().fold(String::new(), |mut acc, key| {
@@ -484,57 +417,4 @@ mod tests {
         assert_eq!(t.get_config_bool(c).await?, false);
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_self_addrs() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        assert!(alice.is_self_addr("alice@example.org").await?);
-        assert_eq!(alice.get_all_self_addrs().await?, vec!["alice@example.org"]);
-        assert!(!alice.is_self_addr("alice@alice.com").await?);
-
-        // Test adding the same primary address
-        alice.set_primary_self_addr("alice@example.org").await?;
-        alice.set_primary_self_addr("Alice@Example.Org").await?;
-        assert_eq!(alice.get_all_self_addrs().await?, vec!["Alice@Example.Org"]);
-
-        // Test adding a new (primary) self address
-        // The address is trimmed during by `LoginParam::from_database()`,
-        // so `set_primary_self_addr()` doesn't have to trim it.
-        alice.set_primary_self_addr(" Alice@alice.com ").await?;
-        assert!(alice.is_self_addr("    aliCe@example.org").await?);
-        assert!(alice.is_self_addr("alice@alice.com").await?);
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec![" Alice@alice.com ", "Alice@Example.Org"]
-        );
-
-        // Check that the entry is not duplicated
-        alice.set_primary_self_addr("alice@alice.com").await?;
-        alice.set_primary_self_addr("alice@alice.com").await?;
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["alice@alice.com", "Alice@Example.Org"]
-        );
-
-        // Test switching back
-        alice.set_primary_self_addr("alice@example.org").await?;
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["alice@example.org", "alice@alice.com"]
-        );
-
-        // Test setting a new primary self address, the previous self address
-        // should be kept as a secondary self address
-        alice.set_primary_self_addr("alice@alice.xyz").await?;
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["alice@alice.xyz", "alice@example.org", "alice@alice.com"]
-        );
-        assert!(alice.is_self_addr("alice@example.org").await?);
-        assert!(alice.is_self_addr("alice@alice.com").await?);
-        assert!(alice.is_self_addr("Alice@alice.xyz").await?);
-
-        Ok(())
-    }
 }

src/configure.rs

@@ -85,7 +85,7 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let mut param = LoginParam::load_candidate_params(self).await?;
+        let mut param = LoginParam::from_database(self, "").await?;
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
@@ -453,14 +453,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     drop(imap);
 
     progress!(ctx, 910);
-
-    if ctx.get_config(Config::ConfiguredAddr).await?.as_deref() != Some(&param.addr) {
-        // Switched account, all server UIDs we know are invalid
-        job::schedule_resync(ctx).await?;
-    }
-
     // the trailing underscore is correct
-    param.save_as_configured_params(ctx).await?;
+    param.save_to_database(ctx, "configured_").await?;
     ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
@@ -705,11 +699,11 @@ pub enum Error {
         error: quick_xml::Error,
     },
 
+    #[error("Failed to get URL: {0}")]
+    ReadUrl(#[from] self::read_url::Error),
+
     #[error("Number of redirection is exceeded")]
     Redirection,
-
-    #[error("{0:#}")]
-    Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]

src/configure/read_url.rs

@@ -1,40 +1,20 @@
 use crate::context::Context;
 
-use anyhow::format_err;
-use anyhow::Context as _;
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("URL request error")]
+    GetError(surf::Error),
+}
 
-pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
-    match read_url_inner(context, url).await {
-        Ok(s) => {
-            info!(context, "Successfully read url {}", url);
-            Ok(s)
-        }
-        Err(e) => {
-            info!(context, "Can't read URL {}: {:#}", url, e);
-            Err(format_err!("Can't read URL {}: {:#}", url, e))
+pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
+    info!(context, "Requesting URL {}", url);
+
+    match surf::get(url).recv_string().await {
+        Ok(res) => Ok(res),
+        Err(err) => {
+            info!(context, "Can\'t read URL {}: {}", url, err);
+
+            Err(Error::GetError(err))
         }
     }
 }
-
-pub async fn read_url_inner(context: &Context, mut url: &str) -> anyhow::Result<String> {
-    let mut _temp; // For the borrow checker
-
-    // Follow up to 10 http-redirects
-    for _i in 0..10 {
-        let mut response = surf::get(url).send().await.map_err(|e| e.into_inner())?;
-        if response.status().is_redirection() {
-            _temp = response
-                .header("location")
-                .context("Redirection doesn't have a target location")?
-                .last()
-                .to_string();
-            info!(context, "Following redirect to {}", _temp);
-            url = &_temp;
-            continue;
-        }
-
-        return response.body_string().await.map_err(|e| e.into_inner());
-    }
-
-    Err(format_err!("Followed 10 redirections"))
-}

src/configure/server_params.rs

@@ -52,8 +52,10 @@ impl ServerParams {
     fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
         if self.hostname.is_empty() {
             vec![
-                // Try "imap.ex.org"/"smtp.ex.org" and "mail.ex.org" first because if a server exists
-                // under this address, it's likely the correct one.
+                Self {
+                    hostname: param_domain.to_string(),
+                    ..self.clone()
+                },
                 Self {
                     hostname: match self.protocol {
                         Protocol::Imap => "imap.".to_string() + param_domain,
@@ -63,12 +65,6 @@ impl ServerParams {
                 },
                 Self {
                     hostname: "mail.".to_string() + param_domain,
-                    ..self.clone()
-                },
-                // Try "ex.org" last because if it's wrong and the server is configured to
-                // not answer at all, configuration may be stuck for several minutes.
-                Self {
-                    hostname: param_domain.to_string(),
                     ..self
                 },
             ]
@@ -300,48 +296,5 @@ mod tests {
                 strict_tls: Some(true)
             }],
         );
-
-        // Test that "example.net" is tried after "*.example.net".
-        let v = expand_param_vector(
-            vec![ServerParams {
-                protocol: Protocol::Imap,
-                hostname: "".to_string(),
-                port: 10480,
-                socket: Socket::Ssl,
-                username: "foobar".to_string(),
-                strict_tls: Some(true),
-            }],
-            "foobar@example.net",
-            "example.net",
-        );
-        assert_eq!(
-            v,
-            vec![
-                ServerParams {
-                    protocol: Protocol::Imap,
-                    hostname: "imap.example.net".to_string(),
-                    port: 10480,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: Some(true)
-                },
-                ServerParams {
-                    protocol: Protocol::Imap,
-                    hostname: "mail.example.net".to_string(),
-                    port: 10480,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: Some(true)
-                },
-                ServerParams {
-                    protocol: Protocol::Imap,
-                    hostname: "example.net".to_string(),
-                    port: 10480,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: Some(true)
-                }
-            ],
-        );
     }
 }

src/constants.rs

@@ -4,6 +4,7 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::ChatId;
+use crate::contact::ContactId;
 
 pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION").to_string());
 
@@ -179,6 +180,16 @@ pub const DC_ELLIPSIS: &str = "[...]";
 /// `char`s), not Unicode Grapheme Clusters.
 pub const DC_DESIRED_TEXT_LEN: usize = 5000;
 
+pub const DC_CONTACT_ID_UNDEFINED: ContactId = ContactId::new(0);
+pub const DC_CONTACT_ID_SELF: ContactId = ContactId::new(1);
+pub const DC_CONTACT_ID_INFO: ContactId = ContactId::new(2);
+pub const DC_CONTACT_ID_DEVICE: ContactId = ContactId::new(5);
+pub const DC_CONTACT_ID_LAST_SPECIAL: ContactId = ContactId::new(9);
+
+// decorative address that is used for DC_CONTACT_ID_DEVICE
+// when an api that returns an email is called.
+pub const DC_CONTACT_ID_DEVICE_ADDR: &str = "device@localhost";
+
 // Flags for empty server job
 
 pub const DC_EMPTY_MVBOX: u32 = 0x01;

src/contact.rs

@@ -14,7 +14,10 @@ use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
 use crate::color::str_to_color;
 use crate::config::Config;
-use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
+use crate::constants::{
+    Blocked, Chattype, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_DEVICE_ADDR, DC_CONTACT_ID_LAST_SPECIAL,
+    DC_CONTACT_ID_SELF, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY,
+};
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input, EmailAddress};
 use crate::events::EventType;
@@ -24,51 +27,24 @@ use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
-use crate::sql::{self, params_iter};
 use crate::{chat, stock_str};
 
 /// Contact ID, including reserved IDs.
 ///
 /// Some contact IDs are reserved to identify special contacts.  This
 /// type can represent both the special as well as normal contacts.
-#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[derive(
+    Debug, Copy, Clone, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize,
+)]
 pub struct ContactId(u32);
 
 impl ContactId {
-    pub const UNDEFINED: ContactId = ContactId::new(0);
-    /// The owner of the account.
-    ///
-    /// The email-address is set by `dc_set_config` using "addr".
-    pub const SELF: ContactId = ContactId::new(1);
-    pub const INFO: ContactId = ContactId::new(2);
-    pub const DEVICE: ContactId = ContactId::new(5);
-    const LAST_SPECIAL: ContactId = ContactId::new(9);
-
-    /// Address to go with [`ContactId::DEVICE`].
-    ///
-    /// This is used by APIs which need to return an email address for this contact.
-    pub const DEVICE_ADDR: &'static str = "device@localhost";
-
     /// Creates a new [`ContactId`].
     pub const fn new(id: u32) -> ContactId {
         ContactId(id)
     }
 
-    /// Whether this is a special [`ContactId`].
-    ///
-    /// Some [`ContactId`]s are reserved for special contacts like [`ContactId::SELF`],
-    /// [`ContactId::INFO`] and [`ContactId::DEVICE`].  This function indicates whether this
-    /// [`ContactId`] is any of the reserved special [`ContactId`]s (`true`) or whether it
-    /// is the [`ContactId`] of a real contact (`false`).
-    pub fn is_special(&self) -> bool {
-        self.0 <= Self::LAST_SPECIAL.0
-    }
-
-    /// Numerical representation of the [`ContactId`].
-    ///
-    /// Each contact ID has a unique numerical representation which is used in the database
-    /// (via [`rusqlite::ToSql`]) and also for FFI purposes.  In Rust code you should never
-    /// need to use this directly.
+    /// Bad evil escape hatch, do not use.
     pub const fn to_u32(&self) -> u32 {
         self.0
     }
@@ -76,19 +52,20 @@ impl ContactId {
 
 impl fmt::Display for ContactId {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        if *self == ContactId::UNDEFINED {
-            write!(f, "Contact#Undefined")
-        } else if *self == ContactId::SELF {
-            write!(f, "Contact#Self")
-        } else if *self == ContactId::INFO {
-            write!(f, "Contact#Info")
-        } else if *self == ContactId::DEVICE {
-            write!(f, "Contact#Device")
-        } else if self.is_special() {
-            write!(f, "Contact#Special{}", self.0)
-        } else {
-            write!(f, "Contact#{}", self.0)
-        }
+        write!(f, "{}", self.0)
+        // if *self == DC_CONTACT_ID_UNDEFINED {
+        //     write!(f, "Contact#Undefined")
+        // } else if *self == DC_CONTACT_ID_SELF {
+        //     write!(f, "Contact#Self")
+        // } else if *self == DC_CONTACT_ID_INFO {
+        //     write!(f, "Contact#Info")
+        // } else if *self == DC_CONTACT_ID_DEVICE {
+        //     write!(f, "Contact#Device")
+        // } else if *self <= DC_CONTACT_ID_LAST_SPECIAL {
+        //     write!(f, "Contact#Special{}", self.0)
+        // } else {
+        //     write!(f, "Contact#{}", self.0)
+        // }
     }
 }
 
@@ -127,6 +104,12 @@ impl rusqlite::types::FromSql for ContactId {
 #[derive(Debug)]
 pub struct Contact {
     /// The contact ID.
+    ///
+    /// Special message IDs:
+    /// - DC_CONTACT_ID_SELF (1) - this is the owner of the account with the email-address set by
+    ///   `dc_set_config` using "addr".
+    ///
+    /// Normal contact IDs are larger than these special ones (larger than DC_CONTACT_ID_LAST_SPECIAL).
     pub id: ContactId,
 
     /// Contact name. It is recommended to use `Contact::get_name`,
@@ -295,7 +278,7 @@ impl Contact {
                 },
             )
             .await?;
-        if contact_id == ContactId::SELF {
+        if contact_id == DC_CONTACT_ID_SELF {
             contact.name = stock_str::self_msg(context).await;
             contact.addr = context
                 .get_config(Config::ConfiguredAddr)
@@ -305,9 +288,9 @@ impl Contact {
                 .get_config(Config::Selfstatus)
                 .await?
                 .unwrap_or_default();
-        } else if contact_id == ContactId::DEVICE {
+        } else if contact_id == DC_CONTACT_ID_DEVICE {
             contact.name = stock_str::device_messages(context).await;
-            contact.addr = ContactId::DEVICE_ADDR.to_string();
+            contact.addr = DC_CONTACT_ID_DEVICE_ADDR.to_string();
             contact.status = stock_str::device_messages_hint(context).await;
         }
         Ok(contact)
@@ -399,17 +382,22 @@ impl Contact {
 
         let addr_normalized = addr_normalize(addr);
 
-        if context.is_self_addr(addr_normalized).await? {
-            return Ok(Some(ContactId::SELF));
+        if let Some(addr_self) = context.get_config(Config::ConfiguredAddr).await? {
+            if addr_cmp(addr_normalized, &addr_self) {
+                return Ok(Some(DC_CONTACT_ID_SELF));
+            }
         }
-
         let id = context
             .sql
             .query_get_value(
                 "SELECT id FROM contacts \
             WHERE addr=?1 COLLATE NOCASE \
             AND id>?2 AND origin>=?3 AND blocked=0;",
-                paramsv![addr_normalized, ContactId::LAST_SPECIAL, min_origin as u32,],
+                paramsv![
+                    addr_normalized,
+                    DC_CONTACT_ID_LAST_SPECIAL,
+                    min_origin as u32,
+                ],
             )
             .await?;
         Ok(id)
@@ -452,9 +440,13 @@ impl Contact {
         ensure!(origin != Origin::Unknown, "Missing valid origin");
 
         let addr = addr_normalize(addr).to_string();
+        let addr_self = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .unwrap_or_default();
 
-        if context.is_self_addr(&addr).await? {
-            return Ok((ContactId::SELF, sth_modified));
+        if addr_cmp(&addr, &addr_self) {
+            return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
         if !may_be_valid_addr(&addr) {
@@ -688,7 +680,11 @@ impl Contact {
         listflags: u32,
         query: Option<impl AsRef<str>>,
     ) -> Result<Vec<ContactId>> {
-        let self_addrs = context.get_all_self_addrs().await?;
+        let self_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .unwrap_or_default();
+
         let mut add_self = false;
         let mut ret = Vec::new();
         let flag_verified_only = (listflags & DC_GCL_VERIFIED_ONLY) != 0;
@@ -699,25 +695,23 @@ impl Contact {
             context
                 .sql
                 .query_map(
-                    format!(
-                        "SELECT c.id FROM contacts c \
+                    "SELECT c.id FROM contacts c \
                  LEFT JOIN acpeerstates ps ON c.addr=ps.addr  \
-                 WHERE c.addr NOT IN ({})
-                 AND c.id>? \
-                 AND c.origin>=? \
+                 WHERE c.addr!=?1 \
+                 AND c.id>?2 \
+                 AND c.origin>=?3 \
                  AND c.blocked=0 \
-                 AND (iif(c.name='',c.authname,c.name) LIKE ? OR c.addr LIKE ?) \
-                 AND (1=? OR LENGTH(ps.verified_key_fingerprint)!=0)  \
+                 AND (iif(c.name='',c.authname,c.name) LIKE ?4 OR c.addr LIKE ?5) \
+                 AND (1=?6 OR LENGTH(ps.verified_key_fingerprint)!=0)  \
                  ORDER BY LOWER(iif(c.name='',c.authname,c.name)||c.addr),c.id;",
-                        sql::repeat_vars(self_addrs.len())
-                    ),
-                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
-                        ContactId::LAST_SPECIAL,
+                    paramsv![
+                        self_addr,
+                        DC_CONTACT_ID_LAST_SPECIAL,
                         Origin::IncomingReplyTo,
                         s3str_like_cmd,
                         s3str_like_cmd,
                         if flag_verified_only { 0i32 } else { 1i32 },
-                    ])),
+                    ],
                     |row| row.get::<_, ContactId>(0),
                     |ids| {
                         for id in ids {
@@ -728,17 +722,13 @@ impl Contact {
                 )
                 .await?;
 
-            if let Some(query) = query {
-                let self_addr = context
-                    .get_config(Config::ConfiguredAddr)
-                    .await?
-                    .unwrap_or_default();
-                let self_name = context
-                    .get_config(Config::Displayname)
-                    .await?
-                    .unwrap_or_default();
-                let self_name2 = stock_str::self_msg(context);
+            let self_name = context
+                .get_config(Config::Displayname)
+                .await?
+                .unwrap_or_default();
+            let self_name2 = stock_str::self_msg(context);
 
+            if let Some(query) = query {
                 if self_addr.contains(query.as_ref())
                     || self_name.contains(query.as_ref())
                     || self_name2.await.contains(query.as_ref())
@@ -754,19 +744,17 @@ impl Contact {
             context
                 .sql
                 .query_map(
-                    format!(
-                        "SELECT id FROM contacts
-                 WHERE addr NOT IN ({})
-                 AND id>?
-                 AND origin>=?
+                    "SELECT id FROM contacts
+                 WHERE addr!=?1
+                 AND id>?2
+                 AND origin>=?3
                  AND blocked=0
                  ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                        sql::repeat_vars(self_addrs.len())
-                    ),
-                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
-                        ContactId::LAST_SPECIAL,
+                    paramsv![
+                        self_addr,
+                        DC_CONTACT_ID_LAST_SPECIAL,
                         Origin::IncomingReplyTo
-                    ])),
+                    ],
                     |row| row.get::<_, ContactId>(0),
                     |ids| {
                         for id in ids {
@@ -779,7 +767,7 @@ impl Contact {
         }
 
         if flag_add_self && add_self {
-            ret.push(ContactId::SELF);
+            ret.push(DC_CONTACT_ID_SELF);
         }
 
         Ok(ret)
@@ -834,7 +822,7 @@ impl Contact {
             .sql
             .count(
                 "SELECT COUNT(*) FROM contacts WHERE id>? AND blocked!=0",
-                paramsv![ContactId::LAST_SPECIAL],
+                paramsv![DC_CONTACT_ID_LAST_SPECIAL],
             )
             .await?;
         Ok(count as usize)
@@ -850,7 +838,7 @@ impl Contact {
             .sql
             .query_map(
                 "SELECT id FROM contacts WHERE id>? AND blocked!=0 ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                paramsv![ContactId::LAST_SPECIAL],
+                paramsv![DC_CONTACT_ID_LAST_SPECIAL],
                 |row| row.get::<_, ContactId>(0),
                 |ids| {
                     ids.collect::<std::result::Result<Vec<_>, _>>()
@@ -868,13 +856,13 @@ impl Contact {
     /// fingerprints of the keys involved.
     pub async fn get_encrinfo(context: &Context, contact_id: ContactId) -> Result<String> {
         ensure!(
-            !contact_id.is_special(),
+            contact_id > DC_CONTACT_ID_LAST_SPECIAL,
             "Can not provide encryption info for special contact"
         );
 
         let mut ret = String::new();
         if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            let loginparam = LoginParam::load_configured_params(context).await?;
+            let loginparam = LoginParam::from_database(context, "configured_").await?;
             let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
 
             if let Some(peerstate) = peerstate.filter(|peerstate| {
@@ -936,7 +924,10 @@ impl Contact {
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
     pub async fn delete(context: &Context, contact_id: ContactId) -> Result<()> {
-        ensure!(!contact_id.is_special(), "Can not delete special contact");
+        ensure!(
+            contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+            "Can not delete special contact"
+        );
 
         let count_chats = context
             .sql
@@ -972,7 +963,7 @@ impl Contact {
 
     /// Get a single contact object.  For a list, see eg. dc_get_contacts().
     ///
-    /// For contact ContactId::SELF (1), the function returns sth.
+    /// For contact DC_CONTACT_ID_SELF (1), the function returns sth.
     /// like "Me" in the selected language and the email address
     /// defined by dc_set_config().
     pub async fn get_by_id(context: &Context, contact_id: ContactId) -> Result<Contact> {
@@ -1065,7 +1056,7 @@ impl Contact {
     /// This is the image set by each remote user on their own
     /// using dc_set_config(context, "selfavatar", image).
     pub async fn get_profile_image(&self, context: &Context) -> Result<Option<PathBuf>> {
-        if self.id == ContactId::SELF {
+        if self.id == DC_CONTACT_ID_SELF {
             if let Some(p) = context.get_config(Config::Selfavatar).await? {
                 return Ok(Some(PathBuf::from(p)));
             }
@@ -1111,7 +1102,7 @@ impl Contact {
     ) -> Result<VerifiedStatus> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
-        if self.id == ContactId::SELF {
+        if self.id == DC_CONTACT_ID_SELF {
             return Ok(VerifiedStatus::BidirectVerified);
         }
 
@@ -1130,6 +1121,26 @@ impl Contact {
         Ok(VerifiedStatus::Unverified)
     }
 
+    pub async fn addr_equals_contact(
+        context: &Context,
+        addr: &str,
+        contact_id: ContactId,
+    ) -> Result<bool> {
+        if addr.is_empty() {
+            return Ok(false);
+        }
+
+        let contact = Contact::load_from_db(context, contact_id).await?;
+        if !contact.addr.is_empty() {
+            let normalized_addr = addr_normalize(addr);
+            if contact.addr == normalized_addr {
+                return Ok(true);
+            }
+        }
+
+        Ok(false)
+    }
+
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
         if !context.sql.is_open().await {
             return Ok(0);
@@ -1139,14 +1150,14 @@ impl Contact {
             .sql
             .count(
                 "SELECT COUNT(*) FROM contacts WHERE id>?;",
-                paramsv![ContactId::LAST_SPECIAL],
+                paramsv![DC_CONTACT_ID_LAST_SPECIAL],
             )
             .await?;
         Ok(count)
     }
 
     pub async fn real_exists_by_id(context: &Context, contact_id: ContactId) -> Result<bool> {
-        if contact_id.is_special() {
+        if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
             return Ok(false);
         }
 
@@ -1219,7 +1230,7 @@ async fn set_block_contact(
     new_blocking: bool,
 ) -> Result<()> {
     ensure!(
-        !contact_id.is_special(),
+        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
         "Can't block special contact {}",
         contact_id
     );
@@ -1289,7 +1300,7 @@ pub(crate) async fn set_profile_image(
     let mut contact = Contact::load_from_db(context, contact_id).await?;
     let changed = match profile_image {
         AvatarAction::Change(profile_image) => {
-            if contact_id == ContactId::SELF {
+            if contact_id == DC_CONTACT_ID_SELF {
                 if was_encrypted {
                     context
                         .set_config(Config::Selfavatar, Some(profile_image))
@@ -1303,7 +1314,7 @@ pub(crate) async fn set_profile_image(
             true
         }
         AvatarAction::Delete => {
-            if contact_id == ContactId::SELF {
+            if contact_id == DC_CONTACT_ID_SELF {
                 if was_encrypted {
                     context.set_config(Config::Selfavatar, None).await?;
                 } else {
@@ -1334,7 +1345,7 @@ pub(crate) async fn set_status(
     encrypted: bool,
     has_chat_version: bool,
 ) -> Result<()> {
-    if contact_id == ContactId::SELF {
+    if contact_id == DC_CONTACT_ID_SELF {
         if encrypted && has_chat_version {
             context
                 .set_config(Config::Selfstatus, Some(&status))
@@ -1359,7 +1370,7 @@ pub(crate) async fn update_last_seen(
     timestamp: i64,
 ) -> Result<()> {
     ensure!(
-        !contact_id.is_special(),
+        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
         "Can not update special contact last seen timestamp"
     );
 
@@ -1420,6 +1431,17 @@ fn cat_fingerprint(
     }
 }
 
+impl Context {
+    /// determine whether the specified addr maps to the/a self addr
+    pub async fn is_self_addr(&self, addr: &str) -> Result<bool> {
+        if let Some(self_addr) = self.get_config(Config::ConfiguredAddr).await? {
+            Ok(addr_cmp(&self_addr, addr))
+        } else {
+            Ok(false)
+        }
+    }
+}
+
 pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
     let norm1 = addr_normalize(addr1).to_lowercase();
     let norm2 = addr_normalize(addr2).to_lowercase();
@@ -1452,17 +1474,6 @@ mod tests {
     use crate::message::Message;
     use crate::test_utils::{self, TestContext};
 
-    #[test]
-    fn test_contact_id_values() {
-        // Some FFI users need to have the values of these fixed, how naughty.  But let's
-        // make sure we don't modify them anyway.
-        assert_eq!(ContactId::UNDEFINED.to_u32(), 0);
-        assert_eq!(ContactId::SELF.to_u32(), 1);
-        assert_eq!(ContactId::INFO.to_u32(), 2);
-        assert_eq!(ContactId::DEVICE.to_u32(), 5);
-        assert_eq!(ContactId::LAST_SPECIAL.to_u32(), 9);
-    }
-
     #[test]
     fn test_may_be_valid_addr() {
         assert_eq!(may_be_valid_addr(""), false);
@@ -1515,8 +1526,6 @@ mod tests {
     async fn test_get_contacts() -> Result<()> {
         let context = TestContext::new().await;
 
-        assert!(context.get_all_self_addrs().await?.is_empty());
-
         // Bob is not in the contacts yet.
         let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).await?;
         assert_eq!(contacts.len(), 0);
@@ -1528,7 +1537,7 @@ mod tests {
             Origin::IncomingReplyTo,
         )
         .await?;
-        assert_ne!(id, ContactId::UNDEFINED);
+        assert_ne!(id, ContactId::new(0));
 
         let contact = Contact::load_from_db(&context.ctx, id).await.unwrap();
         assert_eq!(contact.get_name(), "");
@@ -1606,7 +1615,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bla foo", "one@eins.org", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_id(), contact_id);
@@ -1633,7 +1642,7 @@ mod tests {
             Contact::add_or_lookup(&t, "", "three@drei.sam", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_name(), "");
@@ -1673,7 +1682,7 @@ mod tests {
             Contact::add_or_lookup(&t, "", "alice@w.de", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_name(), "Wonderland, Alice");
@@ -1682,7 +1691,8 @@ mod tests {
         assert_eq!(contact.get_name_n_addr(), "Wonderland, Alice (alice@w.de)");
 
         // check SELF
-        let contact = Contact::load_from_db(&t, ContactId::SELF).await.unwrap();
+        let contact = Contact::load_from_db(&t, DC_CONTACT_ID_SELF).await.unwrap();
+        assert_eq!(DC_CONTACT_ID_SELF, ContactId::new(1));
         assert_eq!(contact.get_name(), stock_str::self_msg(&t).await);
         assert_eq!(contact.get_addr(), ""); // we're not configured
         assert!(!contact.is_blocked());
@@ -1692,7 +1702,7 @@ mod tests {
     async fn test_delete() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
-        assert!(Contact::delete(&alice, ContactId::SELF).await.is_err());
+        assert!(Contact::delete(&alice, DC_CONTACT_ID_SELF).await.is_err());
 
         // Create Bob contact
         let (contact_id, _) =
@@ -1725,7 +1735,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob1", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::Created);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob1");
@@ -1737,7 +1747,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob2", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob2");
@@ -1748,7 +1758,7 @@ mod tests {
         let contact_id = Contact::create(&t, "bob3", "bob@example.org")
             .await
             .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob2");
         assert_eq!(contact.get_name(), "bob3");
@@ -1759,7 +1769,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob4", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob4");
@@ -1773,7 +1783,7 @@ mod tests {
 
         // manually create "claire@example.org" without a given name
         let contact_id = Contact::create(&t, "", "claire@example.org").await.unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "");
         assert_eq!(contact.get_name(), "");
@@ -1947,7 +1957,7 @@ mod tests {
         let id = Contact::lookup_id_by_addr(&alice.ctx, "alice@example.org", Origin::Unknown)
             .await
             .unwrap();
-        assert_eq!(id, Some(ContactId::SELF));
+        assert_eq!(id, Some(DC_CONTACT_ID_SELF));
     }
 
     #[async_std::test]
@@ -1955,9 +1965,9 @@ mod tests {
         let alice = TestContext::new_alice().await;
 
         // Return error for special IDs
-        let encrinfo = Contact::get_encrinfo(&alice, ContactId::SELF).await;
+        let encrinfo = Contact::get_encrinfo(&alice, DC_CONTACT_ID_SELF).await;
         assert!(encrinfo.is_err());
-        let encrinfo = Contact::get_encrinfo(&alice, ContactId::DEVICE).await;
+        let encrinfo = Contact::get_encrinfo(&alice, DC_CONTACT_ID_DEVICE).await;
         assert!(encrinfo.is_err());
 
         let (contact_bob_id, _modified) =
@@ -2138,7 +2148,7 @@ Chat-Version: 1.0
 Date: Sun, 22 Mar 2020 22:37:55 +0000
 
 Hi."#;
-        dc_receive_imf(&alice, mime, false).await?;
+        dc_receive_imf(&alice, mime, "Inbox", false).await?;
         let msg = alice.get_last_msg().await;
 
         let timestamp = msg.get_timestamp();

src/context.rs

@@ -10,6 +10,7 @@ use async_std::{
     channel::{self, Receiver, Sender},
     path::{Path, PathBuf},
     sync::{Arc, Mutex, RwLock},
+    task,
 };
 
 use crate::chat::{get_chat_cnt, ChatId};
@@ -55,6 +56,7 @@ pub struct InnerContext {
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
+    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
     /// Recently loaded quota information, if any.
     /// Set to `None` if quota was never tried to load.
@@ -174,6 +176,7 @@ impl Context {
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
+            ephemeral_task: RwLock::new(None),
             quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
@@ -195,11 +198,6 @@ impl Context {
             return;
         }
 
-        if let Ok(false) = self.is_configured().await {
-            warn!(self, "can not start io on a context that is not configured");
-            return;
-        }
-
         {
             let l = &mut *self.inner.scheduler.write().await;
             if let Err(err) = l.start(self.clone()).await {
@@ -241,24 +239,6 @@ impl Context {
         });
     }
 
-    /// Emits a generic MsgsChanged event (without chat or message id)
-    pub fn emit_msgs_changed_without_ids(&self) {
-        self.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
-    }
-
-    /// Emits a MsgsChanged event with specified chat and message ids
-    pub fn emit_msgs_changed(&self, chat_id: ChatId, msg_id: MsgId) {
-        self.emit_event(EventType::MsgsChanged { chat_id, msg_id });
-    }
-
-    /// Emits an IncomingMsg event with specified chat and message ids
-    pub fn emit_incoming_msg(&self, chat_id: ChatId, msg_id: MsgId) {
-        self.emit_event(EventType::IncomingMsg { chat_id, msg_id });
-    }
-
     /// Returns a receiver for emitted events.
     ///
     /// Multiple emitters can be created, but note that in this case each emitted event will
@@ -334,9 +314,8 @@ impl Context {
 
     pub async fn get_info(&self) -> Result<BTreeMap<&'static str, String>> {
         let unset = "0";
-        let l = LoginParam::load_candidate_params(self).await?;
-        let l2 = LoginParam::load_configured_params(self).await?;
-        let secondary_addrs = self.get_secondary_self_addrs().await?.join(", ");
+        let l = LoginParam::from_database(self, "").await?;
+        let l2 = LoginParam::from_database(self, "configured_").await?;
         let displayname = self.get_config(Config::Displayname).await?;
         let chats = get_chat_cnt(self).await? as usize;
         let unblocked_msgs = message::get_unblocked_msg_cnt(self).await as usize;
@@ -421,7 +400,6 @@ impl Context {
         res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
-        res.insert("secondary_addrs", secondary_addrs);
         res.insert(
             "fetch_existing_msgs",
             self.get_config_int(Config::FetchExistingMsgs)
@@ -628,6 +606,11 @@ impl Context {
         Ok(mvbox.as_deref() == Some(folder_name))
     }
 
+    pub async fn is_spam_folder(&self, folder_name: &str) -> Result<bool> {
+        let spam = self.get_config(Config::ConfiguredSpamFolder).await?;
+        Ok(spam.as_deref() == Some(folder_name))
+    }
+
     pub(crate) fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -659,6 +642,10 @@ impl InnerContext {
                 lock.stop(token).await;
             }
         }
+
+        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
+            ephemeral_task.cancel().await;
+        }
     }
 }
 
@@ -683,7 +670,7 @@ mod tests {
     use crate::chat::{
         get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
     };
-    use crate::contact::ContactId;
+    use crate::constants::DC_CONTACT_ID_SELF;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::dc_tools::dc_create_outgoing_rfc724_mid;
     use crate::message::{Message, Viewtype};
@@ -729,7 +716,9 @@ mod tests {
             dc_create_outgoing_rfc724_mid(None, contact.get_addr())
         );
         println!("{}", msg);
-        dc_receive_imf(t, msg.as_bytes(), false).await.unwrap();
+        dc_receive_imf(t, msg.as_bytes(), "INBOX", false)
+            .await
+            .unwrap();
     }
 
     #[async_std::test]
@@ -967,7 +956,7 @@ mod tests {
     #[async_std::test]
     async fn test_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
-        let self_talk = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
+        let self_talk = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
         let chat = alice
             .create_chat_with_contact("Bob", "bob@example.org")
             .await;

src/dc_tools.rs

@@ -86,7 +86,7 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
 //   `last_smeared_timestamp` is again in sync with the normal time.
 // - however, we do not do all this for the far future,
 //   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
-pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
+const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 
 /// Returns the current smeared timestamp,
 ///
@@ -214,10 +214,7 @@ pub(crate) fn dc_create_id() -> String {
     rng.fill(&mut arr[..]);
 
     // Take 11 base64 characters containing 66 random bits.
-    base64::encode_config(&arr, base64::URL_SAFE)
-        .chars()
-        .take(11)
-        .collect()
+    base64::encode(&arr).chars().take(11).collect()
 }
 
 /// Function generates a Message-ID that can be used for a new outgoing message.
@@ -703,7 +700,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
     async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
-        dc_receive_imf(&t, raw, false).await.unwrap();
+        dc_receive_imf(&t, raw, "INBOX", false).await.unwrap();
         let msg = t.get_last_msg().await;
         let msg_info = get_msg_info(&t, msg.id).await.unwrap();
 
@@ -765,15 +762,6 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         assert_eq!(buf.len(), 11);
     }
 
-    #[test]
-    fn test_dc_create_id_invalid_chars() {
-        for _ in 1..1000 {
-            let buf = dc_create_id();
-            assert!(!buf.contains('/')); // `/` must not be used to be URL-safe
-            assert!(!buf.contains('.')); // `.` is used as a delimiter when extracting grpid from Message-ID
-        }
-    }
-
     #[test]
     fn test_dc_extract_grpid_from_rfc724_mid() {
         // Should return None if we pass invalid mid

src/download.rs

@@ -346,6 +346,7 @@ mod tests {
             &t,
             "Mr.12345678901@example.com",
             header.as_bytes(),
+            "INBOX",
             false,
             Some(100000),
             false,
@@ -363,6 +364,7 @@ mod tests {
             &t,
             "Mr.12345678901@example.com",
             format!("{}\n\n100k text...", header).as_bytes(),
+            "INBOX",
             false,
             None,
             false,
@@ -398,6 +400,7 @@ mod tests {
                     Message-ID: <first@example.org>\n\
                     Date: Sun, 14 Nov 2021 00:10:00 +0000\
                     Content-Type: text/plain",
+            "INBOX",
             false,
             Some(100000),
             false,

src/e2ee.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashSet;
 
-use anyhow::{format_err, Context as _, Result};
+use anyhow::{bail, format_err, Context as _, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
@@ -28,7 +28,13 @@ impl EncryptHelper {
         let prefer_encrypt =
             EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await?)
                 .unwrap_or_default();
-        let addr = context.get_primary_self_addr().await?;
+        let addr = match context.get_config(Config::ConfiguredAddr).await? {
+            None => {
+                bail!("addr not configured!");
+            }
+            Some(addr) => addr,
+        };
+
         let public_key = SignedPublicKey::load_self(context).await?;
 
         Ok(EncryptHelper {
@@ -381,7 +387,13 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 /// [Config::ConfiguredAddr] is configured, this address is returned.
 // TODO, remove this once deltachat::key::Key no longer exists.
 pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context.get_primary_self_addr().await?;
+    let self_addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await?
+        .context(concat!(
+            "Failed to get self address, ",
+            "cannot ensure secret key if not configured."
+        ))?;
     SignedPublicKey::load_self(context).await?;
     Ok(self_addr)
 }

src/ephemeral.rs

@@ -48,9 +48,9 @@
 //!
 //! ## When messages are deleted
 //!
-//! The `ephemeral_loop` task schedules the next due running of
-//! `delete_expired_messages` which in turn emits `MsgsChanged` events
-//! when deleting local messages to make UIs reload displayed messages.
+//! Local deletion happens when the chatlist or chat is loaded. A
+//! `MsgsChanged` event is emitted when a message deletion is due, to
+//! make UI reload displayed messages and cause actual deletion.
 //!
 //! Server deletion happens by updating the `imap` table based on
 //! the database entries which are expired either according to their
@@ -62,18 +62,18 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{ensure, Context as _, Result};
-use async_std::channel::Receiver;
-use async_std::future::timeout;
+use async_std::task;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{send_msg, ChatId};
-use crate::constants::{DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH};
+use crate::constants::{
+    DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
+};
 use crate::contact::ContactId;
 use crate::context::Context;
-use crate::dc_tools::{duration_to_str, time};
+use crate::dc_tools::time;
 use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
-use crate::log::LogExt;
 use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::sql;
@@ -198,7 +198,7 @@ impl ChatId {
         }
         self.inner_set_ephemeral_timer(context, timer).await?;
         let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(stock_ephemeral_timer_changed(context, timer, ContactId::SELF).await);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
         msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
         if let Err(err) = send_msg(context, self, &mut msg).await {
             error!(
@@ -293,7 +293,7 @@ impl MsgId {
                     paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
                 )
                 .await?;
-            context.interrupt_ephemeral_task().await;
+            schedule_ephemeral_task(context).await;
         }
         Ok(())
     }
@@ -315,7 +315,7 @@ pub(crate) async fn start_ephemeral_timers_msgids(
                 "UPDATE msgs SET ephemeral_timestamp = ? + ephemeral_timer
          WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ? + ephemeral_timer) AND ephemeral_timer > 0
          AND id IN ({})",
-                sql::repeat_vars(msg_ids.len())
+                sql::repeat_vars(msg_ids.len())?
             ),
             rusqlite::params_from_iter(
                 std::iter::once(&now as &dyn crate::ToSql)
@@ -325,7 +325,7 @@ pub(crate) async fn start_ephemeral_timers_msgids(
         )
         .await?;
     if count > 0 {
-        context.interrupt_ephemeral_task().await;
+        schedule_ephemeral_task(context).await;
     }
     Ok(())
 }
@@ -338,7 +338,7 @@ pub(crate) async fn start_ephemeral_timers_msgids(
 /// false. This function does not emit the MsgsChanged event itself,
 /// because it is also called when chatlist is reloaded, and emitting
 /// MsgsChanged there will cause infinite reload loop.
-pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Result<()> {
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool> {
     let mut updated = context
         .sql
         .execute(
@@ -354,21 +354,21 @@ WHERE
   AND ephemeral_timestamp <= ?
   AND chat_id != ?
 "#,
-            paramsv![DC_CHAT_ID_TRASH, now, DC_CHAT_ID_TRASH],
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
         )
         .await
         .context("update failed")?
         > 0;
 
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+        let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
             .await?
             .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
+        let device_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
             .await?
             .unwrap_or_default();
 
-        let threshold_timestamp = now.saturating_sub(delete_device_after);
+        let threshold_timestamp = time() - delete_device_after;
 
         // Delete expired messages
         //
@@ -378,8 +378,7 @@ WHERE
             .sql
             .execute(
                 "UPDATE msgs \
-             SET chat_id = ?, txt = '', subject='', txt_raw='', \
-                 mime_headers='', from_id=0, to_id=0, param='' \
+             SET txt = 'DELETED', chat_id = ? \
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
@@ -398,116 +397,72 @@ WHERE
         updated |= rows_modified > 0;
     }
 
-    if updated {
-        context.emit_msgs_changed_without_ids();
-    }
-
-    Ok(())
+    schedule_ephemeral_task(context).await;
+    Ok(updated)
 }
 
-/// Calculates the next timestamp when a message will be deleted due to
-/// `delete_device_after` setting being set.
-async fn next_delete_device_after_timestamp(context: &Context) -> Result<Option<i64>> {
-    if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
-            .await?
-            .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
-            .await?
-            .unwrap_or_default();
-
-        let oldest_message_timestamp: Option<i64> = context
-            .sql
-            .query_get_value(
-                r#"
-                SELECT min(timestamp)
-                FROM msgs
-                WHERE chat_id > ?
-                  AND chat_id != ?
-                  AND chat_id != ?;
-                "#,
-                paramsv![DC_CHAT_ID_TRASH, self_chat_id, device_chat_id],
-            )
-            .await?;
-
-        Ok(oldest_message_timestamp.map(|x| x.saturating_add(delete_device_after)))
-    } else {
-        Ok(None)
-    }
-}
-
-/// Calculates next timestamp when expiration of some message will happen.
+/// Schedule a task to emit MsgsChanged event when the next local
+/// deletion happens. Existing task is cancelled to make sure at most
+/// one such task is scheduled at a time.
 ///
-/// Expiration can happen either because user has set `delete_device_after` setting or because the
-/// message itself has an ephemeral timer.
-async fn next_expiration_timestamp(context: &Context) -> Option<i64> {
+/// UI is expected to reload the chatlist or the chat in response to
+/// MsgsChanged event, this will trigger actual deletion.
+///
+/// This takes into account only per-chat timeouts, because global device
+/// timeouts are at least one hour long and deletion is triggered often enough
+/// by user actions.
+pub async fn schedule_ephemeral_task(context: &Context) {
     let ephemeral_timestamp: Option<i64> = match context
         .sql
         .query_get_value(
             r#"
-            SELECT min(ephemeral_timestamp)
-            FROM msgs
-            WHERE ephemeral_timestamp != 0
-              AND chat_id != ?;
-            "#,
+    SELECT ephemeral_timestamp
+    FROM msgs
+    WHERE ephemeral_timestamp != 0
+      AND chat_id != ?
+    ORDER BY ephemeral_timestamp ASC
+    LIMIT 1;
+    "#,
             paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
         )
         .await
     {
         Err(err) => {
             warn!(context, "Can't calculate next ephemeral timeout: {}", err);
-            None
+            return;
         }
         Ok(ephemeral_timestamp) => ephemeral_timestamp,
     };
 
-    let delete_device_after_timestamp: Option<i64> =
-        match next_delete_device_after_timestamp(context).await {
-            Err(err) => {
-                warn!(
-                    context,
-                    "Can't calculate timestamp of the next message expiration: {}", err
-                );
-                None
-            }
-            Ok(timestamp) => timestamp,
-        };
-
-    ephemeral_timestamp
-        .into_iter()
-        .chain(delete_device_after_timestamp.into_iter())
-        .min()
-}
-
-pub(crate) async fn ephemeral_loop(context: &Context, interrupt_receiver: Receiver<()>) {
-    loop {
-        let ephemeral_timestamp = next_expiration_timestamp(context).await;
+    // Cancel existing task, if any
+    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
+        ephemeral_task.cancel().await;
+    }
 
+    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
         let now = SystemTime::now();
-        let until = if let Some(ephemeral_timestamp) = ephemeral_timestamp {
-            UNIX_EPOCH
-                + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
-                + Duration::from_secs(1)
-        } else {
-            // no messages to be deleted for now, wait long for one to occur
-            now + Duration::from_secs(86400)
-        };
+        let until = UNIX_EPOCH
+            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+            + Duration::from_secs(1);
 
         if let Ok(duration) = until.duration_since(now) {
-            info!(
-                context,
-                "Ephemeral loop waiting for deletion in {} or interrupt",
-                duration_to_str(duration)
-            );
-            if timeout(duration, interrupt_receiver.recv()).await.is_ok() {
-                // received an interruption signal, recompute waiting time (if any)
-                continue;
-            }
+            // Schedule a task, ephemeral_timestamp is in the future
+            let context1 = context.clone();
+            let ephemeral_task = task::spawn(async move {
+                async_std::task::sleep(duration).await;
+                context1.emit_event(EventType::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0),
+                });
+            });
+            *context.ephemeral_task.write().await = Some(ephemeral_task);
+        } else {
+            // Emit event immediately
+            context.emit_event(EventType::MsgsChanged {
+                chat_id: ChatId::new(0),
+                msg_id: MsgId::new(0),
+            });
         }
-
-        delete_expired_messages(context, time())
-            .await
-            .ok_or_log(context);
     }
 }
 
@@ -577,7 +532,6 @@ mod tests {
     use super::*;
     use crate::config::Config;
     use crate::dc_receive_imf::dc_receive_imf;
-    use crate::dc_tools::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
@@ -590,7 +544,7 @@ mod tests {
         let context = TestContext::new().await;
 
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, ContactId::SELF).await,
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
             "Message deletion timer is disabled by me."
         );
 
@@ -598,7 +552,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 1 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 s by me."
@@ -607,7 +561,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 30 s by me."
@@ -616,7 +570,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 minute by me."
@@ -625,7 +579,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 90 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1.5 minutes by me."
@@ -634,7 +588,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 * 60 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 30 minutes by me."
@@ -643,7 +597,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 * 60 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 hour by me."
@@ -652,7 +606,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 5400 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1.5 hours by me."
@@ -663,7 +617,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 2 hours by me."
@@ -674,7 +628,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 day by me."
@@ -685,7 +639,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 24 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 2 days by me."
@@ -696,7 +650,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 week by me."
@@ -707,7 +661,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 4 weeks by me."
@@ -866,106 +820,31 @@ mod tests {
     #[async_std::test]
     async fn test_ephemeral_delete_msgs() -> Result<()> {
         let t = TestContext::new_alice().await;
-        let self_chat = t.get_self_chat().await;
+        let chat = t.get_self_chat().await;
 
-        assert_eq!(next_expiration_timestamp(&t).await, None);
-
-        t.send_text(self_chat.id, "Saved message, which we delete manually")
+        t.send_text(chat.id, "Saved message, which we delete manually")
             .await;
-        let msg = t.get_last_msg_in(self_chat.id).await;
+        let msg = t.get_last_msg_in(chat.id).await;
         msg.id.delete_from_db(&t).await?;
-        check_msg_is_deleted(&t, &self_chat, msg.id).await;
+        check_msg_was_deleted(&t, &chat, msg.id).await;
 
-        self_chat
-            .id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 3600 })
+        chat.id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
             .await
             .unwrap();
+        let msg = t
+            .send_text(chat.id, "Saved message, disappearing after 1s")
+            .await;
 
-        // Send a saved message which will be deleted after 3600s
-        let now = time();
-        let msg = t.send_text(self_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3599, time() + 3601)
-            .await
-            .unwrap();
-
-        // Set DeleteDeviceAfter to 1800s. Thend send a saved message which will
-        // still be deleted after 3600s because DeleteDeviceAfter doesn't apply to saved messages.
-        t.set_config(Config::DeleteDeviceAfter, Some("1800"))
-            .await?;
-
-        let now = time();
-        let msg = t.send_text(self_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3559, time() + 3601)
-            .await
-            .unwrap();
-
-        // Send a message to Bob which will be deleted after 1800s because of DeleteDeviceAfter.
-        let bob_chat = t.create_chat_with_contact("", "bob@example.net").await;
-        let now = time();
-        let msg = t.send_text(bob_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(
-            &t,
-            msg.sender_msg_id,
-            &bob_chat,
-            now + 1799,
-            // The message may appear to be sent MAX_SECONDS_TO_LEND_FROM_FUTURE later and
-            // therefore be deleted MAX_SECONDS_TO_LEND_FROM_FUTURE later.
-            time() + 1801 + MAX_SECONDS_TO_LEND_FROM_FUTURE,
-        )
-        .await
-        .unwrap();
-
-        // Enable ephemeral messages with Bob -> message will be deleted after 60s.
-        // This tests that the message is deleted at min(ephemeral deletion time, DeleteDeviceAfter deletion time).
-        bob_chat
-            .id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
-            .await?;
-
-        let now = time();
-        let msg = t.send_text(bob_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(&t, msg.sender_msg_id, &bob_chat, now + 59, time() + 61)
-            .await
-            .unwrap();
-
-        Ok(())
-    }
-
-    async fn check_msg_will_be_deleted(
-        t: &TestContext,
-        msg_id: MsgId,
-        chat: &Chat,
-        not_deleted_at: i64,
-        deleted_at: i64,
-    ) -> Result<()> {
-        let next_expiration = next_expiration_timestamp(t).await.unwrap();
-
-        assert!(next_expiration > not_deleted_at);
-        delete_expired_messages(t, not_deleted_at).await?;
-
-        let loaded = Message::load_from_db(t, msg_id).await?;
-        assert_eq!(loaded.text.unwrap(), "Message text");
-        assert_eq!(loaded.chat_id, chat.id);
-
-        assert!(next_expiration < deleted_at);
-        delete_expired_messages(t, deleted_at).await?;
-
-        let loaded = Message::load_from_db(t, msg_id).await?;
-        assert_eq!(loaded.text.unwrap(), "");
-        assert_eq!(loaded.chat_id, DC_CHAT_ID_TRASH);
+        async_std::task::sleep(Duration::from_millis(1100)).await;
 
         // Check that the msg was deleted locally.
-        check_msg_is_deleted(t, chat, msg_id).await;
+        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
 
         Ok(())
     }
 
-    async fn check_msg_is_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
         let chat_items = chat::get_chat_msgs(t, chat.id, 0, None).await.unwrap();
         // Check that the chat is empty except for possibly info messages:
         for item in &chat_items {
@@ -977,8 +856,8 @@ mod tests {
 
         // Check that if there is a message left, the text and metadata are gone
         if let Ok(msg) = Message::load_from_db(t, msg_id).await {
-            assert_eq!(msg.from_id, ContactId::UNDEFINED);
-            assert_eq!(msg.to_id, ContactId::UNDEFINED);
+            assert_eq!(msg.from_id, ContactId::new(0));
+            assert_eq!(msg.to_id, ContactId::new(0));
             assert!(msg.text.is_none_or_empty(), "{:?}", msg.text);
             let rawtxt: Option<String> = t
                 .sql
@@ -1115,6 +994,7 @@ mod tests {
                     Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
                     \n\
                     hello\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -1135,6 +1015,7 @@ mod tests {
                     Ephemeral-Timer: 60\n\
                     \n\
                     second message\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -1171,6 +1052,7 @@ mod tests {
                     In-Reply-To: <first@example.com>\n\
                     \n\
                     > hello\n",
+            "INBOX",
             false,
         )
         .await?;

src/events.rs

@@ -10,7 +10,6 @@ use crate::chat::ChatId;
 use crate::contact::ContactId;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
-use crate::webxdc::StatusUpdateSerial;
 
 #[derive(Debug)]
 pub struct Events {
@@ -336,8 +335,5 @@ pub enum EventType {
     SelfavatarChanged,
 
     #[strum(props(id = "2120"))]
-    WebxdcStatusUpdate {
-        msg_id: MsgId,
-        status_update_serial: StatusUpdateSerial,
-    },
+    WebxdcStatusUpdate(MsgId),
 }

src/html.rs

@@ -279,7 +279,7 @@ mod tests {
     use crate::chat;
     use crate::chat::forward_msgs;
     use crate::config::Config;
-    use crate::contact::ContactId;
+    use crate::constants::DC_CONTACT_ID_SELF;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::message::{MessengerMessage, Viewtype};
     use crate::test_utils::TestContext;
@@ -440,9 +440,9 @@ test some special html-characters as < > and & but also " and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, false).await.unwrap();
+        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), ContactId::SELF);
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
         assert!(!msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -456,7 +456,7 @@ test some special html-characters as < > and & but also " and &#x
             .await
             .unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), ContactId::SELF);
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -469,7 +469,7 @@ test some special html-characters as < > and & but also " and &#x
         let chat = bob.create_chat_with_contact("", "alice@example.org").await;
         bob.recv_msg(&alice.pop_sent_msg().await).await;
         let msg = bob.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), ContactId::SELF);
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -489,7 +489,7 @@ test some special html-characters as < > and & but also " and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, false).await.unwrap();
+        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
 
         // forward the message to saved-messages,
@@ -506,7 +506,7 @@ test some special html-characters as < > and & but also " and &#x
         alice.recv_msg(&msg).await;
         let chat = alice.get_self_chat().await;
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), ContactId::SELF);
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.get_showpadlock());
         assert!(msg.is_forwarded());
@@ -555,6 +555,7 @@ test some special html-characters as < > and & but also " and &#x
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/cp1252-html.eml"),
+            "INBOX",
             false,
         )
         .await?;

src/imap.rs

@@ -7,7 +7,6 @@ use std::{
     cmp,
     cmp::max,
     collections::{BTreeMap, BTreeSet},
-    iter::Peekable,
 };
 
 use anyhow::{bail, format_err, Context as _, Result};
@@ -21,10 +20,9 @@ use num_traits::FromPrimitive;
 use crate::chat::{self, ChatId, ChatIdBlocked};
 use crate::config::Config;
 use crate::constants::{
-    Blocked, Chattype, ShowEmails, DC_FETCH_EXISTING_MSGS_COUNT, DC_FOLDERS_CONFIGURED_VERSION,
-    DC_LP_AUTH_OAUTH2,
+    Blocked, Chattype, ShowEmails, DC_CONTACT_ID_SELF, DC_FETCH_EXISTING_MSGS_COUNT,
+    DC_FOLDERS_CONFIGURED_VERSION, DC_LP_AUTH_OAUTH2,
 };
-use crate::contact::ContactId;
 use crate::context::Context;
 use crate::dc_receive_imf::{
     dc_receive_imf_inner, from_field_to_contact_id, get_prefetch_parent_message, ReceivedMsg,
@@ -32,13 +30,14 @@ use crate::dc_receive_imf::{
 use crate::dc_tools::dc_create_id;
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::job;
+use crate::job::{self, Action};
 use crate::login_param::{
     CertificateChecks, LoginParam, ServerAddress, ServerLoginParam, Socks5Config,
 };
 use crate::message::{self, Message, MessageState, MessengerMessage, MsgId, Viewtype};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
+use crate::param::Params;
 use crate::provider::Socket;
 use crate::scheduler::connectivity::ConnectivityStore;
 use crate::scheduler::InterruptInfo;
@@ -131,7 +130,7 @@ impl FolderMeaning {
     fn to_config(self) -> Option<Config> {
         match self {
             FolderMeaning::Unknown => None,
-            FolderMeaning::Spam => None,
+            FolderMeaning::Spam => Some(Config::ConfiguredSpamFolder),
             FolderMeaning::Sent => Some(Config::ConfiguredSentboxFolder),
             FolderMeaning::Drafts => None,
             FolderMeaning::Other => None,
@@ -165,67 +164,6 @@ struct ImapConfig {
     pub can_condstore: bool,
 }
 
-struct UidGrouper<T: Iterator<Item = (i64, u32, String)>> {
-    inner: Peekable<T>,
-}
-
-impl<T, I> From<I> for UidGrouper<T>
-where
-    T: Iterator<Item = (i64, u32, String)>,
-    I: IntoIterator<IntoIter = T>,
-{
-    fn from(inner: I) -> Self {
-        Self {
-            inner: inner.into_iter().peekable(),
-        }
-    }
-}
-
-impl<T: Iterator<Item = (i64, u32, String)>> Iterator for UidGrouper<T> {
-    // Tuple of folder, row IDs, and UID range as a string.
-    type Item = (String, Vec<i64>, String);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let (_, _, folder) = self.inner.peek().cloned()?;
-
-        let mut uid_set = String::new();
-        let mut rowid_set = Vec::new();
-
-        while uid_set.len() < 1000 {
-            // Construct a new range.
-            if let Some((start_rowid, start_uid, _)) = self
-                .inner
-                .next_if(|(_, _, start_folder)| start_folder == &folder)
-            {
-                rowid_set.push(start_rowid);
-                let mut end_uid = start_uid;
-
-                while let Some((next_rowid, next_uid, _)) =
-                    self.inner.next_if(|(_, next_uid, next_folder)| {
-                        next_folder == &folder && *next_uid == end_uid + 1
-                    })
-                {
-                    end_uid = next_uid;
-                    rowid_set.push(next_rowid);
-                }
-
-                let uid_range = UidRange {
-                    start: start_uid,
-                    end: end_uid,
-                };
-                if !uid_set.is_empty() {
-                    uid_set.push(',');
-                }
-                uid_set.push_str(&uid_range.to_string());
-            } else {
-                break;
-            }
-        }
-
-        Some((folder, rowid_set, uid_set))
-    }
-}
-
 impl Imap {
     /// Creates new disconnected IMAP client using the specific login parameters.
     ///
@@ -285,7 +223,7 @@ impl Imap {
             bail!("IMAP Connect without configured params");
         }
 
-        let param = LoginParam::load_configured_params(context).await?;
+        let param = LoginParam::from_database(context, "configured_").await?;
         // the trailing underscore is correct
 
         let imap = Self::new(
@@ -512,29 +450,16 @@ impl Imap {
     ///
     /// Prefetches headers and downloads new message from the folder, moves messages away from the
     /// folder and deletes messages in the folder.
-    pub async fn fetch_move_delete(
-        &mut self,
-        context: &Context,
-        watch_folder: &str,
-        is_spam_folder: bool,
-    ) -> Result<()> {
+    pub async fn fetch_move_delete(&mut self, context: &Context, watch_folder: &str) -> Result<()> {
         if !context.sql.is_open().await {
             // probably shutdown
             bail!("IMAP operation attempted while it is torn down");
         }
         self.prepare(context).await?;
 
-        let msgs_fetched = self
-            .fetch_new_messages(context, watch_folder, is_spam_folder, false)
+        self.fetch_new_messages(context, watch_folder, false)
             .await
             .context("fetch_new_messages")?;
-        if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
-            // New messages were fetched and shall be deleted later, restart ephemeral loop.
-            // Note that the `Config::DeleteDeviceAfter` timer starts as soon as the messages are
-            // fetched while the per-chat ephemeral timers start as soon as the messages are marked
-            // as noticed.
-            context.interrupt_ephemeral_task().await;
-        }
 
         self.move_delete_messages(context, watch_folder)
             .await
@@ -732,17 +657,13 @@ impl Imap {
         Ok(false)
     }
 
-    /// Fetches new messages.
-    ///
-    /// Returns true if at least one message was fetched.
     pub(crate) async fn fetch_new_messages(
         &mut self,
         context: &Context,
         folder: &str,
-        is_spam_folder: bool,
         fetch_existing_msgs: bool,
     ) -> Result<bool> {
-        if should_ignore_folder(context, folder, is_spam_folder).await? {
+        if should_ignore_folder(context, folder).await? {
             info!(context, "Not fetching from {}", folder);
             return Ok(false);
         }
@@ -785,7 +706,7 @@ impl Imap {
             // Get the Message-ID or generate a fake one to identify the message in the database.
             let message_id = prefetch_get_message_id(&headers).unwrap_or_else(dc_create_id);
 
-            let target = match target_folder(context, folder, is_spam_folder, &headers).await? {
+            let target = match target_folder(context, folder, &headers).await? {
                 Some(config) => match context.get_config(config).await? {
                     Some(target) => target,
                     None => folder.to_string(),
@@ -816,7 +737,7 @@ impl Imap {
                 // If the sender is known, the message will be moved to the Inbox or Mvbox
                 // and then we download the message from there.
                 // Also see `spam_target_folder()`.
-                && !is_spam_folder
+                && !context.is_spam_folder(folder).await?
                 && prefetch_should_download(
                     context,
                     &headers,
@@ -912,7 +833,7 @@ impl Imap {
             .execute(
                 format!(
                     "DELETE FROM imap WHERE id IN ({})",
-                    sql::repeat_vars(row_ids.len())
+                    sql::repeat_vars(row_ids.len())?
                 ),
                 rusqlite::params_from_iter(row_ids),
             )
@@ -949,7 +870,7 @@ impl Imap {
                         .execute(
                             format!(
                                 "DELETE FROM imap WHERE id IN ({})",
-                                sql::repeat_vars(row_ids.len())
+                                sql::repeat_vars(row_ids.len())?
                             ),
                             rusqlite::params_from_iter(row_ids),
                         )
@@ -991,7 +912,7 @@ impl Imap {
                     .execute(
                         format!(
                             "UPDATE imap SET target='' WHERE id IN ({})",
-                            sql::repeat_vars(row_ids.len())
+                            sql::repeat_vars(row_ids.len())?
                         ),
                         rusqlite::params_from_iter(row_ids),
                     )
@@ -1011,7 +932,7 @@ impl Imap {
     ///
     /// This is the only place where messages are moved or deleted on the IMAP server.
     async fn move_delete_messages(&mut self, context: &Context, folder: &str) -> Result<()> {
-        let rows = context
+        let mut rows = context
             .sql
             .query_map(
                 "SELECT id, uid, target FROM imap
@@ -1027,12 +948,48 @@ impl Imap {
                 },
                 |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
             )
-            .await?;
+            .await?
+            .into_iter()
+            .peekable();
 
         self.prepare(context).await?;
         self.select_folder(context, Some(folder)).await?;
 
-        for (target, rowid_set, uid_set) in UidGrouper::from(rows) {
+        while let Some((_, _, target)) = rows.peek().cloned() {
+            // Construct next request for the target folder.
+            let mut uid_set = String::new();
+            let mut rowid_set = Vec::new();
+
+            while uid_set.len() < 1000 {
+                // Construct a new range.
+                if let Some((start_rowid, start_uid, _)) =
+                    rows.next_if(|(_, _, start_target)| start_target == &target)
+                {
+                    rowid_set.push(start_rowid);
+                    let mut end_uid = start_uid;
+
+                    while let Some((next_rowid, next_uid, _)) =
+                        rows.next_if(|(_, next_uid, next_target)| {
+                            next_target == &target && *next_uid == end_uid + 1
+                        })
+                    {
+                        end_uid = next_uid;
+                        rowid_set.push(next_rowid);
+                    }
+
+                    let uid_range = UidRange {
+                        start: start_uid,
+                        end: end_uid,
+                    };
+                    if !uid_set.is_empty() {
+                        uid_set.push(',');
+                    }
+                    uid_set.push_str(&uid_range.to_string());
+                } else {
+                    break;
+                }
+            }
+
             // Empty target folder name means messages should be deleted.
             if target.is_empty() {
                 self.delete_message_batch(context, &uid_set, rowid_set)
@@ -1059,62 +1016,6 @@ impl Imap {
         Ok(())
     }
 
-    /// Stores pending `\Seen` flags for messages in `imap_markseen` table.
-    pub(crate) async fn store_seen_flags_on_imap(&mut self, context: &Context) -> Result<()> {
-        self.prepare(context).await?;
-
-        let rows = context
-            .sql
-            .query_map(
-                "SELECT imap.id, uid, folder FROM imap, imap_markseen
-                 WHERE imap.id = imap_markseen.id AND target = folder
-                 ORDER BY folder, uid",
-                [],
-                |row| {
-                    let rowid: i64 = row.get(0)?;
-                    let uid: u32 = row.get(1)?;
-                    let folder: String = row.get(2)?;
-                    Ok((rowid, uid, folder))
-                },
-                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
-            )
-            .await?;
-
-        for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
-            self.select_folder(context, Some(&folder))
-                .await
-                .context("failed to select folder")?;
-
-            if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
-                warn!(
-                    context,
-                    "Cannot mark messages {} in folder {} as seen, will retry later: {}.",
-                    uid_set,
-                    folder,
-                    err
-                );
-            } else {
-                info!(
-                    context,
-                    "Marked messages {} in folder {} as seen.", uid_set, folder
-                );
-                context
-                    .sql
-                    .execute(
-                        format!(
-                            "DELETE FROM imap_markseen WHERE id IN ({})",
-                            sql::repeat_vars(rowid_set.len())
-                        ),
-                        rusqlite::params_from_iter(rowid_set),
-                    )
-                    .await
-                    .context("cannot remove messages marked as seen from imap_markseen table")?;
-            }
-        }
-
-        Ok(())
-    }
-
     /// Synchronizes `\Seen` flags using `CONDSTORE` extension.
     pub(crate) async fn sync_seen_flags(&mut self, context: &Context, folder: &str) -> Result<()> {
         if !self.config.can_condstore {
@@ -1206,9 +1107,14 @@ impl Imap {
             .session
             .as_mut()
             .context("IMAP No Connection established")?;
+        let self_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .context("not configured")?;
 
+        let search_command = format!("FROM \"{}\"", self_addr);
         let uids = session
-            .uid_search(get_imap_self_sent_search_command(context).await?)
+            .uid_search(search_command)
             .await?
             .into_iter()
             .collect();
@@ -1355,6 +1261,8 @@ impl Imap {
                 }
             };
 
+            let folder = folder.to_string();
+
             while let Some(Ok(msg)) = msgs.next().await {
                 let server_uid = msg.uid.unwrap_or_default();
 
@@ -1388,6 +1296,7 @@ impl Imap {
 
                 // XXX put flags into a set and pass them to dc_receive_imf
                 let context = context.clone();
+                let folder = folder.clone();
 
                 // safe, as we checked above that there is a body.
                 let body = body
@@ -1408,6 +1317,7 @@ impl Imap {
                     &context,
                     rfc724_mid,
                     body,
+                    &folder,
                     is_seen,
                     partial,
                     fetching_existing_messages,
@@ -1446,6 +1356,11 @@ impl Imap {
     /// the flag, or other imap-errors, returns true as well.
     ///
     /// Returning error means that the operation can be retried.
+    async fn add_flag_finalized(&mut self, server_uid: u32, flag: &str) -> Result<()> {
+        let s = server_uid.to_string();
+        self.add_flag_finalized_with_set(&s, flag).await
+    }
+
     async fn add_flag_finalized_with_set(&mut self, uid_set: &str, flag: &str) -> Result<()> {
         if self.should_reconnect() {
             bail!("Can't set flag, should reconnect");
@@ -1463,7 +1378,7 @@ impl Imap {
         Ok(())
     }
 
-    pub(crate) async fn prepare_imap_operation_on_msg(
+    pub async fn prepare_imap_operation_on_msg(
         &mut self,
         context: &Context,
         folder: &str,
@@ -1503,6 +1418,32 @@ impl Imap {
         }
     }
 
+    pub(crate) async fn set_seen(
+        &mut self,
+        context: &Context,
+        folder: &str,
+        uid: u32,
+    ) -> ImapActionResult {
+        if let Some(imapresult) = self
+            .prepare_imap_operation_on_msg(context, folder, uid)
+            .await
+        {
+            return imapresult;
+        }
+        // we are connected, and the folder is selected
+        info!(context, "Marking message {}/{} as seen...", folder, uid,);
+
+        if let Err(err) = self.add_flag_finalized(uid, "\\Seen").await {
+            warn!(
+                context,
+                "Cannot mark message {} in folder {} as seen, ignoring: {}.", uid, folder, err
+            );
+            ImapActionResult::Failed
+        } else {
+            ImapActionResult::Success
+        }
+    }
+
     pub async fn ensure_configured_folders(
         &mut self,
         context: &Context,
@@ -1698,7 +1639,7 @@ async fn should_move_out_of_spam(
             if chat_id_blocked.blocked != Blocked::Not {
                 return Ok(false);
             }
-        } else if from_id != ContactId::SELF {
+        } else if from_id != DC_CONTACT_ID_SELF {
             // No chat with this contact found.
             return Ok(false);
         }
@@ -1735,14 +1676,13 @@ async fn spam_target_folder(
 pub async fn target_folder(
     context: &Context,
     folder: &str,
-    is_spam_folder: bool,
     headers: &[mailparse::MailHeader<'_>],
 ) -> Result<Option<Config>> {
     if context.is_mvbox(folder).await? {
         return Ok(None);
     }
 
-    if is_spam_folder {
+    if context.is_spam_folder(folder).await? {
         spam_target_folder(context, headers).await
     } else if needs_move_to_mvbox(context, headers).await? {
         Ok(Some(Config::ConfiguredMvboxFolder))
@@ -1934,11 +1874,13 @@ pub(crate) async fn prefetch_should_download(
     mut flags: impl Iterator<Item = Flag<'_>>,
     show_emails: ShowEmails,
 ) -> Result<bool> {
-    if message::rfc724_mid_exists(context, message_id)
-        .await?
-        .is_some()
-    {
-        markseen_on_imap_table(context, message_id).await?;
+    if let Some(msg_id) = message::rfc724_mid_exists(context, message_id).await? {
+        // We know the Message-ID already, it must be a Bcc: to self.
+        job::add(
+            context,
+            job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+        )
+        .await?;
         return Ok(false);
     }
 
@@ -2072,22 +2014,6 @@ async fn mark_seen_by_uid(
     }
 }
 
-/// Schedule marking the message as Seen on IMAP by adding all known IMAP messages corresponding to
-/// the given Message-ID to `imap_markseen` table.
-pub(crate) async fn markseen_on_imap_table(context: &Context, message_id: &str) -> Result<()> {
-    context
-        .sql
-        .execute(
-            "INSERT OR IGNORE INTO imap_markseen (id)
-             SELECT id FROM imap WHERE rfc724_mid=?",
-            paramsv![message_id],
-        )
-        .await?;
-    context.interrupt_inbox(InterruptInfo::new(false)).await;
-
-    Ok(())
-}
-
 /// uid_next is the next unique identifier value from the last time we fetched a folder
 /// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
 /// This function is used to update our uid_next after fetching messages.
@@ -2169,18 +2095,6 @@ async fn get_modseq(context: &Context, folder: &str) -> Result<u64> {
         .unwrap_or(0))
 }
 
-/// Compute the imap search expression for all self-sent mails (for all self addresses)
-pub(crate) async fn get_imap_self_sent_search_command(context: &Context) -> Result<String> {
-    // See https://www.rfc-editor.org/rfc/rfc3501#section-6.4.4 for syntax of SEARCH and OR
-    let mut search_command = format!("FROM \"{}\"", context.get_primary_self_addr().await?);
-
-    for item in context.get_secondary_self_addrs().await? {
-        search_command = format!("OR ({}) (FROM \"{}\")", search_command, item);
-    }
-
-    Ok(search_command)
-}
-
 /// Deprecated, use get_uid_next() and get_uidvalidity()
 pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result<(u32, u32)> {
     let key = format!("imap.mailbox.{}", folder);
@@ -2200,11 +2114,7 @@ pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result
 ///
 /// This caters for the [`Config::OnlyFetchMvbox`] setting which means mails from folders
 /// not explicitly watched should not be fetched.
-async fn should_ignore_folder(
-    context: &Context,
-    folder: &str,
-    is_spam_folder: bool,
-) -> Result<bool> {
+async fn should_ignore_folder(context: &Context, folder: &str) -> Result<bool> {
     if !context.get_config_bool(Config::OnlyFetchMvbox).await? {
         return Ok(false);
     }
@@ -2212,7 +2122,7 @@ async fn should_ignore_folder(
         // Still respect the SentboxWatch setting.
         return Ok(!context.get_config_bool(Config::SentboxWatch).await?);
     }
-    Ok(!(context.is_mvbox(folder).await? || is_spam_folder))
+    Ok(!(context.is_mvbox(folder).await? || context.is_spam_folder(folder).await?))
 }
 
 /// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000
@@ -2388,6 +2298,9 @@ mod tests {
                                folder, mvbox_move, chat_msg, accepted_chat, outgoing, setupmessage);
 
         let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::ConfiguredSpamFolder, Some("Spam"))
+            .await?;
         t.ctx
             .set_config(Config::ConfiguredMvboxFolder, Some("DeltaChat"))
             .await?;
@@ -2429,13 +2342,11 @@ mod tests {
 
         let (headers, _) = mailparse::parse_headers(bytes)?;
 
-        let is_spam_folder = folder == "Spam";
-        let actual =
-            if let Some(config) = target_folder(&t, folder, is_spam_folder, &headers).await? {
-                t.get_config(config).await?
-            } else {
-                None
-            };
+        let actual = if let Some(config) = target_folder(&t, folder, &headers).await? {
+            t.get_config(config).await?
+        } else {
+            None
+        };
 
         let expected = if expected_destination == folder {
             None
@@ -2549,27 +2460,4 @@ mod tests {
         }
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_get_imap_search_command() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        assert_eq!(
-            get_imap_self_sent_search_command(&t.ctx).await?,
-            r#"FROM "alice@example.org""#
-        );
-
-        t.ctx.set_primary_self_addr("alice@another.com").await?;
-        assert_eq!(
-            get_imap_self_sent_search_command(&t.ctx).await?,
-            r#"OR (FROM "alice@another.com") (FROM "alice@example.org")"#
-        );
-
-        t.ctx.set_primary_self_addr("alice@third.com").await?;
-        assert_eq!(
-            get_imap_self_sent_search_command(&t.ctx).await?,
-            r#"OR (OR (FROM "alice@third.com") (FROM "alice@another.com")) (FROM "alice@example.org")"#
-        );
-
-        Ok(())
-    }
 }

src/imap/idle.rs

@@ -157,10 +157,7 @@ impl Imap {
                     // in anything.  If so, we behave as if IDLE had data but
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
-                    match self
-                        .fetch_new_messages(context, &watch_folder, false, false)
-                        .await
-                    {
+                    match self.fetch_new_messages(context, &watch_folder, false).await {
                         Ok(res) => {
                             info!(context, "fetch_new_messages returned {:?}", res);
                             if res {

src/imap/scan_folders.rs

@@ -60,9 +60,6 @@ impl Imap {
             let is_drafts = folder_meaning == FolderMeaning::Drafts
                 || (folder_meaning == FolderMeaning::Unknown
                     && folder_name_meaning == FolderMeaning::Drafts);
-            let is_spam_folder = folder_meaning == FolderMeaning::Spam
-                || (folder_meaning == FolderMeaning::Unknown
-                    && folder_name_meaning == FolderMeaning::Spam);
 
             // Don't scan folders that are watched anyway
             if !watched_folders.contains(&folder.name().to_string()) && !is_drafts {
@@ -70,7 +67,7 @@ impl Imap {
                 self.server_sent_unsolicited_exists(context)?;
 
                 loop {
-                    self.fetch_move_delete(context, folder.name(), is_spam_folder)
+                    self.fetch_move_delete(context, folder.name())
                         .await
                         .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
 
@@ -82,15 +79,16 @@ impl Imap {
             }
         }
 
-        // Set the `ConfiguredSentboxFolder` or set it to `None` if the folder was deleted.
-        context
-            .set_config(
-                Config::ConfiguredSentboxFolder,
-                folder_configs
-                    .get(&Config::ConfiguredSentboxFolder)
-                    .map(|s| s.as_str()),
-            )
-            .await?;
+        // We iterate over both folder meanings to make sure that if e.g. the "Sent" folder was deleted,
+        // `ConfiguredSentboxFolder` is set to `None`:
+        for config in &[
+            Config::ConfiguredSentboxFolder,
+            Config::ConfiguredSpamFolder,
+        ] {
+            context
+                .set_config(*config, folder_configs.get(config).map(|s| s.as_str()))
+                .await?;
+        }
 
         last_scan.replace(Instant::now());
         Ok(true)

src/imex.rs

@@ -16,7 +16,7 @@ use rand::{thread_rng, Rng};
 use crate::blob::BlobObject;
 use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
 use crate::config::Config;
-use crate::contact::ContactId;
+use crate::constants::DC_CONTACT_ID_SELF;
 use crate::context::Context;
 use crate::dc_tools::{
     dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
@@ -176,7 +176,7 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     )
     .await?;
 
-    let chat_id = ChatId::create_for_contact(context, ContactId::SELF).await?;
+    let chat_id = ChatId::create_for_contact(context, DC_CONTACT_ID_SELF).await?;
     let mut msg = Message {
         viewtype: Viewtype::File,
         ..Default::default()
@@ -351,8 +351,9 @@ async fn set_self_key(
         }
     };
 
-    let self_addr = context.get_primary_self_addr().await?;
-    let addr = EmailAddress::new(&self_addr)?;
+    let self_addr = context.get_config(Config::ConfiguredAddr).await?;
+    ensure!(self_addr.is_some(), "Missing self addr");
+    let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
     let keypair = pgp::KeyPair {
         addr,
         public: public_key,

src/job.rs

@@ -4,7 +4,7 @@
 //! and job types.
 use std::fmt;
 
-use anyhow::{format_err, Context as _, Result};
+use anyhow::{bail, format_err, Context as _, Error, Result};
 use deltachat_derive::{FromSql, ToSql};
 use rand::{thread_rng, Rng};
 
@@ -13,13 +13,14 @@ use crate::contact::{normalize_name, Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::events::EventType;
-use crate::imap::Imap;
+use crate::imap::{Imap, ImapActionResult};
 use crate::location;
+use crate::log::LogExt;
 use crate::message::{Message, MsgId};
 use crate::mimefactory::MimeFactory;
 use crate::param::{Param, Params};
 use crate::scheduler::InterruptInfo;
-use crate::smtp::{smtp_send, SendResult, Smtp};
+use crate::smtp::{smtp_send, Smtp};
 use crate::sql;
 
 // results in ~3 weeks for the last backoff timespan
@@ -31,6 +32,7 @@ const JOB_RETRIES: u32 = 17;
 )]
 #[repr(u32)]
 pub(crate) enum Thread {
+    Unknown = 0,
     Imap = 100,
     Smtp = 5000,
 }
@@ -38,7 +40,7 @@ pub(crate) enum Thread {
 /// Job try result.
 #[derive(Debug, Display)]
 pub enum Status {
-    Finished(Result<()>),
+    Finished(std::result::Result<(), Error>),
     RetryNow,
     RetryLater,
 }
@@ -58,6 +60,12 @@ macro_rules! job_try {
     };
 }
 
+impl Default for Thread {
+    fn default() -> Self {
+        Thread::Unknown
+    }
+}
+
 #[derive(
     Debug,
     Display,
@@ -73,8 +81,12 @@ macro_rules! job_try {
 )]
 #[repr(u32)]
 pub enum Action {
+    Unknown = 0,
+
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
+    Housekeeping = 105, // low priority ...
     FetchExistingMsgs = 110,
+    MarkseenMsgOnImap = 130,
 
     // this is user initiated so it should have a fairly high priority
     UpdateRecentQuota = 140,
@@ -95,13 +107,23 @@ pub enum Action {
     SendMdn = 5010,
 }
 
+impl Default for Action {
+    fn default() -> Self {
+        Action::Unknown
+    }
+}
+
 impl From<Action> for Thread {
     fn from(action: Action) -> Thread {
         use Action::*;
 
         match action {
+            Unknown => Thread::Unknown,
+
+            Housekeeping => Thread::Imap,
             FetchExistingMsgs => Thread::Imap,
             ResyncFolders => Thread::Imap,
+            MarkseenMsgOnImap => Thread::Imap,
             UpdateRecentQuota => Thread::Imap,
             DownloadMsg => Thread::Imap,
 
@@ -296,18 +318,13 @@ impl Job {
             return Status::RetryLater;
         }
 
-        match smtp_send(context, &recipients, &body, smtp, msg_id, 0).await {
-            SendResult::Success => {
-                // Remove additional SendMdn jobs we have aggregated into this one.
-                job_try!(kill_ids(context, &additional_job_ids).await);
-                Status::Finished(Ok(()))
-            }
-            SendResult::Retry => {
-                info!(context, "Temporary SMTP failure while sending an MDN");
-                Status::RetryLater
-            }
-            SendResult::Failure(err) => Status::Finished(Err(err)),
+        let status = smtp_send(context, &recipients, &body, smtp, msg_id, 0).await;
+        if matches!(status, Status::Finished(Ok(_))) {
+            // Remove additional SendMdn jobs we have aggregated into this one.
+            job_try!(kill_ids(context, &additional_job_ids).await);
         }
+
+        status
     }
 
     /// Read the recipients from old emails sent by the user and add them as contacts.
@@ -335,7 +352,7 @@ impl Job {
                 Config::ConfiguredSentboxFolder,
             ] {
                 if let Some(folder) = job_try!(context.get_config(*config).await) {
-                    if let Err(e) = imap.fetch_new_messages(context, &folder, false, true).await {
+                    if let Err(e) = imap.fetch_new_messages(context, &folder, true).await {
                         // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
                         warn!(context, "Could not fetch messages, retrying: {:#}", e);
                         return Status::RetryLater;
@@ -381,6 +398,67 @@ impl Job {
             Status::Finished(Ok(()))
         }
     }
+
+    async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.prepare(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let row = job_try!(
+            context
+                .sql
+                .query_row_optional(
+                    "SELECT uid, folder FROM imap
+                    WHERE rfc724_mid=? AND folder=target
+                    ORDER BY uid ASC
+                    LIMIT 1",
+                    paramsv![msg.rfc724_mid],
+                    |row| {
+                        let uid: u32 = row.get(0)?;
+                        let folder: String = row.get(1)?;
+                        Ok((uid, folder))
+                    }
+                )
+                .await
+        );
+        if let Some((server_uid, server_folder)) = row {
+            let result = imap.set_seen(context, &server_folder, server_uid).await;
+            match result {
+                ImapActionResult::RetryLater => return Status::RetryLater,
+                ImapActionResult::Success | ImapActionResult::Failed => {}
+            }
+        } else {
+            info!(
+                context,
+                "Can't mark the message {} as seen on IMAP because there is no known UID",
+                msg.rfc724_mid
+            );
+        }
+
+        // XXX we send MDN even in case of failure to mark the messages as seen, e.g. if it was
+        // already deleted on the server by another device. The job will not be retried so locally
+        // there is no risk of double-sending MDNs.
+        //
+        // Read receipts for system messages are never sent. These messages have no place to
+        // display received read receipt anyway.  And since their text is locally generated,
+        // quoting them is dangerous as it may contain contact names. E.g., for original message
+        // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
+        // be a display name stored in address book rather than the name sent in the From field by
+        // the user.
+        if msg.param.get_bool(Param::WantsMdn).unwrap_or_default() && !msg.is_system_message() {
+            let mdns_enabled = job_try!(context.get_config_bool(Config::MdnsEnabled).await);
+            if mdns_enabled {
+                if let Err(err) = send_mdn(context, &msg).await {
+                    warn!(context, "could not send out mdn for {}: {}", msg.id, err);
+                    return Status::Finished(Err(err));
+                }
+            }
+        }
+
+        Status::Finished(Ok(()))
+    }
 }
 
 /// Delete all pending jobs with the given action.
@@ -399,7 +477,7 @@ async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
     }
     let q = format!(
         "DELETE FROM jobs WHERE id IN({})",
-        sql::repeat_vars(job_ids.len())
+        sql::repeat_vars(job_ids.len())?
     );
     context
         .sql
@@ -570,13 +648,19 @@ async fn perform_job_action(
     );
 
     let try_res = match job.action {
+        Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))),
         Action::SendMdn => job.send_mdn(context, connection.smtp()).await,
         Action::MaybeSendLocations => location::job_maybe_send_locations(context, job).await,
         Action::MaybeSendLocationsEnded => {
             location::job_maybe_send_locations_ended(context, job).await
         }
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
+        Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
+        Action::Housekeeping => {
+            sql::housekeeping(context).await.ok_or_log(context);
+            Status::Finished(Ok(()))
+        }
         Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
             Ok(status) => status,
             Err(err) => Status::Finished(Err(err)),
@@ -609,13 +693,13 @@ fn get_backoff_time_offset(tries: u32, action: Action) -> i64 {
     }
 }
 
-pub(crate) async fn send_mdn(context: &Context, msg_id: MsgId, from_id: ContactId) -> Result<()> {
+async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     let mut param = Params::new();
-    param.set(Param::MsgId, msg_id.to_u32().to_string());
+    param.set(Param::MsgId, msg.id.to_u32().to_string());
 
     add(
         context,
-        Job::new(Action::SendMdn, from_id.to_u32(), param, 0),
+        Job::new(Action::SendMdn, msg.from_id.to_u32(), param, 0),
     )
     .await?;
 
@@ -640,7 +724,10 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
 
     if delay_seconds == 0 {
         match action {
-            Action::ResyncFolders
+            Action::Unknown => unreachable!(),
+            Action::Housekeeping
+            | Action::ResyncFolders
+            | Action::MarkseenMsgOnImap
             | Action::FetchExistingMsgs
             | Action::UpdateRecentQuota
             | Action::DownloadMsg => {
@@ -656,6 +743,18 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
     Ok(())
 }
 
+async fn load_housekeeping_job(context: &Context) -> Result<Option<Job>> {
+    let last_time = context.get_config_i64(Config::LastHousekeeping).await?;
+
+    let next_time = last_time + (60 * 60 * 24);
+    if next_time <= time() {
+        kill_action(context, Action::Housekeeping).await?;
+        Ok(Some(Job::new(Action::Housekeeping, 0, Params::new(), 0)))
+    } else {
+        Ok(None)
+    }
+}
+
 /// Load jobs from the database.
 ///
 /// Load jobs for this "[Thread]", i.e. either load SMTP jobs or load
@@ -699,7 +798,7 @@ LIMIT 1;
         params = paramsv![thread_i];
     };
 
-    loop {
+    let job = loop {
         let job_res = context
             .sql
             .query_row_optional(query, params.clone(), |row| {
@@ -718,7 +817,7 @@ LIMIT 1;
             .await;
 
         match job_res {
-            Ok(job) => return Ok(job),
+            Ok(job) => break job,
             Err(err) => {
                 // Remove invalid job from the DB
                 info!(context, "cleaning up job, because of {}", err);
@@ -736,6 +835,20 @@ LIMIT 1;
                     .with_context(|| format!("Failed to delete invalid job {}", id))?;
             }
         }
+    };
+
+    match thread {
+        Thread::Unknown => {
+            bail!("unknown thread for job")
+        }
+        Thread::Imap => {
+            if let Some(job) = job {
+                Ok(Some(job))
+            } else {
+                Ok(load_housekeeping_job(context).await?)
+            }
+        }
+        Thread::Smtp => Ok(job),
     }
 }
 
@@ -783,7 +896,8 @@ mod tests {
             &InterruptInfo::new(false),
         )
         .await?;
-        assert!(jobs.is_none());
+        // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
+        assert_eq!(jobs.unwrap().action, Action::Housekeeping);
 
         insert_job(&t, 1, true).await;
         let jobs = load_next(

src/key.rs

@@ -198,7 +198,10 @@ impl DcSecretKey for SignedSecretKey {
 }
 
 async fn generate_keypair(context: &Context) -> Result<KeyPair> {
-    let addr = context.get_primary_self_addr().await?;
+    let addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await?
+        .context("no address configured")?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 

src/location.rs

@@ -6,6 +6,8 @@ use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::chat::{self, ChatId};
+use crate::config::Config;
+use crate::constants::DC_CONTACT_ID_SELF;
 use crate::contact::ContactId;
 use crate::context::Context;
 use crate::dc_tools::time;
@@ -313,7 +315,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                         accuracy,
                         time(),
                         chat_id,
-                        ContactId::SELF,
+                        DC_CONTACT_ID_SELF,
                     ]
             ).await {
                 warn!(context, "failed to store location {:?}", err);
@@ -322,7 +324,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
             }
         }
         if continue_streaming {
-            context.emit_event(EventType::LocationChanged(Some(ContactId::SELF)));
+            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
         schedule_maybe_send_locations(context, false).await.ok();
     }
@@ -423,7 +425,10 @@ pub async fn delete_all(context: &Context) -> Result<()> {
 pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)> {
     let mut last_added_location_id = 0;
 
-    let self_addr = context.get_primary_self_addr().await?;
+    let self_addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await?
+        .unwrap_or_default();
 
     let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
         "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
@@ -458,10 +463,10 @@ pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)
              GROUP BY timestamp \
              ORDER BY timestamp;",
                 paramsv![
-                    ContactId::SELF,
+                    DC_CONTACT_ID_SELF,
                     locations_send_begin,
                     locations_last_sent,
-                    ContactId::SELF
+                    DC_CONTACT_ID_SELF
                 ],
                 |row| {
                     let location_id: i32 = row.get(0)?;
@@ -662,7 +667,7 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
             for (chat_id, locations_send_begin, locations_last_sent) in &rows {
                 if !stmt_locations
                     .exists(paramsv![
-                        ContactId::SELF,
+                        DC_CONTACT_ID_SELF,
                         *locations_send_begin,
                         *locations_last_sent,
                     ])
@@ -755,7 +760,6 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils::TestContext;
 
     #[async_std::test]
@@ -816,68 +820,4 @@ mod tests {
         assert!(!is_marker(" "));
         assert!(!is_marker("\t"));
     }
-
-    /// Tests that location.kml is hidden.
-    #[async_std::test]
-    async fn receive_location_kml() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        dc_receive_imf(
-            &alice,
-            br#"Subject: Hello
-Message-ID: hello@example.net
-To: Alice <alice@example.org>
-From: Bob <bob@example.net>
-Date: Mon, 20 Dec 2021 00:00:00 +0000
-Chat-Version: 1.0
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-Text message."#,
-            false,
-        )
-        .await?;
-        let received_msg = alice.get_last_msg().await;
-        assert_eq!(received_msg.text.unwrap(), "Text message.");
-
-        dc_receive_imf(
-            &alice,
-            br#"Subject: locations
-MIME-Version: 1.0
-To: <alice@example.org>
-From: <bob@example.net>
-Date: Tue, 21 Dec 2021 00:00:00 +0000
-Chat-Version: 1.0
-Message-ID: <foobar@example.net>
-Content-Type: multipart/mixed; boundary="U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF"
-
-
---U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-
-
---U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
-Content-Type: application/vnd.google-earth.kml+xml
-Content-Disposition: attachment; filename="location.kml"
-
-<?xml version="1.0" encoding="UTF-8"?>
-<kml xmlns="http://www.opengis.net/kml/2.2">
-<Document addr="bob@example.net">
-<Placemark><Timestamp><when>2021-11-21T00:00:00Z</when></Timestamp><Point><coordinates accuracy="1.0000000000000000">10.00000000000000,20.00000000000000</coordinates></Point></Placemark>
-</Document>
-</kml>
-
---U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF--"#,
-            false,
-        )
-        .await?;
-
-        // Received location message is not visible, last message stays the same.
-        let received_msg2 = alice.get_last_msg().await;
-        assert_eq!(received_msg2.id, received_msg.id);
-
-        let locations = get_range(&alice, None, None, 0, 0).await?;
-        assert_eq!(locations.len(), 1);
-        Ok(())
-    }
 }

src/login_param.rs

@@ -139,18 +139,8 @@ pub struct LoginParam {
 }
 
 impl LoginParam {
-    /// Load entered (candidate) account settings
-    pub async fn load_candidate_params(context: &Context) -> Result<Self> {
-        LoginParam::from_database(context, "").await
-    }
-
-    /// Load configured (working) account settings
-    pub async fn load_configured_params(context: &Context) -> Result<Self> {
-        LoginParam::from_database(context, "configured_").await
-    }
-
     /// Read the login parameters from the database.
-    async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
+    pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
@@ -252,11 +242,12 @@ impl LoginParam {
     }
 
     /// Save this loginparam to the database.
-    pub async fn save_as_configured_params(&self, context: &Context) -> Result<()> {
-        let prefix = "configured_";
+    pub async fn save_to_database(&self, context: &Context, prefix: impl AsRef<str>) -> Result<()> {
+        let prefix = prefix.as_ref();
         let sql = &context.sql;
 
-        context.set_primary_self_addr(&self.addr).await?;
+        let key = format!("{}addr", prefix);
+        sql.set_raw_config(key, Some(&self.addr)).await?;
 
         let key = format!("{}mail_server", prefix);
         sql.set_raw_config(key, Some(&self.imap.server)).await?;
@@ -447,8 +438,8 @@ mod tests {
             socks5_config: None,
         };
 
-        param.save_as_configured_params(&t).await?;
-        let loaded = LoginParam::load_configured_params(&t).await?;
+        param.save_to_database(&t, "foobar_").await?;
+        let loaded = LoginParam::from_database(&t, "foobar_").await?;
 
         assert_eq!(param, loaded);
         Ok(())

src/message.rs

@@ -9,9 +9,9 @@ use rusqlite::types::ValueRef;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
-use crate::config::Config;
 use crate::constants::{
-    Blocked, Chattype, VideochatType, DC_CHAT_ID_TRASH, DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
+    Blocked, Chattype, VideochatType, DC_CHAT_ID_TRASH, DC_CONTACT_ID_INFO, DC_CONTACT_ID_SELF,
+    DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
 };
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
@@ -22,8 +22,7 @@ use crate::dc_tools::{
 use crate::download::DownloadState;
 use crate::ephemeral::{start_ephemeral_timers_msgids, Timer as EphemeralTimer};
 use crate::events::EventType;
-use crate::imap::markseen_on_imap_table;
-use crate::job;
+use crate::job::{self, Action};
 use crate::log::LogExt;
 use crate::mimeparser::{parse_message_id, FailureReport, SystemMessage};
 use crate::param::{Param, Params};
@@ -432,7 +431,7 @@ impl Message {
     /// this is done by dc_set_location() and dc_send_locations_to_chat().
     ///
     /// Typically results in the event #DC_EVENT_LOCATION_CHANGED with
-    /// contact_id set to ContactId::SELF.
+    /// contact_id set to DC_CONTACT_ID_SELF.
     ///
     /// @param latitude North-south position of the location.
     /// @param longitude East-west position of the location.
@@ -545,7 +544,7 @@ impl Message {
             &chat_loaded
         };
 
-        let contact = if self.from_id != ContactId::SELF {
+        let contact = if self.from_id != DC_CONTACT_ID_SELF {
             match chat.typ {
                 Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
                     Some(Contact::get_by_id(context, self.from_id).await?)
@@ -598,8 +597,8 @@ impl Message {
 
     pub fn is_info(&self) -> bool {
         let cmd = self.param.get_cmd();
-        self.from_id == ContactId::INFO
-            || self.to_id == ContactId::INFO
+        self.from_id == DC_CONTACT_ID_INFO
+            || self.to_id == DC_CONTACT_ID_INFO
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
@@ -1018,7 +1017,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
     ret += &format!(" by {}", name);
     ret += "\n";
 
-    if msg.from_id != ContactId::SELF {
+    if msg.from_id != DC_CONTACT_ID_SELF {
         let s = dc_timestamp_to_str(if 0 != msg.timestamp_rcvd {
             msg.timestamp_rcvd
         } else {
@@ -1039,7 +1038,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
         );
     }
 
-    if msg.from_id == ContactId::INFO || msg.to_id == ContactId::INFO {
+    if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return Ok(ret);
     }
@@ -1254,13 +1253,19 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     }
 
     if !msg_ids.is_empty() {
-        context.emit_msgs_changed_without_ids();
-
-        // Run housekeeping to delete unused blobs.
-        context.set_config(Config::LastHousekeeping, None).await?;
+        context.emit_event(EventType::MsgsChanged {
+            chat_id: ChatId::new(0),
+            msg_id: MsgId::new(0),
+        });
+        job::kill_action(context, Action::Housekeeping).await?;
+        job::add(
+            context,
+            job::Job::new(Action::Housekeeping, 0, Params::new(), 10),
+        )
+        .await?;
     }
 
-    // Interrupt Inbox loop to start message deletion and run housekeeping.
+    // Interrupt Inbox loop to start message deletion.
     context.interrupt_inbox(InterruptInfo::new(false)).await;
     Ok(())
 }
@@ -1290,31 +1295,22 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
                     m.chat_id AS chat_id,
                     m.state AS state,
                     m.ephemeral_timer AS ephemeral_timer,
-                    m.param AS param,
-                    m.from_id AS from_id,
-                    m.rfc724_mid AS rfc724_mid,
                     c.blocked AS blocked
                  FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id
                  WHERE m.id IN ({}) AND m.chat_id>9",
-                sql::repeat_vars(msg_ids.len())
+                sql::repeat_vars(msg_ids.len())?
             ),
             rusqlite::params_from_iter(&msg_ids),
             |row| {
                 let id: MsgId = row.get("id")?;
                 let chat_id: ChatId = row.get("chat_id")?;
                 let state: MessageState = row.get("state")?;
-                let param: Params = row.get::<_, String>("param")?.parse().unwrap_or_default();
-                let from_id: ContactId = row.get("from_id")?;
-                let rfc724_mid: String = row.get("rfc724_mid")?;
                 let blocked: Option<Blocked> = row.get("blocked")?;
                 let ephemeral_timer: EphemeralTimer = row.get("ephemeral_timer")?;
                 Ok((
                     id,
                     chat_id,
                     state,
-                    param,
-                    from_id,
-                    rfc724_mid,
                     blocked.unwrap_or_default(),
                     ephemeral_timer,
                 ))
@@ -1323,52 +1319,30 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
         )
         .await?;
 
-    if msgs.iter().any(
-        |(_id, _chat_id, _state, _param, _from_id, _rfc724_mid, _blocked, ephemeral_timer)| {
+    if msgs
+        .iter()
+        .any(|(_id, _chat_id, _state, _blocked, ephemeral_timer)| {
             *ephemeral_timer != EphemeralTimer::Disabled
-        },
-    ) {
+        })
+    {
         start_ephemeral_timers_msgids(context, &msg_ids)
             .await
             .context("failed to start ephemeral timers")?;
     }
 
     let mut updated_chat_ids = BTreeSet::new();
-    for (
-        id,
-        curr_chat_id,
-        curr_state,
-        curr_param,
-        curr_from_id,
-        curr_rfc724_mid,
-        curr_blocked,
-        _curr_ephemeral_timer,
-    ) in msgs.into_iter()
-    {
+    for (id, curr_chat_id, curr_state, curr_blocked, _curr_ephemeral_timer) in msgs.into_iter() {
         if curr_blocked == Blocked::Not
             && (curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed)
         {
             update_msg_state(context, id, MessageState::InSeen).await?;
             info!(context, "Seen message {}.", id);
 
-            markseen_on_imap_table(context, &curr_rfc724_mid).await?;
-
-            // Read receipts for system messages are never sent. These messages have no place to
-            // display received read receipt anyway.  And since their text is locally generated,
-            // quoting them is dangerous as it may contain contact names. E.g., for original message
-            // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
-            // be a display name stored in address book rather than the name sent in the From field by
-            // the user.
-            if curr_param.get_bool(Param::WantsMdn).unwrap_or_default()
-                && curr_param.get_cmd() == SystemMessage::Unknown
-            {
-                let mdns_enabled = context.get_config_bool(Config::MdnsEnabled).await?;
-                if mdns_enabled {
-                    if let Err(err) = job::send_mdn(context, id, curr_from_id).await {
-                        warn!(context, "could not send out mdn for {}: {}", id, err);
-                    }
-                }
-            }
+            job::add(
+                context,
+                job::Job::new(Action::MarkseenMsgOnImap, id.to_u32(), Params::new(), 0),
+            )
+            .await?;
             updated_chat_ids.insert(curr_chat_id);
         }
     }
@@ -1458,7 +1432,7 @@ pub async fn handle_mdn(
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Result<Option<(ChatId, MsgId)>> {
-    if from_id == ContactId::SELF {
+    if from_id == DC_CONTACT_ID_SELF {
         warn!(
             context,
             "ignoring MDN sent to self, this is a bug on the sender device"
@@ -1663,7 +1637,7 @@ pub async fn estimate_deletion_cnt(
     from_server: bool,
     seconds: i64,
 ) -> Result<usize> {
-    let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+    let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
         .await?
         .unwrap_or_default();
     let threshold_timestamp = time() - seconds;
@@ -1831,6 +1805,7 @@ mod tests {
 
     use crate::chat::{marknoticed_chat, ChatItem};
     use crate::chatlist::Chatlist;
+    use crate::constants::DC_CONTACT_ID_DEVICE;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils as test;
     use crate::test_utils::TestContext;
@@ -1970,7 +1945,7 @@ mod tests {
         // test that get_width() and get_height() are returning some dimensions for images;
         // (as the device-chat contains a welcome-images, we check that)
         t.update_device_chats().await.ok();
-        let device_chat_id = ChatId::get_for_contact(&t, ContactId::DEVICE)
+        let device_chat_id = ChatId::get_for_contact(&t, DC_CONTACT_ID_DEVICE)
             .await
             .unwrap();
 
@@ -2044,6 +2019,7 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello\n",
+            "INBOX",
             false,
         )
         .await
@@ -2252,6 +2228,7 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -2269,6 +2246,7 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello again\n",
+            "INBOX",
             false,
         )
         .await?;

src/mimefactory.rs

@@ -133,7 +133,11 @@ impl<'a> MimeFactory<'a> {
     ) -> Result<MimeFactory<'a>> {
         let chat = Chat::load_from_db(context, msg.chat_id).await?;
 
-        let from_addr = context.get_primary_self_addr().await?;
+        let from_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .unwrap_or_default();
+
         let config_displayname = context
             .get_config(Config::Displayname)
             .await?
@@ -233,7 +237,10 @@ impl<'a> MimeFactory<'a> {
         ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
         let contact = Contact::load_from_db(context, msg.from_id).await?;
-        let from_addr = context.get_primary_self_addr().await?;
+        let from_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .unwrap_or_default();
         let from_displayname = context
             .get_config(Config::Displayname)
             .await?
@@ -271,7 +278,10 @@ impl<'a> MimeFactory<'a> {
         &self,
         context: &Context,
     ) -> Result<Vec<(Option<Peerstate>, &str)>> {
-        let self_addr = context.get_primary_self_addr().await?;
+        let self_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .context("not configured")?;
 
         let mut res = Vec::new();
         for (_, addr) in self
@@ -850,12 +860,9 @@ impl<'a> MimeFactory<'a> {
         }
 
         if chat.typ == Chattype::Group {
-            // Send group ID unless it is an ad hoc group that has no ID.
-            if !chat.grpid.is_empty() {
-                headers
-                    .protected
-                    .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
-            }
+            headers
+                .protected
+                .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
             let encoded = encode_words(&chat.name);
             headers
@@ -1665,6 +1672,7 @@ mod tests {
             Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
             \n\
             hello\n",
+            "INBOX",
             false,
         )
         .await
@@ -1757,6 +1765,7 @@ mod tests {
                 t.get_last_msg().await.rfc724_mid
             )
             .as_bytes(),
+            "INBOX",
             false,
         )
         .await?;
@@ -1867,6 +1876,7 @@ mod tests {
                     Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                     \n\
                     Some other, completely unrelated content\n",
+                "INBOX",
                 false,
             )
             .await
@@ -1891,7 +1901,9 @@ mod tests {
             .await
             .unwrap();
 
-        dc_receive_imf(context, imf_raw, false).await.unwrap();
+        dc_receive_imf(context, imf_raw, "INBOX", false)
+            .await
+            .unwrap();
 
         let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
 

src/mimeparser.rs

@@ -2915,6 +2915,7 @@ On 2020-10-25, Bob wrote:
         dc_receive_imf(
             &t.ctx,
             include_bytes!("../test-data/message/subj_with_multimedia_msg.eml"),
+            "INBOX",
             false,
         )
         .await
@@ -3063,7 +3064,7 @@ Subject: ...
 
 Some quote.
 "###;
-        dc_receive_imf(&t, raw, false).await?;
+        dc_receive_imf(&t, raw, "INBOX", false).await?;
 
         // Delta Chat generates In-Reply-To with a starting tab when Message-ID is too long.
         let raw = br###"In-Reply-To:
@@ -3080,7 +3081,7 @@ Subject: ...
 Some reply
 "###;
 
-        dc_receive_imf(&t, raw, false).await?;
+        dc_receive_imf(&t, raw, "INBOX", false).await?;
 
         let msg = t.get_last_msg().await;
         assert_eq!(msg.get_text().unwrap(), "Some reply");
@@ -3108,13 +3109,13 @@ Message.
 "###;
 
         // Bob receives message.
-        dc_receive_imf(&bob, raw, false).await?;
+        dc_receive_imf(&bob, raw, "INBOX", false).await?;
         let msg = bob.get_last_msg().await;
         // Message is incoming.
         assert!(msg.param.get_bool(Param::WantsMdn).unwrap());
 
         // Alice receives copy-to-self.
-        dc_receive_imf(&alice, raw, false).await?;
+        dc_receive_imf(&alice, raw, "INBOX", false).await?;
         let msg = alice.get_last_msg().await;
         // Message is outgoing, don't send read receipt to self.
         assert!(msg.param.get_bool(Param::WantsMdn).is_none());
@@ -3140,6 +3141,7 @@ Message.
                  \n\
                  hello\n"
                 .as_bytes(),
+            "INBOX",
             false,
         )
         .await?;
@@ -3177,6 +3179,7 @@ Message.
                  \n\
                  --SNIPP--"
             .as_bytes(),
+            "INBOX",
             false,
         )
         .await?;
@@ -3199,7 +3202,7 @@ Message.
 
         let original =
             include_bytes!("../test-data/message/ms_exchange_report_original_message.eml");
-        dc_receive_imf(&t, original, false).await?;
+        dc_receive_imf(&t, original, "INBOX", false).await?;
         let original_msg_id = t.get_last_msg().await.id;
 
         // 1. Test mimeparser directly
@@ -3214,7 +3217,7 @@ Message.
         assert!(mimeparser.mdn_reports[0].additional_message_ids.is_empty());
 
         // 2. Test that marking the original msg as read works
-        dc_receive_imf(&t, mdn, false).await?;
+        dc_receive_imf(&t, mdn, "INBOX", false).await?;
 
         assert_eq!(
             original_msg_id.get_state(&t).await?,

src/peerstate.rs

@@ -4,13 +4,11 @@ use std::collections::HashSet;
 use std::fmt;
 
 use crate::aheader::{Aheader, EncryptPreference};
-use crate::chat::{self};
-use crate::chatlist::Chatlist;
+use crate::chat::{self, ChatIdBlocked};
+use crate::constants::Blocked;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
-use crate::message::Message;
-use crate::mimeparser::SystemMessage;
 use crate::sql::Sql;
 use crate::stock_str;
 use anyhow::{bail, Result};
@@ -273,34 +271,14 @@ impl Peerstate {
                 .query_get_value("SELECT id FROM contacts WHERE addr=?;", paramsv![self.addr])
                 .await?
             {
-                let chats = Chatlist::try_load(context, 0, None, contact_id).await?;
+                let chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Request)
+                    .await?
+                    .id;
+
                 let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
-                for (chat_id, msg_id) in chats.iter() {
-                    let timestamp_sort = if let Some(msg_id) = msg_id {
-                        let lastmsg = Message::load_from_db(context, *msg_id).await?;
-                        lastmsg.timestamp_sort
-                    } else {
-                        context
-                            .sql
-                            .query_get_value(
-                                "SELECT created_timestamp FROM chats WHERE id=?;",
-                                paramsv![chat_id],
-                            )
-                            .await?
-                            .unwrap_or(0)
-                    };
-                    chat::add_info_msg_with_cmd(
-                        context,
-                        *chat_id,
-                        &msg,
-                        SystemMessage::Unknown,
-                        timestamp_sort,
-                        Some(timestamp),
-                        None,
-                    )
-                    .await?;
-                    context.emit_event(EventType::ChatModified(*chat_id));
-                }
+
+                chat::add_info_msg(context, chat_id, &msg, timestamp).await?;
+                context.emit_event(EventType::ChatModified(chat_id));
             } else {
                 bail!("contact with peerstate.addr {:?} not found", &self.addr);
             }

src/qr.rs

@@ -711,7 +711,7 @@ mod tests {
             ..
         } = qr
         {
-            assert_ne!(contact_id, ContactId::UNDEFINED);
+            assert_ne!(contact_id, ContactId::new(0));
             assert_eq!(grpname, "test ? test !");
         } else {
             bail!("Wrong QR code type");
@@ -729,7 +729,7 @@ mod tests {
             ..
         } = qr
         {
-            assert_ne!(contact_id, ContactId::UNDEFINED);
+            assert_ne!(contact_id, ContactId::new(0));
             assert_eq!(grpname, "test ? test !");
 
             let contact = Contact::get_by_id(&ctx.ctx, contact_id).await?;
@@ -751,7 +751,7 @@ mod tests {
         ).await?;
 
         if let Qr::AskVerifyContact { contact_id, .. } = qr {
-            assert_ne!(contact_id, ContactId::UNDEFINED);
+            assert_ne!(contact_id, ContactId::new(0));
         } else {
             bail!("Wrong QR code type");
         }

src/qr_code_generator.rs

@@ -6,7 +6,8 @@ use crate::{
     chat::{Chat, ChatId},
     color::color_int_to_hex_string,
     config::Config,
-    contact::{Contact, ContactId},
+    constants::DC_CONTACT_ID_SELF,
+    contact::Contact,
     context::Context,
     securejoin, stock_str,
 };
@@ -40,7 +41,7 @@ async fn generate_join_group_qr_code(context: &Context, chat_id: ChatId) -> Resu
 }
 
 async fn generate_verification_qr(context: &Context) -> Result<String> {
-    let contact = Contact::get_by_id(context, ContactId::SELF).await?;
+    let contact = Contact::get_by_id(context, DC_CONTACT_ID_SELF).await?;
 
     let avatar = match contact.get_profile_image(context).await? {
         Some(path) => {

src/scheduler.rs

@@ -8,13 +8,11 @@ use async_std::{
 use crate::config::Config;
 use crate::context::Context;
 use crate::dc_tools::maybe_add_time_based_warnings;
-use crate::dc_tools::time;
-use crate::ephemeral::{self, delete_expired_imap_messages};
+use crate::ephemeral::delete_expired_imap_messages;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
 use crate::log::LogExt;
 use crate::smtp::{send_smtp_messages, Smtp};
-use crate::sql;
 
 use self::connectivity::ConnectivityStore;
 
@@ -36,8 +34,6 @@ pub(crate) enum Scheduler {
         sentbox_handle: Option<task::JoinHandle<()>>,
         smtp: SmtpConnectionState,
         smtp_handle: Option<task::JoinHandle<()>>,
-        ephemeral_handle: Option<task::JoinHandle<()>>,
-        ephemeral_interrupt_send: Sender<()>,
     },
 }
 
@@ -63,10 +59,6 @@ impl Context {
     pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
         self.scheduler.read().await.interrupt_smtp(info).await;
     }
-
-    pub(crate) async fn interrupt_ephemeral_task(&self) {
-        self.scheduler.read().await.interrupt_ephemeral_task().await;
-    }
 }
 
 async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConnectionHandlers) {
@@ -87,6 +79,8 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
             .expect("inbox loop, missing started receiver");
         let ctx = ctx1;
 
+        // track number of continously executed jobs
+        let mut jobs_loaded = 0;
         let mut info = InterruptInfo::default();
         loop {
             let job = match job::load_next(&ctx, Thread::Imap, &info).await {
@@ -98,25 +92,21 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
             };
 
             match job {
-                Some(job) => {
+                Some(job) if jobs_loaded <= 20 => {
+                    jobs_loaded += 1;
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
                     info = Default::default();
                 }
+                Some(job) => {
+                    // Let the fetch run, but return back to the job afterwards.
+                    jobs_loaded = 0;
+                    info!(ctx, "postponing imap-job {} to run fetch...", job);
+                    fetch(&ctx, &mut connection).await;
+                }
                 None => {
-                    maybe_add_time_based_warnings(&ctx).await;
+                    jobs_loaded = 0;
 
-                    match ctx.get_config_i64(Config::LastHousekeeping).await {
-                        Ok(last_housekeeping_time) => {
-                            let next_housekeeping_time =
-                                last_housekeeping_time.saturating_add(60 * 60 * 24);
-                            if next_housekeeping_time <= time() {
-                                sql::housekeeping(&ctx).await.ok_or_log(&ctx);
-                            }
-                        }
-                        Err(err) => {
-                            warn!(ctx, "Failed to get last housekeeping time: {}", err);
-                        }
-                    };
+                    maybe_add_time_based_warnings(&ctx).await;
 
                     info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await;
                 }
@@ -137,6 +127,32 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         .expect("inbox loop, missing shutdown receiver");
 }
 
+async fn fetch(ctx: &Context, connection: &mut Imap) {
+    match ctx.get_config(Config::ConfiguredInboxFolder).await {
+        Ok(Some(watch_folder)) => {
+            if let Err(err) = connection.prepare(ctx).await {
+                warn!(ctx, "Could not connect: {}", err);
+                return;
+            }
+
+            // fetch
+            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
+                connection.trigger_reconnect(ctx).await;
+                warn!(ctx, "{:#}", err);
+            }
+        }
+        Ok(None) => {
+            info!(ctx, "Can not fetch inbox folder, not set");
+        }
+        Err(err) => {
+            warn!(
+                ctx,
+                "Can not fetch inbox folder, failed to get config: {:?}", err
+            );
+        }
+    }
+}
+
 async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
     match ctx.get_config(folder).await {
         Ok(Some(watch_folder)) => {
@@ -146,30 +162,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 return connection.fake_idle(ctx, Some(watch_folder)).await;
             }
 
-            if folder == Config::ConfiguredInboxFolder {
-                if let Err(err) = connection
-                    .store_seen_flags_on_imap(ctx)
-                    .await
-                    .context("store_seen_flags_on_imap failed")
-                {
-                    warn!(ctx, "{:#}", err);
-                }
-            }
-
-            // Fetch the watched folder.
-            if let Err(err) = connection
-                .fetch_move_delete(ctx, &watch_folder, false)
-                .await
-            {
-                connection.trigger_reconnect(ctx).await;
-                warn!(ctx, "{:#}", err);
-                return InterruptInfo::new(false);
-            }
-
-            // Mark expired messages for deletion. Marked messages will be deleted from the server
-            // on the next iteration of `fetch_move_delete`. `delete_expired_imap_messages` is not
-            // called right before `fetch_move_delete` because it is not well optimized and would
-            // otherwise slow down message fetching.
+            // Mark expired messages for deletion.
             if let Err(err) = delete_expired_imap_messages(ctx)
                 .await
                 .context("delete_expired_imap_messages failed")
@@ -177,6 +170,13 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 warn!(ctx, "{:#}", err);
             }
 
+            // Fetch the watched folder.
+            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
+                connection.trigger_reconnect(ctx).await;
+                warn!(ctx, "{:#}", err);
+                return InterruptInfo::new(false);
+            }
+
             // Scan additional folders only after finishing fetching the watched folder.
             //
             // On iOS the application has strictly limited time to work in background, so we may not
@@ -196,10 +196,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                         // In most cases this will select the watched folder and return because there are
                         // no new messages. We want to select the watched folder anyway before going IDLE
                         // there, so this does not take additional protocol round-trip.
-                        if let Err(err) = connection
-                            .fetch_move_delete(ctx, &watch_folder, false)
-                            .await
-                        {
+                        if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
                             connection.trigger_reconnect(ctx).await;
                             warn!(ctx, "{:#}", err);
                             return InterruptInfo::new(false);
@@ -308,7 +305,6 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
             .expect("smtp loop, missing started receiver");
         let ctx = ctx1;
 
-        let mut timeout = None;
         let mut interrupt_info = Default::default();
         loop {
             let job = match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
@@ -326,16 +322,9 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
                     interrupt_info = Default::default();
                 }
                 None => {
-                    let res = send_smtp_messages(&ctx, &mut connection).await;
-                    if let Err(err) = &res {
+                    if let Err(err) = send_smtp_messages(&ctx, &mut connection).await {
                         warn!(ctx, "send_smtp_messages failed: {:#}", err);
                     }
-                    let success = res.unwrap_or(false);
-                    timeout = if success {
-                        None
-                    } else {
-                        Some(timeout.map_or(30, |timeout: u64| timeout.saturating_mul(3)))
-                    };
 
                     // Fake Idle
                     info!(ctx, "smtp fake idle - started");
@@ -344,27 +333,7 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
                         Some(err) => connection.connectivity.set_err(&ctx, err).await,
                     }
 
-                    // If send_smtp_messages() failed, we set a timeout for the fake-idle so that
-                    // sending is retried (at the latest) after the timeout. If sending fails
-                    // again, we increase the timeout exponentially, in order not to do lots of
-                    // unnecessary retries.
-                    if let Some(timeout) = timeout {
-                        info!(
-                            ctx,
-                            "smtp has messages to retry, planning to retry {} seconds later",
-                            timeout
-                        );
-                        let duration = std::time::Duration::from_secs(timeout);
-                        interrupt_info = async_std::future::timeout(duration, async {
-                            idle_interrupt_receiver.recv().await.unwrap_or_default()
-                        })
-                        .await
-                        .unwrap_or_default();
-                    } else {
-                        info!(ctx, "smtp has no messages to retry, waiting for interrupt");
-                        interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
-                    };
-
+                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
                     info!(ctx, "smtp fake idle - interrupted")
                 }
             }
@@ -398,7 +367,6 @@ impl Scheduler {
         let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1);
         let mut sentbox_handle = None;
         let (smtp_start_send, smtp_start_recv) = channel::bounded(1);
-        let (ephemeral_interrupt_send, ephemeral_interrupt_recv) = channel::bounded(1);
 
         let inbox_handle = {
             let ctx = ctx.clone();
@@ -460,13 +428,6 @@ impl Scheduler {
             }))
         };
 
-        let ephemeral_handle = {
-            let ctx = ctx.clone();
-            Some(task::spawn(async move {
-                ephemeral::ephemeral_loop(&ctx, ephemeral_interrupt_recv).await;
-            }))
-        };
-
         *self = Scheduler::Running {
             inbox,
             mvbox,
@@ -476,8 +437,6 @@ impl Scheduler {
             mvbox_handle,
             sentbox_handle,
             smtp_handle,
-            ephemeral_handle,
-            ephemeral_interrupt_send,
         };
 
         // wait for all loops to be started
@@ -543,16 +502,6 @@ impl Scheduler {
         }
     }
 
-    async fn interrupt_ephemeral_task(&self) {
-        if let Scheduler::Running {
-            ref ephemeral_interrupt_send,
-            ..
-        } = self
-        {
-            ephemeral_interrupt_send.try_send(()).ok();
-        }
-    }
-
     /// Halts the scheduler, must be called first, and then `stop`.
     pub(crate) async fn pre_stop(&self) -> StopToken {
         match self {
@@ -599,7 +548,6 @@ impl Scheduler {
                 mvbox_handle,
                 sentbox_handle,
                 smtp_handle,
-                ephemeral_handle,
                 ..
             } => {
                 if let Some(handle) = inbox_handle.take() {
@@ -614,9 +562,6 @@ impl Scheduler {
                 if let Some(handle) = smtp_handle.take() {
                     handle.await;
                 }
-                if let Some(handle) = ephemeral_handle.take() {
-                    handle.cancel().await;
-                }
 
                 *self = Scheduler::Stopped;
             }

src/scheduler/connectivity.rs

@@ -304,7 +304,7 @@ impl Context {
             <html>
             <head>
                 <meta charset="UTF-8" />
-                <meta name="viewport" content="initial-scale=1.0; user-scalable=no" />
+                <meta name="viewport" content="initial-scale=1.0" />
                 <style>
                     ul {
                         list-style-type: none;
@@ -449,7 +449,13 @@ impl Context {
         //                                [======67%=====       ]
         // =============================================================================================
 
-        let domain = dc_tools::EmailAddress::new(&self.get_primary_self_addr().await?)?.domain;
+        let domain = dc_tools::EmailAddress::new(
+            &self
+                .get_config(Config::ConfiguredAddr)
+                .await?
+                .unwrap_or_default(),
+        )?
+        .domain;
         ret += &format!(
             "<h3>{}</h3><ul>",
             stock_str::storage_on_domain(self, domain).await
@@ -538,7 +544,7 @@ impl Context {
                 self.schedule_quota_update().await?;
             }
         } else {
-            ret += &format!("<li>{}</li>", stock_str::not_connected(self).await);
+            ret += &format!("<li>{}</li>", stock_str::one_moment(self).await);
             self.schedule_quota_update().await?;
         }
         ret += "</ul>";

src/securejoin.rs

@@ -8,7 +8,7 @@ use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat, ChatId, ChatIdBlocked};
 use crate::config::Config;
-use crate::constants::Blocked;
+use crate::constants::{Blocked, DC_CONTACT_ID_LAST_SPECIAL};
 use crate::contact::{Contact, ContactId, Origin, VerifiedStatus};
 use crate::context::Context;
 use crate::dc_tools::time;
@@ -66,7 +66,19 @@ pub async fn dc_get_securejoin_qr(context: &Context, group: Option<ChatId>) -> R
         .is_none();
     let invitenumber = token::lookup_or_new(context, Namespace::InviteNumber, group).await;
     let auth = token::lookup_or_new(context, Namespace::Auth, group).await;
-    let self_addr = context.get_primary_self_addr().await?;
+    let self_addr = match context.get_config(Config::ConfiguredAddr).await {
+        Ok(Some(addr)) => addr,
+        Ok(None) => {
+            bail!("Not configured, cannot generate QR code.");
+        }
+        Err(err) => {
+            bail!(
+                "Unable to retrieve configuration, cannot generate QR code: {:?}",
+                err
+            );
+        }
+    };
+
     let self_name = context
         .get_config(Config::Displayname)
         .await?
@@ -87,12 +99,6 @@ pub async fn dc_get_securejoin_qr(context: &Context, group: Option<ChatId>) -> R
     let qr = if let Some(group) = group {
         // parameters used: a=g=x=i=s=
         let chat = Chat::load_from_db(context, group).await?;
-        if chat.grpid.is_empty() {
-            bail!(
-                "can't generate securejoin QR code for ad-hoc group {}",
-                group
-            );
-        }
         let group_name = chat.get_name();
         let group_name_urlencoded = utf8_percent_encode(group_name, NON_ALPHANUMERIC).to_string();
         if sync_token {
@@ -304,7 +310,7 @@ pub(crate) async fn handle_securejoin_handshake(
     mime_message: &MimeMessage,
     contact_id: ContactId,
 ) -> Result<HandshakeMessage> {
-    if contact_id.is_special() {
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
@@ -567,7 +573,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
     mime_message: &MimeMessage,
     contact_id: ContactId,
 ) -> Result<HandshakeMessage> {
-    if contact_id.is_special() {
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
@@ -716,7 +722,6 @@ mod tests {
     use crate::chat::ProtectionStatus;
     use crate::chatlist::Chatlist;
     use crate::constants::Chattype;
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::peerstate::Peerstate;
     use crate::test_utils::{TestContext, TestContextManager};
 
@@ -1315,25 +1320,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_adhoc_group_no_qr() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        alice.set_config(Config::ShowEmails, Some("2")).await?;
-
-        let mime = br#"Subject: First thread
-Message-ID: first@example.org
-To: Alice <alice@example.org>, Bob <bob@example.net>
-From: Claire <claire@example.org>
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-First thread."#;
-
-        dc_receive_imf(&alice, mime, false).await?;
-        let msg = alice.get_last_msg().await;
-        let chat_id = msg.chat_id;
-
-        assert!(dc_get_securejoin_qr(&alice, Some(chat_id)).await.is_err());
-        Ok(())
-    }
 }

src/smtp.rs

@@ -4,14 +4,14 @@ pub mod send;
 
 use std::time::{Duration, SystemTime};
 
-use anyhow::{bail, format_err, Context as _, Error, Result};
+use anyhow::{bail, format_err, Context as _, Result};
 use async_smtp::smtp::client::net::ClientTlsParameters;
 use async_smtp::smtp::response::{Category, Code, Detail};
 use async_smtp::{smtp, EmailAddress, ServerAddress};
-use async_std::task;
 
 use crate::constants::DC_LP_AUTH_OAUTH2;
 use crate::events::EventType;
+use crate::job::Status;
 use crate::login_param::{
     dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam, Socks5Config,
 };
@@ -50,10 +50,7 @@ impl Smtp {
     /// Disconnect the SMTP transport and drop it entirely.
     pub async fn disconnect(&mut self) {
         if let Some(mut transport) = self.transport.take() {
-            // Closing connection with a QUIT command may take some time, especially if it's a
-            // stale connection and an attempt to send the command times out. Send a command in a
-            // separate task to avoid waiting for reply or timeout.
-            task::spawn(async move { transport.close().await });
+            transport.close().await.ok();
         }
         self.last_success = None;
     }
@@ -87,7 +84,7 @@ impl Smtp {
         }
 
         self.connectivity.set_connecting(context).await;
-        let lp = LoginParam::load_configured_params(context).await?;
+        let lp = LoginParam::from_database(context, "configured_").await?;
         self.connect(
             context,
             &lp.smtp,
@@ -199,18 +196,12 @@ impl Smtp {
     }
 }
 
-pub(crate) enum SendResult {
-    /// Message was sent successfully.
-    Success,
-
-    /// Permanent error, message sending has failed.
-    Failure(Error),
-
-    /// Temporary error, the message should be retried later.
-    Retry,
-}
-
 /// Tries to send a message.
+///
+/// Returns Status::Finished if sending the message should not be retried anymore,
+/// Status::RetryLater if sending should be postponed and Status::RetryNow if it is suspected that
+/// temporary failure is caused by stale connection, in which case a second attempt to send the
+/// same message may be done immediately.
 pub(crate) async fn smtp_send(
     context: &Context,
     recipients: &[async_smtp::EmailAddress],
@@ -218,7 +209,7 @@ pub(crate) async fn smtp_send(
     smtp: &mut Smtp,
     msg_id: MsgId,
     rowid: i64,
-) -> SendResult {
+) -> Status {
     if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
         info!(context, "smtp-sending out mime message:");
         println!("{}", message);
@@ -226,20 +217,6 @@ pub(crate) async fn smtp_send(
 
     smtp.connectivity.set_working(context).await;
 
-    if smtp.has_maybe_stale_connection().await {
-        info!(context, "Closing stale connection");
-        smtp.disconnect().await;
-
-        if let Err(err) = smtp
-            .connect_configured(context)
-            .await
-            .context("failed to reopen stale SMTP connection")
-        {
-            smtp.last_send_error = Some(format!("{:#}", err));
-            return SendResult::Retry;
-        }
-    }
-
     let send_result = smtp
         .send(context, recipients, message.as_bytes(), rowid)
         .await;
@@ -276,14 +253,14 @@ pub(crate) async fn smtp_send(
 
                     if maybe_transient {
                         info!(context, "Permanent error that is likely to actually be transient, postponing retry for later");
-                        SendResult::Retry
+                        Status::RetryLater
                     } else {
                         info!(context, "Permanent error, message sending failed");
                         // If we do not retry, add an info message to the chat.
                         // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
                         // should definitely go here, because user has to open the link to
                         // resume message sending.
-                        SendResult::Failure(format_err!("Permanent SMTP error: {}", err))
+                        Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                     }
                 }
                 async_smtp::smtp::error::Error::Transient(ref response) => {
@@ -300,29 +277,35 @@ pub(crate) async fn smtp_send(
                             // receive as a transient error are misconfigurations of the smtp server.
                             // See <https://tools.ietf.org/html/rfc3463#section-3.2>
                             info!(context, "Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", first_word);
-                            SendResult::Failure(format_err!("Permanent SMTP error: {}", err))
+                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                         } else {
                             info!(
                                 context,
                                 "Transient error with status code {}, postponing retry for later",
                                 first_word
                             );
-                            SendResult::Retry
+                            Status::RetryLater
                         }
                     } else {
                         info!(
                             context,
                             "Transient error without status code, postponing retry for later"
                         );
-                        SendResult::Retry
+                        Status::RetryLater
                     }
                 }
                 _ => {
                     info!(
                         context,
-                        "Message sending failed without error returned by the server, retry later"
+                        "Message sending failed without error returned by the server"
                     );
-                    SendResult::Retry
+                    if smtp.has_maybe_stale_connection().await {
+                        info!(context, "Connection is probably stale, retry immediately");
+                        Status::RetryNow
+                    } else {
+                        info!(context, "Connection is not stale, retry later");
+                        Status::RetryLater
+                    }
                 }
             };
 
@@ -336,24 +319,24 @@ pub(crate) async fn smtp_send(
             // Local error, job is invalid, do not retry.
             smtp.disconnect().await;
             warn!(context, "SMTP job is invalid: {}", err);
-            SendResult::Failure(err.into())
+            Status::Finished(Err(err.into()))
         }
         Err(crate::smtp::send::Error::NoTransport) => {
             // Should never happen.
             // It does not even make sense to disconnect here.
             error!(context, "SMTP job failed because SMTP has no transport");
-            SendResult::Failure(format_err!("SMTP has not transport"))
+            Status::Finished(Err(format_err!("SMTP has not transport")))
         }
         Err(crate::smtp::send::Error::Other(err)) => {
             // Local error, job is invalid, do not retry.
             smtp.disconnect().await;
             warn!(context, "unable to load job: {}", err);
-            SendResult::Failure(err)
+            Status::Finished(Err(err))
         }
-        Ok(()) => SendResult::Success,
+        Ok(()) => Status::Finished(Ok(())),
     };
 
-    if let SendResult::Failure(err) = &status {
+    if let Status::Finished(Err(err)) = &status {
         // We couldn't send the message, so mark it as failed
         message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
     }
@@ -449,7 +432,7 @@ pub(crate) async fn send_msg_to_smtp(
         return Ok(());
     }
 
-    let status = smtp_send(
+    let status = match smtp_send(
         context,
         &recipients_list,
         body.as_str(),
@@ -457,25 +440,47 @@ pub(crate) async fn send_msg_to_smtp(
         msg_id,
         rowid,
     )
-    .await;
+    .await
+    {
+        Status::RetryNow => {
+            // Do a single retry immediately without increasing retry counter in case of stale
+            // connection.
+            info!(context, "Doing immediate retry to send message.");
 
+            // smtp_send just closed stale SMTP connection, reconnect and try again.
+            if let Err(err) = smtp
+                .connect_configured(context)
+                .await
+                .context("failed to reopen stale SMTP connection")
+            {
+                smtp.last_send_error = Some(format!("{:#}", err));
+                return Err(err);
+            }
+
+            smtp_send(
+                context,
+                &recipients_list,
+                body.as_str(),
+                smtp,
+                msg_id,
+                rowid,
+            )
+            .await
+        }
+        status => status,
+    };
     match status {
-        SendResult::Retry => {}
-        SendResult::Success | SendResult::Failure(_) => {
+        Status::Finished(res) => {
             context
                 .sql
                 .execute("DELETE FROM smtp WHERE id=?", paramsv![rowid])
                 .await?;
+            if res.is_ok() {
+                msg_id.set_delivered(context).await?;
+            }
+            res
         }
-    };
-
-    match status {
-        SendResult::Retry => Err(format_err!("Retry")),
-        SendResult::Success => {
-            msg_id.set_delivered(context).await?;
-            Ok(())
-        }
-        SendResult::Failure(err) => Err(format_err!("{}", err)),
+        Status::RetryNow | Status::RetryLater => Err(format_err!("Retry")),
     }
 }
 
@@ -483,9 +488,10 @@ pub(crate) async fn send_msg_to_smtp(
 ///
 /// Logs and ignores SMTP errors to ensure that a single SMTP message constantly failing to be sent
 /// does not block other messages in the queue from being sent.
-///
-/// Returns true if all messages were sent successfully, false otherwise.
-pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp) -> Result<bool> {
+pub(crate) async fn send_smtp_messages(
+    context: &Context,
+    connection: &mut Smtp,
+) -> anyhow::Result<()> {
     context.send_sync_msg().await?; // Add sync message to the end of the queue if needed.
     let rowids = context
         .sql
@@ -503,12 +509,10 @@ pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp)
             },
         )
         .await?;
-    let mut success = true;
     for rowid in rowids {
         if let Err(err) = send_msg_to_smtp(context, connection, rowid).await {
             info!(context, "Failed to send message over SMTP: {:#}.", err);
-            success = false;
         }
     }
-    Ok(success)
+    Ok(())
 }

src/sql.rs

@@ -34,17 +34,6 @@ macro_rules! paramsv {
     };
 }
 
-#[macro_export]
-macro_rules! params_iterv {
-    ($($param:expr),+ $(,)?) => {
-        vec![$(&$param as &dyn $crate::ToSql),+]
-    };
-}
-
-pub(crate) fn params_iter(iter: &[impl crate::ToSql]) -> impl Iterator<Item = &dyn crate::ToSql> {
-    iter.iter().map(|item| item as &dyn crate::ToSql)
-}
-
 mod migrations;
 
 /// A wrapper around the underlying Sqlite3 object.
@@ -192,7 +181,6 @@ impl Sql {
                      PRAGMA secure_delete=on;
                      PRAGMA busy_timeout = {};
                      PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
-                     PRAGMA foreign_keys=on;
                      ",
                     Duration::from_secs(10).as_millis()
                 ))?;
@@ -611,55 +599,10 @@ impl Sql {
 }
 
 pub async fn housekeeping(context: &Context) -> Result<()> {
-    if let Err(err) = remove_unused_files(context).await {
-        warn!(
-            context,
-            "Housekeeping: cannot remove unusued files: {}", err
-        );
+    if let Err(err) = crate::ephemeral::delete_expired_messages(context).await {
+        warn!(context, "Failed to delete expired messages: {}", err);
     }
 
-    if let Err(err) = start_ephemeral_timers(context).await {
-        warn!(
-            context,
-            "Housekeeping: cannot start ephemeral timers: {}", err
-        );
-    }
-
-    if let Err(err) = prune_tombstones(&context.sql).await {
-        warn!(
-            context,
-            "Housekeeping: Cannot prune message tombstones: {}", err
-        );
-    }
-
-    if let Err(err) = deduplicate_peerstates(&context.sql).await {
-        warn!(context, "Failed to deduplicate peerstates: {}", err)
-    }
-
-    context.schedule_quota_update().await?;
-
-    // Try to clear the freelist to free some space on the disk. This
-    // only works if auto_vacuum is enabled.
-    if let Err(err) = context
-        .sql
-        .execute("PRAGMA incremental_vacuum", paramsv![])
-        .await
-    {
-        warn!(context, "Failed to run incremental vacuum: {}", err);
-    }
-
-    if let Err(e) = context
-        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
-        .await
-    {
-        warn!(context, "Can't set config: {}", e);
-    }
-
-    info!(context, "Housekeeping done.");
-    Ok(())
-}
-
-pub async fn remove_unused_files(context: &Context) -> Result<()> {
     let mut files_in_use = HashSet::new();
     let mut unreferenced_count = 0;
 
@@ -774,6 +717,44 @@ pub async fn remove_unused_files(context: &Context) -> Result<()> {
         }
     }
 
+    if let Err(err) = start_ephemeral_timers(context).await {
+        warn!(
+            context,
+            "Housekeeping: cannot start ephemeral timers: {}", err
+        );
+    }
+
+    if let Err(err) = prune_tombstones(&context.sql).await {
+        warn!(
+            context,
+            "Housekeeping: Cannot prune message tombstones: {}", err
+        );
+    }
+
+    if let Err(err) = deduplicate_peerstates(&context.sql).await {
+        warn!(context, "Failed to deduplicate peerstates: {}", err)
+    }
+
+    context.schedule_quota_update().await?;
+
+    // Try to clear the freelist to free some space on the disk. This
+    // only works if auto_vacuum is enabled.
+    if let Err(err) = context
+        .sql
+        .execute("PRAGMA incremental_vacuum", paramsv![])
+        .await
+    {
+        warn!(context, "Failed to run incremental vacuum: {}", err);
+    }
+
+    if let Err(e) = context
+        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
+        .await
+    {
+        warn!(context, "Can't set config: {}", e);
+    }
+
+    info!(context, "Housekeeping done.");
     Ok(())
 }
 
@@ -843,10 +824,13 @@ async fn prune_tombstones(sql: &Sql) -> Result<()> {
 ///
 /// Use this together with [`rusqlite::ParamsFromIter`] to use dynamically generated
 /// parameter lists.
-pub fn repeat_vars(count: usize) -> String {
+pub fn repeat_vars(count: usize) -> Result<String> {
+    if count == 0 {
+        bail!("Must have at least one repeat variable");
+    }
     let mut s = "?,".repeat(count);
     s.pop(); // Remove trailing comma
-    s
+    Ok(s)
 }
 
 #[cfg(test)]

src/sql/migrations.rs

@@ -395,7 +395,7 @@ UPDATE chats SET protected=1, type=120 WHERE type=130;"#,
 
     if dbversion < 71 {
         info!(context, "[migration] v71");
-        if let Ok(addr) = context.get_primary_self_addr().await {
+        if let Some(addr) = context.get_config(Config::ConfiguredAddr).await? {
             if let Ok(domain) = addr.parse::<EmailAddress>().map(|email| email.domain) {
                 context
                     .set_config(
@@ -608,22 +608,6 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid);
         )
         .await?;
     }
-    if dbversion < 88 {
-        info!(context, "[migration] v88");
-        sql.execute_migration("DROP TABLE IF EXISTS backup_blobs;", 88)
-            .await?;
-    }
-    if dbversion < 89 {
-        info!(context, "[migration] v89");
-        sql.execute_migration(
-            r#"CREATE TABLE imap_markseen (
-              id INTEGER,
-              FOREIGN KEY(id) REFERENCES imap(id) ON DELETE CASCADE
-            );"#,
-            89,
-        )
-        .await?;
-    }
 
     Ok((
         recalc_fingerprints,

src/stock_str.rs

@@ -10,6 +10,7 @@ use strum_macros::EnumProperty;
 use crate::blob::BlobObject;
 use crate::chat::{self, Chat, ChatId, ProtectionStatus};
 use crate::config::Config;
+use crate::constants::DC_CONTACT_ID_SELF;
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::dc_tools::dc_timestamp_to_str;
@@ -287,6 +288,9 @@ pub enum StockMessage {
     #[strum(props(fallback = "Storage on %1$s"))]
     StorageOnDomain = 105,
 
+    #[strum(props(fallback = "One moment…"))]
+    OneMoment = 106,
+
     #[strum(props(fallback = "Connected"))]
     Connected = 107,
 
@@ -329,9 +333,6 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Scan to join group %1$s"))]
     SecureJoinGroupQRDescription = 120,
-
-    #[strum(props(fallback = "Not connected"))]
-    NotConnected = 121,
 }
 
 impl StockMessage {
@@ -396,7 +397,7 @@ trait StockStringMods: AsRef<str> + Sized {
         Box::pin(async move {
             let message = self.as_ref().trim_end_matches('.');
             match contact_id {
-                ContactId::SELF => msg_action_by_me(context, message).await,
+                DC_CONTACT_ID_SELF => msg_action_by_me(context, message).await,
                 _ => {
                     let displayname = Contact::get_by_id(context, contact_id)
                         .await
@@ -1009,9 +1010,9 @@ pub(crate) async fn storage_on_domain(context: &Context, domain: impl AsRef<str>
         .replace1(domain)
 }
 
-/// Stock string: `Not connected`.
-pub(crate) async fn not_connected(context: &Context) -> String {
-    translated(context, StockMessage::NotConnected).await
+/// Stock string: `One moment…`.
+pub(crate) async fn one_moment(context: &Context) -> String {
+    translated(context, StockMessage::OneMoment).await
 }
 
 /// Stock string: `Connected`.
@@ -1128,7 +1129,7 @@ impl Context {
             self.sql
                 .set_raw_config_bool("self-chat-added", true)
                 .await?;
-            ChatId::create_for_contact(self, ContactId::SELF).await?;
+            ChatId::create_for_contact(self, DC_CONTACT_ID_SELF).await?;
         }
 
         // add welcome-messages. by the label, this is done only once,
@@ -1148,13 +1149,14 @@ impl Context {
 
 #[cfg(test)]
 mod tests {
-    use num_traits::ToPrimitive;
+    use super::*;
+    use crate::test_utils::TestContext;
+
+    use crate::constants::DC_CONTACT_ID_SELF;
 
     use crate::chat::Chat;
     use crate::chatlist::Chatlist;
-    use crate::test_utils::TestContext;
-
-    use super::*;
+    use num_traits::ToPrimitive;
 
     #[test]
     fn test_enum_mapping() {
@@ -1231,7 +1233,7 @@ mod tests {
     async fn test_stock_system_msg_add_member_by_me() {
         let t = TestContext::new().await;
         assert_eq!(
-            msg_add_member(&t, "alice@example.org", ContactId::SELF).await,
+            msg_add_member(&t, "alice@example.org", DC_CONTACT_ID_SELF).await,
             "Member alice@example.org added by me."
         )
     }
@@ -1243,7 +1245,7 @@ mod tests {
             .await
             .expect("failed to create contact");
         assert_eq!(
-            msg_add_member(&t, "alice@example.org", ContactId::SELF).await,
+            msg_add_member(&t, "alice@example.org", DC_CONTACT_ID_SELF).await,
             "Member Alice (alice@example.org) added by me."
         );
     }

src/summary.rs

@@ -1,8 +1,8 @@
 //! # Message summary for chatlist.
 
 use crate::chat::Chat;
-use crate::constants::Chattype;
-use crate::contact::{Contact, ContactId};
+use crate::constants::{Chattype, DC_CONTACT_ID_SELF};
+use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::dc_truncate;
 use crate::message::{Message, MessageState, Viewtype};
@@ -60,7 +60,7 @@ impl Summary {
     ) -> Self {
         let prefix = if msg.state == MessageState::OutDraft {
             Some(SummaryPrefix::Draft(stock_str::draft(context).await))
-        } else if msg.from_id == ContactId::SELF {
+        } else if msg.from_id == DC_CONTACT_ID_SELF {
             if msg.is_info() || chat.is_self_talk() {
                 None
             } else {

src/sync.rs

@@ -2,8 +2,7 @@
 
 use crate::chat::{Chat, ChatId};
 use crate::config::Config;
-use crate::constants::Blocked;
-use crate::contact::ContactId;
+use crate::constants::{Blocked, DC_CONTACT_ID_SELF};
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::message::{Message, MsgId, Viewtype};
@@ -127,7 +126,7 @@ impl Context {
     pub async fn send_sync_msg(&self) -> Result<Option<MsgId>> {
         if let Some((json, ids)) = self.build_sync_json().await? {
             let chat_id =
-                ChatId::create_for_contact_with_blocked(self, ContactId::SELF, Blocked::Yes)
+                ChatId::create_for_contact_with_blocked(self, DC_CONTACT_ID_SELF, Blocked::Yes)
                     .await?;
             let mut msg = Message {
                 chat_id,
@@ -484,7 +483,7 @@ mod tests {
         // check that the used self-talk is not visible to the user
         // but that creation will still work (in this case, the chat is empty)
         assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 0);
-        let chat_id = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
+        let chat_id = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
         let chat = Chat::load_from_db(&alice, chat_id).await?;
         assert!(chat.is_self_talk());
         assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 1);

src/test_utils.rs

@@ -22,8 +22,8 @@ use crate::chat::{self, Chat, ChatId};
 use crate::chatlist::Chatlist;
 use crate::config::Config;
 use crate::constants::Chattype;
-use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
-use crate::contact::{Contact, ContactId, Modifier, Origin};
+use crate::constants::{DC_CONTACT_ID_SELF, DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
+use crate::contact::{Contact, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_receive_imf::dc_receive_imf;
 use crate::dc_tools::EmailAddress;
@@ -88,13 +88,6 @@ impl TestContextBuilder {
         self.with_key_pair(bob_keypair())
     }
 
-    /// Configures as fiona@example.net with fixed secret key.
-    ///
-    /// This is a shortcut for `.with_key_pair(bob_keypair()).
-    pub fn configure_fiona(self) -> Self {
-        self.with_key_pair(fiona_keypair())
-    }
-
     /// Configures the new [`TestContext`] with the provided [`KeyPair`].
     ///
     /// This will extract the email address from the key and configure the context with the
@@ -190,13 +183,6 @@ impl TestContext {
         Self::builder().configure_bob().build().await
     }
 
-    /// Creates a new configured [`TestContext`].
-    ///
-    /// This is a shortcut which configures fiona@example.net with a fixed key.
-    pub async fn new_fiona() -> Self {
-        Self::builder().configure_fiona().build().await
-    }
-
     /// Internal constructor.
     ///
     /// `name` is used to identify this context in e.g. log output.  This is useful mostly
@@ -378,7 +364,7 @@ impl TestContext {
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n"
                 .to_owned()
                 + msg.payload();
-        dc_receive_imf(&self.ctx, received_msg.as_bytes(), false)
+        dc_receive_imf(&self.ctx, received_msg.as_bytes(), "INBOX", false)
             .await
             .unwrap();
     }
@@ -421,7 +407,12 @@ impl TestContext {
             .await
             .unwrap_or_default()
             .unwrap_or_default();
-        let addr = other.ctx.get_primary_self_addr().await.unwrap();
+        let addr = other
+            .ctx
+            .get_config(Config::ConfiguredAddr)
+            .await
+            .unwrap()
+            .unwrap();
         // MailinglistAddress is the lowest allowed origin, we'd prefer to not modify the
         // origin when creating this contact.
         let (contact_id, modified) =
@@ -475,7 +466,7 @@ impl TestContext {
 
     /// Retrieves the "self" chat.
     pub async fn get_self_chat(&self) -> Chat {
-        let chat_id = ChatId::create_for_contact(self, ContactId::SELF)
+        let chat_id = ChatId::create_for_contact(self, DC_CONTACT_ID_SELF)
             .await
             .unwrap();
         Chat::load_from_db(self, chat_id).await.unwrap()
@@ -705,24 +696,6 @@ pub fn bob_keypair() -> KeyPair {
     }
 }
 
-/// Load a pre-generated keypair for fiona@example.net from disk.
-///
-/// Like [alice_keypair] but a different key and identity.
-pub fn fiona_keypair() -> key::KeyPair {
-    let addr = EmailAddress::new("fiona@example.net").unwrap();
-    let public = key::SignedPublicKey::from_asc(include_str!("../test-data/key/fiona-public.asc"))
-        .unwrap()
-        .0;
-    let secret = key::SignedSecretKey::from_asc(include_str!("../test-data/key/fiona-secret.asc"))
-        .unwrap()
-        .0;
-    key::KeyPair {
-        addr,
-        public,
-        secret,
-    }
-}
-
 /// Utility to help wait for and retrieve events.
 ///
 /// This buffers the events in order they are emitted.  This allows consuming events in
@@ -878,7 +851,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.get_from_id() == ContactId::SELF {
+        if msg.get_from_id() == DC_CONTACT_ID_SELF {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"

src/update_helper.rs

@@ -99,6 +99,7 @@ mod tests {
                  Date: Sun, 22 Mar 2021 23:37:57 +0000\n\
                  \n\
                  second message\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -112,6 +113,7 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
                  \n\
                  first message\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -141,6 +143,7 @@ mod tests {
                  Date: Sun, 22 Mar 2021 01:00:00 +0000\n\
                  \n\
                  first message\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -160,6 +163,7 @@ mod tests {
                  Date: Sun, 22 Mar 2021 03:00:00 +0000\n\
                  \n\
                  third message\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -175,6 +179,7 @@ mod tests {
                  Date: Sun, 22 Mar 2021 02:00:00 +0000\n\
                  \n\
                  second message\n",
+            "INBOX",
             false,
         )
         .await?;

src/webxdc.rs

@@ -19,12 +19,6 @@ use std::fs::File;
 use std::io::{Read, Seek, SeekFrom};
 use zip::ZipArchive;
 
-/// The current API version.
-/// If `min_api` in manifest.toml is set to a larger value,
-/// the Webxdc's index.html is replaced by an error message.
-/// In the future, that may be useful to avoid new Webxdc being loaded on old Delta Chats.
-const WEBXDC_API_VERSION: u32 = 1;
-
 pub const WEBXDC_SUFFIX: &str = "xdc";
 const WEBXDC_DEFAULT_ICON: &str = "__webxdc__/default-icon.png";
 
@@ -50,7 +44,6 @@ const WEBXDC_RECEIVING_LIMIT: u64 = 4194304;
 #[non_exhaustive]
 struct WebxdcManifest {
     name: Option<String>,
-    min_api: Option<u32>,
 }
 
 /// Parsed information from WebxdcManifest and fallbacks.
@@ -226,7 +219,6 @@ impl Context {
                 info.as_str(),
                 SystemMessage::Unknown,
                 timestamp,
-                None,
                 Some(instance),
             )
             .await?;
@@ -239,7 +231,10 @@ impl Context {
             {
                 instance.param.set(Param::WebxdcSummary, summary);
                 instance.update_param(self).await;
-                self.emit_msgs_changed(instance.chat_id, instance.id);
+                self.emit_event(EventType::MsgsChanged {
+                    chat_id: instance.chat_id,
+                    msg_id: instance.id,
+                });
             }
         }
 
@@ -251,14 +246,9 @@ impl Context {
             )
             .await?;
 
-        let status_update_serial = StatusUpdateSerial(u32::try_from(rowid)?);
+        self.emit_event(EventType::WebxdcStatusUpdate(instance.id));
 
-        self.emit_event(EventType::WebxdcStatusUpdate {
-            msg_id: instance.id,
-            status_update_serial,
-        });
-
-        Ok(status_update_serial)
+        Ok(StatusUpdateSerial(u32::try_from(rowid)?))
     }
 
     /// Sends a status update for an webxdc instance.
@@ -508,21 +498,6 @@ impl Message {
         };
 
         let mut archive = self.get_webxdc_archive(context).await?;
-
-        if name == "index.html" {
-            if let Ok(bytes) = get_blob(&mut archive, "manifest.toml").await {
-                if let Ok(manifest) = parse_webxdc_manifest(&bytes).await {
-                    if let Some(min_api) = manifest.min_api {
-                        if min_api > WEBXDC_API_VERSION {
-                            return Ok(Vec::from(
-                                "<!DOCTYPE html>This Webxdc requires a newer Delta Chat version.",
-                            ));
-                        }
-                    }
-                }
-            }
-        }
-
         get_blob(&mut archive, name).await
     }
 
@@ -535,16 +510,10 @@ impl Message {
             if let Ok(manifest) = parse_webxdc_manifest(&bytes).await {
                 manifest
             } else {
-                WebxdcManifest {
-                    name: None,
-                    min_api: None,
-                }
+                WebxdcManifest { name: None }
             }
         } else {
-            WebxdcManifest {
-                name: None,
-                min_api: None,
-            }
+            WebxdcManifest { name: None }
         };
 
         if let Some(ref name) = manifest.name {
@@ -771,6 +740,7 @@ mod tests {
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/webxdc_good_extension.eml"),
+            "INBOX",
             false,
         )
         .await?;
@@ -781,6 +751,7 @@ mod tests {
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/webxdc_bad_extension.eml"),
+            "INBOX",
             false,
         )
         .await?;
@@ -1016,10 +987,7 @@ mod tests {
             .get_matching(|evt| matches!(evt, EventType::WebxdcStatusUpdate { .. }))
             .await;
         match event {
-            EventType::WebxdcStatusUpdate {
-                msg_id,
-                status_update_serial: _,
-            } => {
+            EventType::WebxdcStatusUpdate(msg_id) => {
                 assert_eq!(msg_id, instance_id);
             }
             _ => unreachable!(),
@@ -1326,38 +1294,6 @@ sth_for_the = "future""#
         )
         .await?;
         assert_eq!(manifest.name, Some("foz".to_string()));
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_parse_webxdc_manifest_min_api() -> Result<()> {
-        let manifest = parse_webxdc_manifest(r#"min_api = 3"#.as_bytes()).await?;
-        assert_eq!(manifest.min_api, Some(3));
-
-        let result = parse_webxdc_manifest(r#"min_api = "1""#.as_bytes()).await;
-        assert!(result.is_err());
-
-        let result = parse_webxdc_manifest(r#"min_api = 1.2"#.as_bytes()).await;
-        assert!(result.is_err());
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_webxdc_min_api_too_large() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "chat").await?;
-        let mut instance = create_webxdc_instance(
-            &t,
-            "with-min-api-1001.xdc",
-            include_bytes!("../test-data/webxdc/with-min-api-1001.xdc"),
-        )
-        .await?;
-        send_msg(&t, chat_id, &mut instance).await?;
-
-        let instance = t.get_last_msg().await;
-        let html = instance.get_webxdc_blob(&t, "index.html").await?;
-        assert!(String::from_utf8_lossy(&*html).contains("requires a newer Delta Chat version"));
 
         Ok(())
     }