Fix two tests (Tokio)

* ignore status/footer updates from mailinglist messages

mailinglist software often modified existing footers
or adds footers on their own.

therefore,therefore, these footers often do not reflect the status/footer set by the user.

* test status footers not updated from mailinglists

.github/mergeable.yml

@@ -22,5 +22,5 @@ mergeable:
       - do: checks
         status: 'action_required' 
         payload:
-          title: Changlog might need an update
+          title: Changelog might need an update
           summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on:
 
 env:
   RUSTFLAGS: -Dwarnings
-
+   
 jobs:
 
   fmt:
@@ -83,13 +83,13 @@ jobs:
             rust: 1.61.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.56.0
+          # Minimum Supported Rust Version = 1.56.1
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.56.0
+            rust: 1.56.1
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/node-package.yml

@@ -120,6 +120,9 @@ jobs:
       - name: install dependencies without running scripts
         run: |
           npm install --ignore-scripts
+      - name: build constants
+        run: |
+          npm run build:core:constants
       - name: build typescript part
         run: |
           npm run build:bindings:ts

CHANGELOG.md

@@ -2,10 +2,79 @@
 
 ## Unreleased
 
+### Changes
+- switch from `async-std` to `tokio` as the async runtime #3449
+
+### Fixes
+- mailing list: remove square-brackets only for first name #3452
+- do not use footers from mailinglists as the contact status #3460
+
+
+## 1.87.0
+
+### Changes
+- limit the rate of MDN sending #3402
+- ignore ratelimits for bots #3439
+- remove `msgs_mdns` references to deleted messages during housekeeping #3387
+- format message lines starting with `>` as quotes #3434
+- node: remove `split2` dependency #3418
+- node: add git installation info to readme #3418
+- limit the rate of webxdc update sending #3417
+
+### Fixes
+- set a default error if NDN does not provide an error
+- python: avoid exceptions when messages/contacts/chats are compared with `None`
+- node: wait for the event loop to stop before destroying contexts #3431 #3451
+- emit configuration errors via event on failure #3433
+- report configure and imex success/failure after freeing ongoing process #3442
+
+### API-Changes
+- python: added `Message.get_status_updates()`  #3416
+- python: added `Message.send_status_update()`  #3416
+- python: added `Message.is_webxdc()`  #3416
+- python: added `Message.is_videochat_invitation()`  #3416
+- python: added support for "videochat" and "webxdc" view types to `Message.new_empty()`  #3416
+
+
+## 1.86.0
+
+### API-Changes
+- python: added optional `closed` parameter to `Account` constructor #3394
+- python: added optional `passphrase` parameter to `Account.export_all()` and `Account.import_all()` #3394
+- python: added `Account.open()` #3394
+- python: added `Chat.is_single()` #3394
+- python: added `Chat.is_mailinglist()` #3394
+- python: added `Chat.is_broadcast()` #3394
+- python: added `Chat.is_multiuser()` #3394
+- python: added `Chat.is_self_talk()` #3394
+- python: added `Chat.is_device_talk()` #3394
+- python: added `Chat.is_pinned()` #3394
+- python: added `Chat.pin()` #3394
+- python: added `Chat.unpin()` #3394
+- python: added `Chat.archive()` #3394
+- python: added `Chat.unarchive()` #3394
+- python: added `Message.get_summarytext()` #3394
+- python: added optional `closed` parameter to `ACFactory.get_unconfigured_account()` (pytest plugin) #3394
+- python: added optional `passphrase` parameter to `ACFactory.get_pseudo_configured_account()` (pytest plugin) #3394
+
+### Changes
+- clean up series of webxdc info messages;
+  `DC_EVENT_MSGS_CHANGED` is emitted on changes of existing info messages #3395
+- update provider database #3399
+- refactorings #3375 #3403 #3398 #3404
+
+### Fixes
+- do not reset our database if imported backup cannot be decrypted #3397
+- node: remove `npx` from build script, this broke flathub build #3396
+
+
+## 1.85.0
+
 ### Changes
 - refactorings #3373 #3345 #3380 #3382
 - node: move split2 to devDependencies
-
+- python: build Python 3.10 wheels #3392
+- update Rust dependencies
 
 ### Fixes
 - delete outgoing MDNs found in the Sent folder on Gmail #3372
@@ -34,10 +103,11 @@
 ### Fixes
 - fix node prebuild & package ci #3337
 
+
 ## 1.82.0
 
 ### API-Changes
-- re-add removed DC_MSG_ID_MARKER1 as in use on iOS #3330
+- re-add removed `DC_MSG_ID_MARKER1` as in use on iOS #3330
 
 ### Changes
 - refactorings #3328

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.84.0"
+version = "1.87.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -9,6 +9,7 @@ rust-version = "1.56"
 [profile.dev]
 debug = 0
 panic = 'abort'
+opt-level = 1
 
 [profile.release]
 lto = true
@@ -19,19 +20,19 @@ deltachat_derive = { path = "./deltachat_derive" }
 
 ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
-async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", default-features=false, features = ["smtp-transport", "socks5"] }
-async-std-resolver = "0.21"
-async-std = { version = "1" }
-async-tar = { version = "0.4", default-features=false }
+async-imap = { version = "0.6", default-features = false, features = ["runtime-tokio"] }
+async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] }
+async-smtp = { version = "0.5", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] }
+trust-dns-resolver = "0.21"
+tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
+tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
 backtrace = "0.3"
 base64 = "0.13"
 bitflags = "1.3"
-chrono = "0.4"
+chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
 dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+encoded-words = "0.2"
 escaper = "0.1"
 futures = "0.3"
 hex = "0.4.0"
@@ -65,22 +66,24 @@ sha2 = "0.10"
 smallvec = "1"
 strum = "0.24"
 strum_macros = "0.24"
-surf = { version = "2.3", default-features = false, features = ["h1-client"] }
 thiserror = "1"
 toml = "0.5"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
-fast-socks5 = "0.4"
+fast-socks5 = "0.8"
 humansize = "1"
 qrcodegen = "1.7.0"
 tagger = "4.3.3"
 textwrap = "0.15.0"
-zip = { version = "0.6.2", default-features = false, features = ["deflate"] }
+async-channel = "1.6.1"
+futures-lite = "1.12.0"
+tokio-stream = { version = "0.1.9", features = ["fs"] }
+reqwest = { version = "0.11.11", features = ["json"] }
+async_zip = { git = "https://github.com/dignifiedquire/rs-async-zip", branch = "main", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = { version = "0.3.4", features = ["async_std"] }
+criterion = { version = "0.3.4", features = ["async_tokio"] }
 futures-lite = "1.12"
 log = "0.4"
 pretty_env_logger = "0.4"
@@ -132,5 +135,10 @@ harness = false
 default = ["vendored"]
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl"]
+vendored = [
+  "async-native-tls/vendored",
+  "async-smtp/native-tls-vendored",
+  "rusqlite/bundled-sqlcipher-vendored-openssl",
+  "reqwest/native-tls-vendored"
+]
 nightly = ["pgp/nightly"]

README.md

@@ -102,9 +102,6 @@ $ cargo build -p deltachat_ffi --release
 
 ## Debugging environment variables 
 
-- `DCC_IMAP_DEBUG`: if set IMAP protocol commands and responses will be
-  printed
-
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
 - `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and

benches/contacts.rs

@@ -1,14 +1,14 @@
-use async_std::task::block_on;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::contact::Contact;
 use deltachat::context::Context;
+use deltachat::Events;
 use tempfile::tempdir;
 
 async fn address_book_benchmark(n: u32, read_count: u32) {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(&dbfile, id, Events::new()).await.unwrap();
 
     let book = (0..n)
         .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
@@ -24,12 +24,16 @@ async fn address_book_benchmark(n: u32, read_count: u32) {
 }
 
 fn criterion_benchmark(c: &mut Criterion) {
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
     c.bench_function("create 500 contacts", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
+        b.to_async(&rt)
+            .iter(|| async { address_book_benchmark(black_box(500), black_box(0)).await })
     });
 
     c.bench_function("create 100 contacts and read it 1000 times", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
+        b.to_async(&rt)
+            .iter(|| async { address_book_benchmark(black_box(100), black_box(1000)).await })
     });
 }
 

benches/create_account.rs

@@ -1,12 +1,11 @@
-use async_std::path::PathBuf;
-use async_std::task::block_on;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::accounts::Accounts;
+use std::path::PathBuf;
 use tempfile::tempdir;
 
 async fn create_accounts(n: u32) {
     let dir = tempdir().unwrap();
-    let p: PathBuf = dir.path().join("accounts").into();
+    let p: PathBuf = dir.path().join("accounts");
 
     let mut accounts = Accounts::new(p.clone()).await.unwrap();
 
@@ -18,7 +17,8 @@ async fn create_accounts(n: u32) {
 
 fn criterion_benchmark(c: &mut Criterion) {
     c.bench_function("create 1 account", |b| {
-        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        b.to_async(&rt).iter(|| create_accounts(black_box(1)))
     });
 }
 

benches/get_chat_msgs.rs

@@ -1,15 +1,15 @@
-use async_std::path::Path;
+use std::path::Path;
 
-use criterion::async_executor::AsyncStdExecutor;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 
 use deltachat::chat::{self, ChatId};
 use deltachat::chatlist::Chatlist;
 use deltachat::context::Context;
+use deltachat::Events;
 
 async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(dbfile, id, Events::new()).await.unwrap();
 
     for c in chats.iter().take(10) {
         black_box(chat::get_chat_msgs(&context, *c, 0).await.ok());
@@ -20,15 +20,19 @@ fn criterion_benchmark(c: &mut Criterion) {
     // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
     // messages, such as your primary account.
     if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let chats: Vec<_> = async_std::task::block_on(async {
-            let context = Context::new((&path).into(), 100).await.unwrap();
+        let rt = tokio::runtime::Runtime::new().unwrap();
+
+        let chats: Vec<_> = rt.block_on(async {
+            let context = Context::new(Path::new(&path), 100, Events::new())
+                .await
+                .unwrap();
             let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
             let len = chatlist.len();
             (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
         });
 
         c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
-            b.to_async(AsyncStdExecutor)
+            b.to_async(&rt)
                 .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
         });
     } else {

benches/get_chatlist.rs

@@ -1,8 +1,10 @@
-use criterion::async_executor::AsyncStdExecutor;
+use std::path::Path;
+
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 
 use deltachat::chatlist::Chatlist;
 use deltachat::context::Context;
+use deltachat::Events;
 
 async fn get_chat_list_benchmark(context: &Context) {
     Chatlist::try_load(context, 0, None, None).await.unwrap();
@@ -12,10 +14,14 @@ fn criterion_benchmark(c: &mut Criterion) {
     // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
     // messages, such as your primary account.
     if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let context =
-            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let context = rt.block_on(async {
+            Context::new(Path::new(&path), 100, Events::new())
+                .await
+                .unwrap()
+        });
         c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
-            b.to_async(AsyncStdExecutor)
+            b.to_async(&rt)
                 .iter(|| get_chat_list_benchmark(black_box(&context)))
         });
     } else {

benches/receive_emails.rs

@@ -1,13 +1,12 @@
-use async_std::{path::PathBuf, task::block_on};
-use criterion::{
-    async_executor::AsyncStdExecutor, black_box, criterion_group, criterion_main, BatchSize,
-    Criterion,
-};
+use std::path::PathBuf;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::{
     config::Config,
     context::Context,
     dc_receive_imf::dc_receive_imf,
     imex::{imex, ImexMode},
+    Events,
 };
 use tempfile::tempdir;
 
@@ -42,13 +41,13 @@ async fn create_context() -> Context {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(&dbfile, id, Events::new()).await.unwrap();
 
     let backup: PathBuf = std::env::current_dir()
         .unwrap()
-        .join("delta-chat-backup.tar")
-        .into();
-    if backup.exists().await {
+        .join("delta-chat-backup.tar");
+
+    if backup.exists() {
         println!("Importing backup");
         imex(&context, ImexMode::ImportBackup, &backup, None)
             .await
@@ -71,11 +70,15 @@ async fn create_context() -> Context {
 fn criterion_benchmark(c: &mut Criterion) {
     let mut group = c.benchmark_group("Receive messages");
     group.bench_function("Receive 100 simple text msgs", |b| {
-        b.to_async(AsyncStdExecutor).iter_batched(
-            || block_on(create_context()),
-            |context| recv_all_emails(black_box(context)),
-            BatchSize::LargeInput,
-        );
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let context = rt.block_on(create_context());
+
+        b.to_async(&rt).iter(|| {
+            let ctx = context.clone();
+            async move {
+                recv_all_emails(black_box(ctx)).await;
+            }
+        });
     });
     group.finish();
 }

benches/search_msgs.rs

@@ -1,12 +1,13 @@
-use async_std::task::block_on;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::context::Context;
+use deltachat::Events;
 use std::path::Path;
 
-async fn search_benchmark(path: impl AsRef<Path>) {
-    let dbfile = path.as_ref();
+async fn search_benchmark(dbfile: impl AsRef<Path>) {
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(dbfile.as_ref(), id, Events::new())
+        .await
+        .unwrap();
 
     for _ in 0..10u32 {
         context.search_msgs(None, "hello").await.unwrap();
@@ -17,8 +18,10 @@ fn criterion_benchmark(c: &mut Criterion) {
     // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
     // messages, such as your primary account.
     if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+
         c.bench_function("search hello", |b| {
-            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
+            b.to_async(&rt).iter(|| search_benchmark(black_box(&path)))
         });
     } else {
         println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.84.0"
+version = "1.87.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,10 +20,11 @@ libc = "0.2"
 human-panic = "1"
 num-traits = "0.2"
 serde_json = "1.0"
-async-std = "1"
+tokio = { version = "1", features = ["rt-multi-thread"] }
 anyhow = "1"
 thiserror = "1"
 rand = "0.7"
+once_cell = "1.12.0"
 
 [features]
 default = ["vendored"]

deltachat-ffi/deltachat.h

@@ -1791,6 +1791,11 @@ void            dc_delete_msgs               (dc_context_t* context, const uint3
 /**
  * Forward messages to another chat.
  *
+ * All types of messages can be forwarded,
+ * however, they will be flagged as such (dc_msg_is_forwarded() is set).
+ *
+ * Original sender, info-state and webxdc updates are not forwarded on purpose.
+ *
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_ids An array of uint32_t containing all message IDs that should be forwarded.
@@ -3979,7 +3984,7 @@ int             dc_msg_is_sent                (const dc_msg_t* msg);
  *
  * For privacy reasons, we do not provide the name or the e-mail address of the
  * original author (in a typical GUI, you select the messages text and click on
- * "forwared"; you won't expect other data to be send to the new recipient,
+ * "forwarded"; you won't expect other data to be send to the new recipient,
  * esp. as the new recipient may not be in any relationship to the original author)
  *
  * @memberof dc_msg_t

deltachat-ffi/src/lib.rs

@@ -1,4 +1,4 @@
-#![deny(clippy::all)]
+#![deny(unused, clippy::all)]
 #![allow(
     non_camel_case_types,
     non_snake_case,
@@ -11,23 +11,23 @@
 
 #[macro_use]
 extern crate human_panic;
-extern crate num_traits;
-extern crate serde_json;
 
 use std::collections::BTreeMap;
 use std::convert::TryFrom;
 use std::fmt::Write;
+use std::future::Future;
 use std::ops::Deref;
 use std::ptr;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use anyhow::Context as _;
-use async_std::sync::RwLock;
-use async_std::task::{block_on, spawn};
 use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use num_traits::{FromPrimitive, ToPrimitive};
+use once_cell::sync::Lazy;
 use rand::Rng;
+use tokio::runtime::Runtime;
+use tokio::sync::RwLock;
 
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
@@ -40,6 +40,7 @@ use deltachat::stock_str::StockMessage;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
+use tokio::task::JoinHandle;
 
 mod dc_array;
 mod lot;
@@ -63,6 +64,23 @@ use deltachat::chatlist::Chatlist;
 /// Struct representing the deltachat context.
 pub type dc_context_t = Context;
 
+static RT: Lazy<Runtime> = Lazy::new(|| Runtime::new().expect("unable to create tokio runtime"));
+
+fn block_on<T>(fut: T) -> T::Output
+where
+    T: Future,
+{
+    RT.block_on(fut)
+}
+
+fn spawn<T>(fut: T) -> JoinHandle<T::Output>
+where
+    T: Future + Send + 'static,
+    T::Output: Send + 'static,
+{
+    RT.spawn(fut)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_context_new(
     _os_name: *const libc::c_char,
@@ -79,7 +97,7 @@ pub unsafe extern "C" fn dc_context_new(
     let ctx = if blobdir.is_null() || *blobdir == 0 {
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
-        block_on(Context::new(as_path(dbfile).to_path_buf().into(), id))
+        block_on(Context::new(as_path(dbfile), id, Events::new()))
     } else {
         eprintln!("blobdir can not be defined explicitly anymore");
         return ptr::null_mut();
@@ -103,10 +121,7 @@ pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *
     }
 
     let id = rand::thread_rng().gen();
-    match block_on(Context::new_closed(
-        as_path(dbfile).to_path_buf().into(),
-        id,
-    )) {
+    match block_on(Context::new_closed(as_path(dbfile), id, Events::new())) {
         Ok(context) => Box::into_raw(Box::new(context)),
         Err(err) => {
             eprintln!("failed to create context: {:#}", err);
@@ -678,10 +693,13 @@ pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *
     }
     let events = &*events;
 
-    events
-        .recv_sync()
-        .map(|ev| Box::into_raw(Box::new(ev)))
-        .unwrap_or_else(ptr::null_mut)
+    block_on(async move {
+        events
+            .recv()
+            .await
+            .map(|ev| Box::into_raw(Box::new(ev)))
+            .unwrap_or_else(ptr::null_mut)
+    })
 }
 
 #[no_mangle]
@@ -2390,7 +2408,7 @@ pub unsafe extern "C" fn dc_get_last_error(context: *mut dc_context_t) -> *mut l
         return "".strdup();
     }
     let ctx = &*context;
-    block_on(ctx.get_last_error()).strdup()
+    ctx.get_last_error().strdup()
 }
 
 // dc_array_t
@@ -3576,7 +3594,8 @@ pub unsafe extern "C" fn dc_msg_latefiling_mediasize(
         ffi_msg
             .message
             .latefiling_mediasize(ctx, width, height, duration)
-    });
+    })
+    .ok_or_log_msg(ctx, "Cannot set media size");
 }
 
 #[no_mangle]
@@ -4122,7 +4141,7 @@ pub unsafe extern "C" fn dc_accounts_new(
         return ptr::null_mut();
     }
 
-    let accs = block_on(Accounts::new(as_path(dbfile).to_path_buf().into()));
+    let accs = block_on(Accounts::new(as_path(dbfile).into()));
 
     match accs {
         Ok(accs) => Box::into_raw(Box::new(AccountsWrapper::new(accs))),
@@ -4294,7 +4313,7 @@ pub unsafe extern "C" fn dc_accounts_migrate_account(
     block_on(async move {
         let mut accounts = accounts.write().await;
         match accounts
-            .migrate_account(async_std::path::PathBuf::from(dbfile))
+            .migrate_account(std::path::PathBuf::from(dbfile))
             .await
         {
             Ok(id) => id,
@@ -4377,7 +4396,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
-pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
+pub type dc_accounts_event_emitter_t = EventEmitter;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_get_event_emitter(
@@ -4414,9 +4433,7 @@ pub unsafe extern "C" fn dc_accounts_get_next_event(
         return ptr::null_mut();
     }
     let emitter = &mut *emitter;
-
-    emitter
-        .recv_sync()
+    block_on(emitter.recv())
         .map(|ev| Box::into_raw(Box::new(ev)))
         .unwrap_or_else(ptr::null_mut)
 }

draft/webxdc-dev-reference.md

@@ -1,6 +1,12 @@
 # Webxdc Developer Reference
 
-(This document may eventually be merged with the [webxdc guidebook](https://deltachat.github.io/webxdc_docs/), where you may currently find other useful information.)
+This document gives a quick overview about the Webxdc specification,
+It is meant for both, developing Webxdc apps
+and developing Webxdc implementations.
+
+The [Webxdc guidebook](https://deltachat.github.io/webxdc_docs/) shows more detailed information
+when developing Webxdc apps.
+
 
 ## Webxdc File Format
 
@@ -34,8 +40,9 @@ To get a shared state, the peers use `sendUpdate()` to send updates to each othe
 - `update`: an object with the following properties:  
     - `update.payload`: any javascript primitive, array or object.
     - `update.info`: optional, short, informational message that will be added to the chat,
-       eg. "Alice voted" or "Bob scored 123 in MyGame";
-       usually only one line of text is shown,
+       eg. "Alice voted" or "Bob scored 123 in MyGame".
+       usually only one line of text is shown
+       and if there are series of info messages, older ones may be dropped.
        use this option sparingly to not spam the chat.
     - `update.document`: optional, name of the document in edit,
        must not be used eg. in games where the Webxdc does not create documents
@@ -137,9 +144,37 @@ round corners etc. will be added by the implementations as needed.
 If no icon is set, a default icon will be used.
 
 
+## Other APIs and Tags Usage Hints
+
+- `localStorage`, `sessionStorage`, `indexedDB` are okay to be used
+- `visibilitychange`-events are okay to be used
+- `window.navigator.language` is okay to be used, on desktop it is the system language
+- `<a href="localfile.html">` and other internal links are okay to be used
+- `<a href="mailto:addr@example.org?body=...">`- mailto links are okay to be used
+- `<meta name="viewport" ...>` usage is okay to be used
+  and useful esp. different webviews have different defaults
+
+
+### Discouraged Things
+
+- `document.cookie` is known not to work on desktop and iOS
+  use `localStorage` instead
+- `unload`-, `beforeunload`- and `pagehide`-events are known not to work on iOS and are flaky on other systems
+  (also partly discouraged by [mozilla](https://developer.mozilla.org/en-US/docs/Web/API/Window/unload_event))
+  use `visibilitychange` instead
+- `<title>` and `document.title` is ignored by Webxdc;
+  use the `name` property from `manifest.toml` instead
+- newest js features may not work on all webviews,
+  you may want to transpile your code down to an older js version
+  eg. with <https://babeljs.io>
+- `<a href="https://example.org/foo">` and other external links are blocked by definition;
+  instead, embed content or use `mailto:` link to offer a way for contact
+- `<input type="file">` is discouraged currently; this may change in future
+
+
 ## Webxdc Examples
 
-The following example shows an input field and  every input is show on all peers.
+The following example shows an input field and every input is show on all peers.
 
 ```html
 <!DOCTYPE html>
@@ -169,30 +204,10 @@ The following example shows an input field and  every input is show on all peers
 </html>
 ```
 
-[Webxdc Development Tool](https://github.com/deltachat/webxdc-dev)
-offers an **Webxdc Simulator** that can be used in many browsers without any installation needed.
+More examples at [github.com/webxdc](https://github.com/webxdc) and
+[topic #webxdc](https://github.com/topics/webxdc)
+
+[github.com/webxdc/hello](https://github.com/webxdc/hello)
+offers an **Webxdc Tool** that can be used in many browsers without any installation needed.
 You can also use that repository as a template for your own Webxdc -
 just clone and start adapting things to your need.
-
-
-### Advanced Examples
-
-- [2048](https://github.com/adbenitez/2048.xdc)
-- [Draw](https://github.com/adbenitez/draw.xdc)
-- [Poll](https://github.com/r10s/webxdc-poll/)
-- [Tic Tac Toe](https://github.com/Simon-Laux/tictactoe.xdc)
-- Even more with [Topic #webxdc on Github](https://github.com/topics/webxdc) or in the [webxdc GitHub organization](https://github.com/webxdc)
-
-
-## Closing Remarks
-
-- older devices might not have the newest js features in their webview,
-  you may want to transpile your code down to an older js version eg. with https://babeljs.io
-- viewport and scaling features are implementation specific,
-  if you want to have an explicit behavior, you can add eg.
-  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
-- the `<title>` tag should not be used and its content is usually not displayed;
-  instead, use the `name` property from `manifest.toml`
-- there are tons of ideas for enhancements of the API and the file format,
-  eg. in the future, we will may define icon- and manifest-files,
-  allow to aggregate the state or add metadata.

examples/repl/cmdline.rs

@@ -1,9 +1,10 @@
 extern crate dirs;
 
+use std::path::Path;
 use std::str::FromStr;
+use std::time::{Duration, SystemTime};
 
 use anyhow::{bail, ensure, Result};
-use async_std::path::Path;
 use deltachat::chat::{
     self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
 };
@@ -22,8 +23,7 @@ use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
 use deltachat::{config, provider};
-use std::fs;
-use std::time::{Duration, SystemTime};
+use tokio::fs;
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
@@ -135,17 +135,13 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     } else {
         /* import a directory */
         let dir_name = std::path::Path::new(&real_spec);
-        let dir = std::fs::read_dir(dir_name);
+        let dir = fs::read_dir(dir_name).await;
         if dir.is_err() {
             error!(context, "Import: Cannot open directory \"{}\".", &real_spec,);
             return false;
         } else {
-            let dir = dir.unwrap();
-            for entry in dir {
-                if entry.is_err() {
-                    break;
-                }
-                let entry = entry.unwrap();
+            let mut dir = dir.unwrap();
+            while let Ok(Some(entry)) = dir.next_entry().await {
                 let name_f = entry.file_name();
                 let name = name_f.to_string_lossy();
                 if name.ends_with(".eml") {
@@ -492,7 +488,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let setup_code = create_setup_code(&context);
             let file_name = blobdir.join("autocrypt-setup-message.html");
             let file_content = render_setup_file(&context, &setup_code).await?;
-            async_std::fs::write(&file_name, file_content).await?;
+            fs::write(&file_name, file_content).await?;
             println!(
                 "Setup message written to: {}\nSetup code: {}",
                 file_name.display(),
@@ -532,7 +528,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 .join("connectivity.html");
             match context.get_connectivity_html().await {
                 Ok(html) => {
-                    fs::write(&file, html)?;
+                    fs::write(&file, html).await?;
                     println!("Report written to: {:#?}", file);
                 }
                 Err(err) => {
@@ -892,7 +888,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "No html-file given.");
             let path: &Path = arg1.as_ref();
-            let html = &*fs::read(&path)?;
+            let html = &*fs::read(&path).await?;
             let html = String::from_utf8_lossy(html);
 
             let mut msg = Message::new(Viewtype::Text);
@@ -1079,7 +1075,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 .unwrap_or_default()
                 .join(format!("msg-{}.html", id.to_u32()));
             let html = id.get_html(&context).await?.unwrap_or_default();
-            fs::write(&file, html)?;
+            fs::write(&file, html).await?;
             println!("HTML written to: {:#?}", file);
         }
         "listfresh" => {

examples/repl/main.rs

@@ -9,17 +9,18 @@ extern crate deltachat;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
+use std::path::Path;
 use std::process::Command;
 
 use ansi_term::Color;
 use anyhow::{bail, Error};
-use async_std::path::Path;
 use deltachat::chat::ChatId;
 use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
+use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use deltachat::securejoin::*;
-use deltachat::EventType;
+use deltachat::{EventType, Events};
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
@@ -30,11 +31,11 @@ use rustyline::validate::Validator;
 use rustyline::{
     Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
 };
+use tokio::fs;
+use tokio::runtime::Handle;
 
 mod cmdline;
 use self::cmdline::*;
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
-use std::fs;
 
 /// Event Handler
 fn receive_event(event: EventType) {
@@ -298,10 +299,10 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new(Path::new(&args[1]), 0, Events::new()).await?;
 
     let events = context.get_event_emitter();
-    async_std::task::spawn(async move {
+    tokio::task::spawn(async move {
         while let Some(event) = events.recv().await {
             receive_event(event.typ);
         }
@@ -316,8 +317,9 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::channel::bounded(100);
-    let input_loop = async_std::task::spawn_blocking(move || {
+
+    let ctx = context.clone();
+    let input_loop = tokio::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
             highlighter: MatchingBracketHighlighter::new(),
@@ -339,16 +341,30 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line)).unwrap();
+                    let contine = Handle::current().block_on(async {
+                        match handle_cmd(line.trim(), ctx.clone(), &mut selected_chat).await {
+                            Ok(ExitResult::Continue) => true,
+                            Ok(ExitResult::Exit) => {
+                                println!("Exiting ...");
+                                false
+                            }
+                            Err(err) => {
+                                println!("Error: {}", err);
+                                true
+                            }
+                        }
+                    });
+
+                    if !contine {
+                        break;
+                    }
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
-                    drop(reader_s);
                     break;
                 }
                 Err(err) => {
                     println!("Error: {}", err);
-                    drop(reader_s);
                     break;
                 }
             }
@@ -359,15 +375,8 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         Ok::<_, Error>(())
     });
 
-    while let Ok(line) = reader_r.recv().await {
-        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
-            Ok(ExitResult::Continue) => {}
-            Ok(ExitResult::Exit) => break,
-            Err(err) => println!("Error: {}", err),
-        }
-    }
     context.stop_io().await;
-    input_loop.await?;
+    input_loop.await??;
 
     Ok(())
 }
@@ -437,7 +446,7 @@ async fn handle_cmd(
             let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
             match get_securejoin_qr_svg(&ctx, group).await {
                 Ok(svg) => {
-                    fs::write(&file, svg)?;
+                    fs::write(&file, svg).await?;
                     println!("QR code svg written to: {:#?}", file);
                 }
                 Err(err) => {
@@ -458,11 +467,12 @@ async fn handle_cmd(
     Ok(ExitResult::Continue)
 }
 
-fn main() -> Result<(), Error> {
+#[tokio::main]
+async fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
     let args = std::env::args().collect();
-    async_std::task::block_on(async move { start(args).await })?;
+    start(args).await?;
 
     Ok(())
 }

examples/simple.rs

@@ -6,7 +6,7 @@ use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::message::Message;
-use deltachat::EventType;
+use deltachat::{EventType, Events};
 
 fn cb(event: EventType) {
     match event {
@@ -29,21 +29,21 @@ fn cb(event: EventType) {
 }
 
 /// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
-#[async_std::main]
+#[tokio::main]
 async fn main() {
     pretty_env_logger::try_init_timed().ok();
 
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new(dbfile.into(), 0)
+    let ctx = Context::new(&dbfile, 0, Events::new())
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
     log::info!("info: {:#?}", info);
 
     let events = ctx.get_event_emitter();
-    let events_spawn = async_std::task::spawn(async move {
+    let events_spawn = tokio::task::spawn(async move {
         while let Some(event) = events.recv().await {
             cb(event.typ);
         }
@@ -80,7 +80,7 @@ async fn main() {
     }
 
     // wait for the message to be sent out
-    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
 
     log::info!("fetching chats..");
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
@@ -96,5 +96,5 @@ async fn main() {
     ctx.stop_io().await;
     log::info!("closing");
     drop(ctx);
-    events_spawn.await;
+    events_spawn.await.unwrap();
 }

node/README.md

@@ -60,6 +60,19 @@ building from source or clone this repository and follow this steps:
 > not inside this folder. (We need this in order to include the rust source
 > code in the npm package.)
 
+### Use a git branch in deltachat-desktop
+
+You can directly install a core branch, but make sure:
+- that you have typescript in your project dependencies, as it is likely required
+- you know that there are **no prebuilds** and so core is built during installation which is why it takes so long
+
+```
+npm install https://github.com/deltachat/deltachat-core-rust.git#branch
+```
+
+If you want prebuilds for a branch that has a core pr, you might find an npm tar.gz package for that branch at <https://download.delta.chat/node/preview/>.
+The github ci also posts a link to it in the checks for each pr.
+
 ### Use build-from-source in deltachat-desktop
 
 If you want to use the manually built node bindings in the desktop client (for

node/scripts/generate-constants.js

@@ -1,73 +1,65 @@
 #!/usr/bin/env node
-
 const fs = require('fs')
 const path = require('path')
-const split = require('split2')
 
 const data = []
-const regex = /^#define\s+(\w+)\s+(\w+)/i
-const header = path.resolve(
-  __dirname,
-  '../../deltachat-ffi/deltachat.h'
-)
+const header = path.resolve(__dirname, '../../deltachat-ffi/deltachat.h')
 
 console.log('Generating constants...')
 
-fs.createReadStream(header)
-  .pipe(split())
-  .on('data', (line) => {
-    const match = regex.exec(line)
-    if (match) {
-      const key = match[1]
-      const value = parseInt(match[2])
-      if (isNaN(value)) return
+const header_data = fs.readFileSync(header, 'UTF-8')
+const regex = /^#define\s+(\w+)\s+(\w+)/gm
+while (null != (match = regex.exec(header_data))) {
+  const key = match[1]
+  const value = parseInt(match[2])
+  if (!isNaN(value)) {
+    data.push({ key, value })
+  }
+}
 
-      data.push({ key, value })
-    }
+delete header_data
+
+const constants = data
+  .filter(
+    ({ key }) => key.toUpperCase()[0] === key[0] // check if define name is uppercase
+  )
+  .sort((lhs, rhs) => {
+    if (lhs.key < rhs.key) return -1
+    else if (lhs.key > rhs.key) return 1
+    return 0
   })
-  .on('end', () => {
-    const constants = data
-      .filter(
-        ({ key }) => key.toUpperCase()[0] === key[0] // check if define name is uppercase
-      )
-      .sort((lhs, rhs) => {
-        if (lhs.key < rhs.key) return -1
-        else if (lhs.key > rhs.key) return 1
-        return 0
-      })
-      .map((row) => {
-        return `  ${row.key}: ${row.value}`
-      })
-      .join(',\n')
+  .map((row) => {
+    return `  ${row.key}: ${row.value}`
+  })
+  .join(',\n')
 
-    const events = data
-      .sort((lhs, rhs) => {
-        if (lhs.value < rhs.value) return -1
-        else if (lhs.value > rhs.value) return 1
-        return 0
-      })
-      .filter((i) => {
-        return i.key.startsWith('DC_EVENT_')
-      })
-      .map((i) => {
-        return `  ${i.value}: '${i.key}'`
-      })
-      .join(',\n')
+const events = data
+  .sort((lhs, rhs) => {
+    if (lhs.value < rhs.value) return -1
+    else if (lhs.value > rhs.value) return 1
+    return 0
+  })
+  .filter((i) => {
+    return i.key.startsWith('DC_EVENT_')
+  })
+  .map((i) => {
+    return `  ${i.value}: '${i.key}'`
+  })
+  .join(',\n')
 
-    // backwards compat
-    fs.writeFileSync(
-      path.resolve(__dirname, '../constants.js'),
-      `// Generated!\n\nmodule.exports = {\n${constants}\n}\n`
-    )
-    // backwards compat
-    fs.writeFileSync(
-      path.resolve(__dirname, '../events.js'),
-      `/* eslint-disable quotes */\n// Generated!\n\nmodule.exports = {\n${events}\n}\n`
-    )
+// backwards compat
+fs.writeFileSync(
+  path.resolve(__dirname, '../constants.js'),
+  `// Generated!\n\nmodule.exports = {\n${constants}\n}\n`
+)
+// backwards compat
+fs.writeFileSync(
+  path.resolve(__dirname, '../events.js'),
+  `/* eslint-disable quotes */\n// Generated!\n\nmodule.exports = {\n${events}\n}\n`
+)
 
-    fs.writeFileSync(
-      path.resolve(__dirname, '../lib/constants.ts'),
-      `// Generated!\n\nexport enum C {\n${constants.replace(/:/g, ' =')},\n}\n
+fs.writeFileSync(
+  path.resolve(__dirname, '../lib/constants.ts'),
+  `// Generated!\n\nexport enum C {\n${constants.replace(/:/g, ' =')},\n}\n
 // Generated!\n\nexport const EventId2EventName: { [key: number]: string } = {\n${events},\n}\n`
-    )
-  })
+)

node/src/module.c

@@ -348,7 +348,7 @@ NAPI_METHOD(dcn_start_event_handler) {
     callback,
     0,
     async_resource_name,
-    1,
+    1000, // max_queue_size
     1,
     NULL,
     NULL,
@@ -371,6 +371,11 @@ NAPI_METHOD(dcn_context_unref) {
 
   TRACE("Unrefing dc_context");
   dcn_context->gc = 1;
+  if (dcn_context->event_handler_thread != 0) {
+    dc_stop_io(dcn_context->dc_context);
+    uv_thread_join(&dcn_context->event_handler_thread);
+    dcn_context->event_handler_thread = 0;
+  }
   dc_context_unref(dcn_context->dc_context);
   dcn_context->dc_context = NULL;
 
@@ -2922,6 +2927,11 @@ NAPI_METHOD(dcn_accounts_unref) {
 
   TRACE("Unrefing dc_accounts");
   dcn_accounts->gc = 1;
+  if (dcn_accounts->event_handler_thread != 0) {
+    dc_accounts_stop_io(dcn_accounts->dc_accounts);
+    uv_thread_join(&dcn_accounts->event_handler_thread);
+    dcn_accounts->event_handler_thread = 0;
+  }
   dc_accounts_unref(dcn_accounts->dc_accounts);
   dcn_accounts->dc_accounts = NULL;
 
@@ -3093,8 +3103,8 @@ static void accounts_event_handler_thread_func(void* arg)
     }
     event = dc_accounts_get_next_event(dc_accounts_event_emitter);
     if (event == NULL) {
-      //TRACE("received NULL event, skipping");
-      continue;
+      TRACE("no more events");
+      break;
     }
 
     if (!dcn_accounts->threadsafe_event_handler) {
@@ -3216,7 +3226,7 @@ NAPI_METHOD(dcn_accounts_start_event_handler) {
     callback,
     0,
     async_resource_name,
-    1,
+    1000, // max_queue_size
     1,
     NULL,
     NULL,

node/test/test.js

@@ -128,7 +128,7 @@ describe('Basic offline Tests', function () {
 
     await expect(
       context.configure({ addr: 'delta1@delta.localhost' })
-    ).to.eventually.be.rejectedWith('Please enter a password.')
+    ).to.eventually.be.rejectedWith('Missing (IMAP) password.')
     await expect(context.configure({ mailPw: 'delta1' })).to.eventually.be
       .rejected
 
@@ -601,8 +601,8 @@ describe('Offline Tests with unconfigured account', function () {
     strictEqual(chatList.getCount(), 1, 'only one archived')
   })
 
-  it('Remove qoute from (draft) message', function () {
-    context.addDeviceMessage('test_qoute', 'test')
+  it('Remove quote from (draft) message', function () {
+    context.addDeviceMessage('test_quote', 'test')
     const msgId = context.getChatMessages(10, 0, 0)[0]
     const msg = context.messageNew()
 

package.json

@@ -19,7 +19,6 @@
     "prebuildify": "^3.0.0",
     "prebuildify-ci": "^1.0.4",
     "prettier": "^2.0.5",
-    "split2": "^4.1.0",
     "typedoc": "^0.17.0",
     "typescript": "^3.9.10"
   },
@@ -42,7 +41,7 @@
     "build": "npm run build:core && npm run build:bindings",
     "build:bindings": "npm run build:bindings:c && npm run build:bindings:ts",
     "build:bindings:c": "npm run build:bindings:c:c && npm run build:bindings:c:postinstall",
-    "build:bindings:c:c": "cd node && npx node-gyp rebuild",
+    "build:bindings:c:c": "cd node && node-gyp rebuild",
     "build:bindings:c:postinstall": "node node/scripts/postinstall.js",
     "build:bindings:ts": "cd node && tsc",
     "build:core": "npm run build:core:rust && npm run build:core:constants",
@@ -58,8 +57,8 @@
     "prebuildify": "cd node && prebuildify -t 16.13.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "test": "npm run test:lint && npm run test:mocha",
     "test:lint": "npm run lint",
-    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec"
+    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.84.0"
-}
+  "version": "1.87.0"
+}

python/doc/_templates/globaltoc.html

@@ -11,8 +11,7 @@
 <ul>
   <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
+  <li><a href="https://web.libera.chat/#deltachat">#deltachat</a></li>
 </ul>
 
-<b>#deltachat [freenode]</b>
-
 </div>

python/examples/echo_and_quit.py

@@ -1,4 +1,3 @@
-
 # content of echo_and_quit.py
 
 from deltachat import account_hookimpl, run_cmdline

python/examples/group_tracking.py

@@ -1,4 +1,3 @@
-
 # content of group_tracking.py
 
 from deltachat import account_hookimpl, run_cmdline
@@ -33,15 +32,21 @@ class GroupTrackingPlugin:
 
     @account_hookimpl
     def ac_member_added(self, chat, contact, actor, message):
-        print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+        print(
+            "ac_member_added {} to chat {} from {}".format(
+                contact.addr, chat.id, actor or message.get_sender_contact().addr
+            )
+        )
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
     def ac_member_removed(self, chat, contact, actor, message):
-        print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+        print(
+            "ac_member_removed {} from chat {} by {}".format(
+                contact.addr, chat.id, actor or message.get_sender_contact().addr
+            )
+        )
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -1,20 +1,20 @@
-
-import pytest
-import py
 import echo_and_quit
 import group_tracking
+import py
+import pytest
+
 from deltachat.events import FFIEventLogger
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def datadir():
     """The py.path.local object of the test-data/ directory."""
     for path in reversed(py.path.local(__file__).parts()):
-        datadir = path.join('test-data')
+        datadir = path.join("test-data")
         if datadir.isdir():
             return datadir
     else:
-        pytest.skip('test-data directory not found')
+        pytest.skip("test-data directory not found")
 
 
 def test_echo_quit_plugin(acfactory, lp):
@@ -22,7 +22,7 @@ def test_echo_quit_plugin(acfactory, lp):
     botproc = acfactory.run_bot_process(echo_and_quit)
 
     lp.sec("creating a temp account to contact the bot")
-    ac1, = acfactory.get_online_accounts(1)
+    (ac1,) = acfactory.get_online_accounts(1)
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -44,9 +44,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     ac1, ac2 = acfactory.get_online_accounts(2)
 
-    botproc.fnmatch_lines("""
+    botproc.fnmatch_lines(
+        """
         *ac_configure_completed*
-    """)
+    """
+    )
     ac1.add_account_plugin(FFIEventLogger(ac1))
     ac2.add_account_plugin(FFIEventLogger(ac2))
 
@@ -56,9 +58,11 @@ def test_group_tracking_plugin(acfactory, lp):
     ch.add_contact(bot_contact)
     ch.send_text("hello")
 
-    botproc.fnmatch_lines("""
+    botproc.fnmatch_lines(
+        """
         *ac_chat_modified*bot test group*
-    """)
+    """
+    )
 
     lp.sec("adding third member {}".format(ac2.get_config("addr")))
     contact3 = ac1.create_contact(ac2.get_config("addr"))
@@ -68,12 +72,20 @@ def test_group_tracking_plugin(acfactory, lp):
     assert "hello" in reply.text
 
     lp.sec("now looking at what the bot received")
-    botproc.fnmatch_lines("""
+    botproc.fnmatch_lines(
+        """
         *ac_member_added {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+    """.format(
+            contact3.addr, ac1.get_config("addr")
+        )
+    )
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
-    botproc.fnmatch_lines("""
+    botproc.fnmatch_lines(
+        """
         *ac_member_removed {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+    """.format(
+            contact3.addr, ac1.get_config("addr")
+        )
+    )

python/install_python_bindings.py

@@ -18,15 +18,13 @@ if __name__ == "__main__":
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
 
-    if target == 'release':
+    if target == "release":
         os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
 
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll", shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
-        subprocess.check_call([
-            sys.executable, "-m", "pip", "install", "-e", "."
-        ])
+        subprocess.check_call([sys.executable, "-m", "pip", "install", "-e", "."])

python/pyproject.toml

@@ -6,3 +6,9 @@ build-backend = "setuptools.build_meta"
 root = ".."
 tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
 git_describe_command = "git describe --dirty --tags --long --match py-*.*"
+
+[tool.black]
+line-length = 120
+
+[tool.isort]
+profile = "black"

python/setup.py

@@ -1,36 +1,37 @@
-import setuptools
 import os
 import re
 
+import setuptools
+
 
 def main():
     with open("README.rst") as f:
         long_description = f.read()
     setuptools.setup(
-        name='deltachat',
-        description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
+        name="deltachat",
+        description="Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat",
         long_description=long_description,
-        author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
+        author="holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors",
+        install_requires=["cffi>=1.0.0", "pluggy", "imap-tools", "requests"],
         setup_requires=[
-            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
-            'pkgconfig',
+            "setuptools_scm",  # required for compatibility with `python3 setup.py sdist`
+            "pkgconfig",
         ],
-        packages=setuptools.find_packages('src'),
-        package_dir={'': 'src'},
-        cffi_modules=['src/deltachat/_build.py:ffibuilder'],
-        entry_points = {
-            'pytest11': [
-                'deltachat.testplugin = deltachat.testplugin',
+        packages=setuptools.find_packages("src"),
+        package_dir={"": "src"},
+        cffi_modules=["src/deltachat/_build.py:ffibuilder"],
+        entry_points={
+            "pytest11": [
+                "deltachat.testplugin = deltachat.testplugin",
             ],
         },
         classifiers=[
-            'Development Status :: 4 - Beta',
-            'Intended Audience :: Developers',
-            'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
-            'Programming Language :: Python :: 3',
-            'Topic :: Communications :: Email',
-            'Topic :: Software Development :: Libraries',
+            "Development Status :: 4 - Beta",
+            "Intended Audience :: Developers",
+            "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+            "Programming Language :: Python :: 3",
+            "Topic :: Communications :: Email",
+            "Topic :: Software Development :: Libraries",
         ],
     )
 

python/src/deltachat/__init__.py

@@ -1,15 +1,15 @@
 import sys
 
-from . import capi, const, hookspec # noqa
-from .capi import ffi  # noqa
-from .account import Account, get_core_info  # noqa
-from .message import Message  # noqa
-from .contact import Contact  # noqa
-from .chat import Chat        # noqa
-from .hookspec import account_hookimpl, global_hookimpl  # noqa
-from . import events
+from pkg_resources import DistributionNotFound, get_distribution
+
+from . import capi, const, events, hookspec  # noqa
+from .account import Account, get_core_info  # noqa
+from .capi import ffi  # noqa
+from .chat import Chat  # noqa
+from .contact import Contact  # noqa
+from .hookspec import account_hookimpl, global_hookimpl  # noqa
+from .message import Message  # noqa
 
-from pkg_resources import get_distribution, DistributionNotFound
 try:
     __version__ = get_distribution(__name__).version
 except DistributionNotFound:
@@ -26,7 +26,7 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
 
 
 def register_global_plugin(plugin):
-    """ Register a global plugin which implements one or more
+    """Register a global plugin which implements one or more
     of the :class:`deltachat.hookspec.Global` hooks.
     """
     gm = hookspec.Global._get_plugin_manager()
@@ -43,9 +43,10 @@ register_global_plugin(events)
 
 
 def run_cmdline(argv=None, account_plugins=None):
-    """ Run a simple default command line app, registering the specified
-    account plugins. """
+    """Run a simple default command line app, registering the specified
+    account plugins."""
     import argparse
+
     if argv is None:
         argv = sys.argv
 
@@ -69,9 +70,9 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.add_account_plugin(plugin)
 
     if not ac.is_configured():
-        assert args.email and args.password, (
-            "you must specify --email and --password once to configure this database/account"
-        )
+        assert (
+            args.email and args.password
+        ), "you must specify --email and --password once to configure this database/account"
         ac.set_config("addr", args.email)
         ac.set_config("mail_pw", args.password)
         ac.set_config("mvbox_move", "0")

python/src/deltachat/_build.py

@@ -20,30 +20,33 @@ def local_build_flags(projdir, target):
     :param target: The rust build target, `debug` or `release`.
     """
     flags = {}
-    if platform.system() == 'Darwin':
-        flags['libraries'] = ['resolv', 'dl']
-        flags['extra_link_args'] = [
-                '-framework', 'CoreFoundation',
-                '-framework', 'CoreServices',
-                '-framework', 'Security',
+    if platform.system() == "Darwin":
+        flags["libraries"] = ["resolv", "dl"]
+        flags["extra_link_args"] = [
+            "-framework",
+            "CoreFoundation",
+            "-framework",
+            "CoreServices",
+            "-framework",
+            "Security",
         ]
-    elif platform.system() == 'Linux':
-        flags['libraries'] = ['rt', 'dl', 'm']
-        flags['extra_link_args'] = []
+    elif platform.system() == "Linux":
+        flags["libraries"] = ["rt", "dl", "m"]
+        flags["extra_link_args"] = []
     else:
         raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
     target_dir = os.environ.get("CARGO_TARGET_DIR")
     if target_dir is None:
-        target_dir = os.path.join(projdir, 'target')
-    flags['extra_objects'] = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags['extra_objects'][0]), flags['extra_objects']
-    flags['include_dirs'] = [os.path.join(projdir, 'deltachat-ffi')]
+        target_dir = os.path.join(projdir, "target")
+    flags["extra_objects"] = [os.path.join(target_dir, target, "libdeltachat.a")]
+    assert os.path.exists(flags["extra_objects"][0]), flags["extra_objects"]
+    flags["include_dirs"] = [os.path.join(projdir, "deltachat-ffi")]
     return flags
 
 
 def system_build_flags():
     """Construct build flags for building against an installed libdeltachat."""
-    return pkgconfig.parse('deltachat')
+    return pkgconfig.parse("deltachat")
 
 
 def extract_functions(flags):
@@ -61,11 +64,13 @@ def extract_functions(flags):
         src_name = os.path.join(tmpdir, "include.h")
         dst_name = os.path.join(tmpdir, "expanded.h")
         with open(src_name, "w") as src_fp:
-            src_fp.write('#include <deltachat.h>')
-        cc.preprocess(source=src_name,
-                      output_file=dst_name,
-                      include_dirs=flags['include_dirs'],
-                      macros=[('PY_CFFI', '1')])
+            src_fp.write("#include <deltachat.h>")
+        cc.preprocess(
+            source=src_name,
+            output_file=dst_name,
+            include_dirs=flags["include_dirs"],
+            macros=[("PY_CFFI", "1")],
+        )
         with open(dst_name, "r") as dst_fp:
             return dst_fp.read()
     finally:
@@ -87,7 +92,9 @@ def find_header(flags):
         obj_name = os.path.join(tmpdir, "where.o")
         dst_name = os.path.join(tmpdir, "where")
         with open(src_name, "w") as src_fp:
-            src_fp.write(textwrap.dedent("""
+            src_fp.write(
+                textwrap.dedent(
+                    """
                 #include <stdio.h>
                 #include <deltachat.h>
 
@@ -95,18 +102,20 @@ def find_header(flags):
                     printf("%s", _dc_header_file_location());
                     return 0;
                 }
-            """))
+            """
+                )
+            )
         cwd = os.getcwd()
         try:
             os.chdir(tmpdir)
-            cc.compile(sources=["where.c"],
-                       include_dirs=flags['include_dirs'],
-                       macros=[("PY_CFFI_INC", "1")])
+            cc.compile(
+                sources=["where.c"],
+                include_dirs=flags["include_dirs"],
+                macros=[("PY_CFFI_INC", "1")],
+            )
         finally:
             os.chdir(cwd)
-        cc.link_executable(objects=[obj_name],
-                           output_progname="where",
-                           output_dir=tmpdir)
+        cc.link_executable(objects=[obj_name], output_progname="where", output_dir=tmpdir)
         return subprocess.check_output(dst_name)
     finally:
         shutil.rmtree(tmpdir)
@@ -123,7 +132,8 @@ def extract_defines(flags):
     cdef() method.
     """
     header = find_header(flags)
-    defines_re = re.compile(r"""
+    defines_re = re.compile(
+        r"""
         \#define\s+   # The start of a define.
         (             # Begin capturing group which captures the define name.
             (?:       # A nested group which is not captured, this allows us
@@ -151,26 +161,28 @@ def extract_defines(flags):
         )             # Close the capturing group, this contains
                       # the entire name e.g. DC_MSG_TEXT.
         \s+\S+        # Ensure there is whitespace followed by a value.
-    """, re.VERBOSE)
+    """,
+        re.VERBOSE,
+    )
     defines = []
     with open(header) as fp:
         for line in fp:
             match = defines_re.match(line)
             if match:
                 defines.append(match.group(1))
-    return '\n'.join('#define {} ...'.format(d) for d in defines)
+    return "\n".join("#define {} ...".format(d) for d in defines)
 
 
 def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
+    projdir = os.environ.get("DCC_RS_DEV")
     if projdir:
-        target = os.environ.get('DCC_RS_TARGET', 'release')
+        target = os.environ.get("DCC_RS_TARGET", "release")
         flags = local_build_flags(projdir, target)
     else:
         flags = system_build_flags()
     builder = cffi.FFI()
     builder.set_source(
-        'deltachat.capi',
+        "deltachat.capi",
         """
             #include <deltachat.h>
             int dc_event_has_string_data(int e)
@@ -180,11 +192,13 @@ def ffibuilder():
         """,
         **flags,
     )
-    builder.cdef("""
+    builder.cdef(
+        """
         typedef int... time_t;
         void free(void *ptr);
         extern int dc_event_has_string_data(int);
-    """)
+    """
+    )
     function_defs = extract_functions(flags)
     defines = extract_defines(flags)
     builder.cdef(function_defs)
@@ -192,8 +206,9 @@ def ffibuilder():
     return builder
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     import os.path
-    pkgdir = os.path.join(os.path.dirname(__file__), '..')
+
+    pkgdir = os.path.join(os.path.dirname(__file__), "..")
     builder = ffibuilder()
     builder.compile(tmpdir=pkgdir, verbose=True)

python/src/deltachat/account.py

@@ -1,37 +1,46 @@
 """ Account class implementation. """
 
 from __future__ import print_function
+
+import os
+from array import array
 from contextlib import contextmanager
 from email.utils import parseaddr
 from threading import Event
-import os
-from array import array
-from . import const
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from . import const, hookspec
 from .capi import ffi, lib
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array, DCLot
 from .chat import Chat
-from .message import Message
 from .contact import Contact
-from .tracker import ImexTracker, ConfigureTracker
-from . import hookspec
+from .cutil import (
+    DCLot,
+    as_dc_charpointer,
+    from_dc_charpointer,
+    from_optional_dc_charpointer,
+    iter_array,
+)
 from .events import EventThread
-from typing import Union, Any, Dict, Optional, List, Generator
+from .message import Message
+from .tracker import ConfigureTracker, ImexTracker
 
 
 class MissingCredentials(ValueError):
-    """ Account is missing `addr` and `mail_pw` config values. """
+    """Account is missing `addr` and `mail_pw` config values."""
 
 
 def get_core_info():
-    """ get some system info. """
+    """get some system info."""
     from tempfile import NamedTemporaryFile
 
     with NamedTemporaryFile() as path:
         path.close()
-        return get_dc_info_as_dict(ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL),
-            lib.dc_context_unref,
-        ))
+        return get_dc_info_as_dict(
+            ffi.gc(
+                lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL),
+                lib.dc_context_unref,
+            )
+        )
 
 
 def get_dc_info_as_dict(dc_context):
@@ -46,18 +55,22 @@ def get_dc_info_as_dict(dc_context):
 
 
 class Account(object):
-    """ Each account is tied to a sqlite database file which is fully managed
+    """Each account is tied to a sqlite database file which is fully managed
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
+
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True) -> None:
-        """ initialize account object.
+    def __init__(self, db_path, os_name=None, logging=True, closed=False) -> None:
+        """initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param os_name: this will be put to the X-Mailer header in outgoing messages
+        :param os_name: [Deprecated]
+        :param logging: enable logging for this account
+        :param closed: set to True to avoid automatically opening the account
+                       after creation.
         """
         # initialize per-account plugin system
         self._pm = hookspec.PerAccount._make_plugin_manager()
@@ -70,7 +83,7 @@ class Account(object):
             db_path = db_path.encode("utf8")
 
         self._dc_context = ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(os_name), db_path, ffi.NULL),
+            lib.dc_context_new_closed(db_path) if closed else lib.dc_context_new(ffi.NULL, db_path, ffi.NULL),
             lib.dc_context_unref,
         )
         if self._dc_context == ffi.NULL:
@@ -82,12 +95,23 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
+    def open(self, passphrase: Optional[str] = None) -> bool:
+        """Open the account's database with the given passphrase.
+        This can only be used on a closed account. If the account is new, this
+        operation sets the database passphrase. For existing databases the passphrase
+        should be the one used to encrypt the database the first time.
+
+        :returns: True if the database is opened with this passphrase, False if the
+                  passphrase is incorrect or an error occurred.
+        """
+        return bool(lib.dc_context_open(self._dc_context, as_dc_charpointer(passphrase)))
+
     def disable_logging(self) -> None:
-        """ disable logging. """
+        """disable logging."""
         self._logging = False
 
     def enable_logging(self) -> None:
-        """ re-enable logging. """
+        """re-enable logging."""
         self._logging = True
 
     def __repr__(self):
@@ -102,11 +126,10 @@ class Account(object):
 
     def _check_config_key(self, name: str) -> None:
         if name not in self._configkeys:
-            raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
-                           name, self._configkeys))
+            raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(name, self._configkeys))
 
     def get_info(self) -> Dict[str, str]:
-        """ return dictionary of built config parameters. """
+        """return dictionary of built config parameters."""
         return get_dc_info_as_dict(self._dc_context)
 
     def dump_account_info(self, logfile):
@@ -126,7 +149,7 @@ class Account(object):
         log("")
 
     def set_stock_translation(self, id: int, string: str) -> None:
-        """ set stock translation string.
+        """set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
@@ -138,7 +161,7 @@ class Account(object):
             raise ValueError("could not set translation string")
 
     def set_config(self, name: str, value: Optional[str]) -> None:
-        """ set configuration values.
+        """set configuration values.
 
         :param name: config key name (unicode)
         :param value: value to set (unicode)
@@ -157,7 +180,7 @@ class Account(object):
         lib.dc_set_config(self._dc_context, namebytes, valuebytes)
 
     def get_config(self, name: str) -> str:
-        """ return unicode string value.
+        """return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
         :returns: unicode value
@@ -175,15 +198,17 @@ class Account(object):
 
         In other words, you don't need this.
         """
-        res = lib.dc_preconfigure_keypair(self._dc_context,
-                                          as_dc_charpointer(addr),
-                                          as_dc_charpointer(public),
-                                          as_dc_charpointer(secret))
+        res = lib.dc_preconfigure_keypair(
+            self._dc_context,
+            as_dc_charpointer(addr),
+            as_dc_charpointer(public),
+            as_dc_charpointer(secret),
+        )
         if res == 0:
             raise Exception("Failed to set key")
 
     def update_config(self, kwargs: Dict[str, Any]) -> None:
-        """ update config values.
+        """update config values.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
@@ -193,18 +218,18 @@ class Account(object):
             self.set_config(key, value)
 
     def is_configured(self) -> bool:
-        """ determine if the account is configured already; an initial connection
+        """determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
         :returns: True if account is configured.
         """
-        return True if lib.dc_is_configured(self._dc_context) else False
+        return bool(lib.dc_is_configured(self._dc_context))
 
     def is_open(self) -> bool:
         """Determine if account is open
 
         :returns True if account is open."""
-        return True if lib.dc_context_is_open(self._dc_context) else False
+        return bool(lib.dc_context_is_open(self._dc_context))
 
     def set_avatar(self, img_path: Optional[str]) -> None:
         """Set self avatar.
@@ -219,18 +244,17 @@ class Account(object):
             self.set_config("selfavatar", img_path)
 
     def check_is_configured(self) -> None:
-        """ Raise ValueError if this account is not configured. """
+        """Raise ValueError if this account is not configured."""
         if not self.is_configured():
             raise ValueError("need to configure first")
 
     def get_latest_backupfile(self, backupdir) -> Optional[str]:
-        """ return the latest backup file in a given directory.
-        """
+        """return the latest backup file in a given directory."""
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
         return from_optional_dc_charpointer(res)
 
     def get_blobdir(self) -> str:
-        """ return the directory for files.
+        """return the directory for files.
 
         All sent files are copied to this directory if necessary.
         Place files there directly to avoid copying.
@@ -238,7 +262,7 @@ class Account(object):
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
     def get_self_contact(self) -> Contact:
-        """ return this account's identity as a :class:`deltachat.contact.Contact`.
+        """return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
@@ -280,14 +304,14 @@ class Account(object):
         elif isinstance(obj, str):
             displayname, addr = parseaddr(obj)
         else:
-            raise TypeError("don't know how to create chat for %r" % (obj, ))
+            raise TypeError("don't know how to create chat for %r" % (obj,))
 
         if name is None and displayname:
             name = displayname
         return (name, addr)
 
     def delete_contact(self, contact: Contact) -> bool:
-        """ delete a Contact.
+        """delete a Contact.
 
         :param contact: contact object obtained
         :returns: True if deletion succeeded (contact was deleted)
@@ -298,7 +322,7 @@ class Account(object):
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
     def get_contact_by_addr(self, email: str) -> Optional[Contact]:
-        """ get a contact for the email address or None if it's blocked or doesn't exist. """
+        """get a contact for the email address or None if it's blocked or doesn't exist."""
         _, addr = parseaddr(email)
         addr = as_dc_charpointer(addr)
         contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
@@ -307,21 +331,18 @@ class Account(object):
         return None
 
     def get_contact_by_id(self, contact_id: int) -> Contact:
-        """ return Contact instance or raise an exception.
+        """return Contact instance or raise an exception.
         :param contact_id: integer id of this contact.
         :returns: :class:`deltachat.contact.Contact` instance.
         """
         return Contact(self, contact_id)
 
     def get_blocked_contacts(self) -> List[Contact]:
-        """ return a list of all blocked contacts.
+        """return a list of all blocked contacts.
 
         :returns: list of :class:`deltachat.contact.Contact` objects.
         """
-        dc_array = ffi.gc(
-            lib.dc_get_blocked_contacts(self._dc_context),
-            lib.dc_array_unref
-        )
+        dc_array = ffi.gc(lib.dc_get_blocked_contacts(self._dc_context), lib.dc_array_unref)
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
     def get_contacts(
@@ -344,22 +365,16 @@ class Account(object):
             flags |= const.DC_GCL_VERIFIED_ONLY
         if with_self:
             flags |= const.DC_GCL_ADD_SELF
-        dc_array = ffi.gc(
-            lib.dc_get_contacts(self._dc_context, flags, query),
-            lib.dc_array_unref
-        )
+        dc_array = ffi.gc(lib.dc_get_contacts(self._dc_context, flags, query), lib.dc_array_unref)
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
     def get_fresh_messages(self) -> Generator[Message, None, None]:
-        """ yield all fresh messages from all chats. """
-        dc_array = ffi.gc(
-            lib.dc_get_fresh_msgs(self._dc_context),
-            lib.dc_array_unref
-        )
+        """yield all fresh messages from all chats."""
+        dc_array = ffi.gc(lib.dc_get_fresh_msgs(self._dc_context), lib.dc_array_unref)
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
     def create_chat(self, obj) -> Chat:
-        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        """Create a 1:1 chat with Account, Contact or e-mail address."""
         return self.create_contact(obj).create_chat()
 
     def create_group_chat(
@@ -385,14 +400,11 @@ class Account(object):
         return chat
 
     def get_chats(self) -> List[Chat]:
-        """ return list of chats.
+        """return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
         """
-        dc_chatlist = ffi.gc(
-            lib.dc_get_chatlist(self._dc_context, 0, ffi.NULL, 0),
-            lib.dc_chatlist_unref
-        )
+        dc_chatlist = ffi.gc(lib.dc_get_chatlist(self._dc_context, 0, ffi.NULL, 0), lib.dc_chatlist_unref)
 
         assert dc_chatlist != ffi.NULL
         chatlist = []
@@ -405,14 +417,14 @@ class Account(object):
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
     def get_message_by_id(self, msg_id: int) -> Message:
-        """ return Message instance.
+        """return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
     def get_chat_by_id(self, chat_id: int) -> Chat:
-        """ return Chat instance.
+        """return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
         :raises: ValueError if chat does not exist.
@@ -424,7 +436,7 @@ class Account(object):
         return Chat(self, chat_id)
 
     def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
-        """ mark the given set of messages as seen.
+        """mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
@@ -438,7 +450,7 @@ class Account(object):
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
     def forward_messages(self, messages: List[Message], chat: Chat) -> None:
-        """ Forward list of messages to a chat.
+        """Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
         :param chat: :class:`deltachat.chat.Chat` object.
@@ -448,7 +460,7 @@ class Account(object):
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
     def delete_messages(self, messages: List[Message]) -> None:
-        """ delete messages (local and remote).
+        """delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
         :returns: None
@@ -457,31 +469,34 @@ class Account(object):
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
     def export_self_keys(self, path):
-        """ export public and private keys to the specified directory.
+        """export public and private keys to the specified directory.
 
         Note that the account does not have to be started.
         """
         return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
 
-    def export_all(self, path):
-        """return new file containing a backup of all database state
-        (chats, contacts, keys, media, ...). The file is created in the
-        the `path` directory.
+    def export_all(self, path: str, passphrase: Optional[str] = None) -> str:
+        """Export a backup of all database state (chats, contacts, keys, media, ...).
+
+        :param path: the directory where the backup will be stored.
+        :param passphrase: the backup will be encrypted with the passphrase, if it is
+                           None or empty string, the backup is not encrypted.
+        :returns: path to the created backup.
 
         Note that the account has to be stopped; call stop_io() if necessary.
         """
-        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
+        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP, passphrase)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
-    def _export(self, path, imex_cmd):
+    def _export(self, path: str, imex_cmd: int, passphrase: Optional[str] = None) -> list:
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            self.imex(path, imex_cmd, passphrase)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
-        """ Import private keys found in the `path` directory.
+        """Import private keys found in the `path` directory.
         The last imported key is made the default keys unless its name
         contains the string legacy. Public keys are not imported.
 
@@ -489,21 +504,23 @@ class Account(object):
         """
         self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
 
-    def import_all(self, path):
-        """import delta chat state from the specified backup `path` (a file).
-
+    def import_all(self, path: str, passphrase: Optional[str] = None) -> None:
+        """Import Delta Chat state from the specified backup file.
         The account must be in unconfigured state for import to attempted.
+
+        :param path: path to the backup file.
+        :param passphrase: if not None or empty, the backup will be opened with the passphrase.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
+        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP, passphrase=passphrase)
 
-    def _import(self, path, imex_cmd):
+    def _import(self, path: str, imex_cmd: int, passphrase: Optional[str] = None) -> None:
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            self.imex(path, imex_cmd, passphrase)
             imex_tracker.wait_finish()
 
-    def imex(self, path, imex_cmd):
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+    def imex(self, path: str, imex_cmd: int, passphrase: Optional[str] = None) -> None:
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), as_dc_charpointer(passphrase))
 
     def initiate_key_transfer(self) -> str:
         """return setup code after a Autocrypt setup message
@@ -517,7 +534,7 @@ class Account(object):
         return from_dc_charpointer(res)
 
     def get_setup_contact_qr(self) -> str:
-        """ get/create Setup-Contact QR Code as ascii-string.
+        """get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
         in a second channel (typically used by mobiles with QRcode-show + scan UX)
@@ -527,18 +544,15 @@ class Account(object):
         return from_dc_charpointer(res)
 
     def check_qr(self, qr):
-        """ check qr code and return :class:`ScannedQRCode` instance representing the result"""
-        res = ffi.gc(
-            lib.dc_check_qr(self._dc_context, as_dc_charpointer(qr)),
-            lib.dc_lot_unref
-        )
+        """check qr code and return :class:`ScannedQRCode` instance representing the result"""
+        res = ffi.gc(lib.dc_check_qr(self._dc_context, as_dc_charpointer(qr)), lib.dc_lot_unref)
         lot = DCLot(res)
         if lot.state() == const.DC_QR_ERROR:
             raise ValueError("invalid or unknown QR code: {}".format(lot.text1()))
         return ScannedQRCode(lot)
 
     def qr_setup_contact(self, qr):
-        """ setup contact and return a Chat after contact is established.
+        """setup contact and return a Chat after contact is established.
 
         Note that this function may block for a long time as messages are exchanged
         with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
@@ -552,7 +566,7 @@ class Account(object):
         return Chat(self, chat_id)
 
     def qr_join_chat(self, qr):
-        """ join a chat group through a QR code.
+        """join a chat group through a QR code.
 
         Note that this function may block for a long time as messages are exchanged
         with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
@@ -566,9 +580,7 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(
-        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
-    ) -> None:
+    def set_location(self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0) -> None:
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -587,7 +599,7 @@ class Account(object):
     #
 
     def add_account_plugin(self, plugin, name=None):
-        """ add an account plugin which implements one or more of
+        """add an account plugin which implements one or more of
         the :class:`deltachat.hookspec.PerAccount` hooks.
         """
         if name and self._pm.has_plugin(name=name):
@@ -597,18 +609,18 @@ class Account(object):
         return plugin
 
     def remove_account_plugin(self, plugin, name=None):
-        """ remove an account plugin. """
+        """remove an account plugin."""
         self._pm.unregister(plugin, name=name)
 
     @contextmanager
     def temp_plugin(self, plugin):
-        """ run a with-block with the given plugin temporarily registered. """
+        """run a with-block with the given plugin temporarily registered."""
         self._pm.register(plugin)
         yield plugin
         self._pm.unregister(plugin)
 
     def stop_ongoing(self):
-        """ Stop ongoing securejoin, configuration or other core jobs. """
+        """Stop ongoing securejoin, configuration or other core jobs."""
         lib.dc_stop_ongoing_process(self._dc_context)
 
     def get_connectivity(self):
@@ -621,7 +633,7 @@ class Account(object):
         return lib.dc_all_work_done(self._dc_context)
 
     def start_io(self):
-        """ start this account's IO scheduling (Rust-core async scheduler)
+        """start this account's IO scheduling (Rust-core async scheduler)
 
         If this account is not configured an Exception is raised.
         You need to call account.configure() and account.wait_configure_finish()
@@ -665,7 +677,7 @@ class Account(object):
         lib.dc_maybe_network(self._dc_context)
 
     def configure(self, reconfigure: bool = False) -> ConfigureTracker:
-        """ Start configuration process and return a Configtracker instance
+        """Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
         """
@@ -678,11 +690,11 @@ class Account(object):
         return configtracker
 
     def wait_shutdown(self) -> None:
-        """ wait until shutdown of this account has completed. """
+        """wait until shutdown of this account has completed."""
         self._shutdown_event.wait()
 
     def stop_io(self) -> None:
-        """ stop core IO scheduler if it is running. """
+        """stop core IO scheduler if it is running."""
         self.log("stop_ongoing")
         self.stop_ongoing()
 
@@ -690,7 +702,7 @@ class Account(object):
         lib.dc_stop_io(self._dc_context)
 
     def shutdown(self) -> None:
-        """ shutdown and destroy account (stop callback thread, close and remove
+        """shutdown and destroy account (stop callback thread, close and remove
         underlying dc_context)."""
         if self._dc_context is None:
             return

python/src/deltachat/chat.py

@@ -1,32 +1,40 @@
 """ Chat and Location related API. """
 
-import mimetypes
 import calendar
 import json
-from datetime import datetime, timezone
+import mimetypes
 import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array
-from .capi import lib, ffi
-from . import const
-from .message import Message
+from datetime import datetime, timezone
 from typing import Optional
 
+from . import const
+from .capi import ffi, lib
+from .cutil import (
+    as_dc_charpointer,
+    from_dc_charpointer,
+    from_optional_dc_charpointer,
+    iter_array,
+)
+from .message import Message
+
 
 class Chat(object):
-    """ Chat object which manages members and through which you can send and retrieve messages.
+    """Chat object which manages members and through which you can send and retrieve messages.
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
     def __init__(self, account, id) -> None:
         from .account import Account
+
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
     def __eq__(self, other) -> bool:
-        return self.id == getattr(other, "id", None) and \
-               self.account._dc_context == other.account._dc_context
+        if other is None:
+            return False
+        return self.id == getattr(other, "id", None) and self.account._dc_context == other.account._dc_context
 
     def __ne__(self, other) -> bool:
         return not (self == other)
@@ -36,10 +44,7 @@ class Chat(object):
 
     @property
     def _dc_chat(self):
-        return ffi.gc(
-            lib.dc_get_chat(self.account._dc_context, self.id),
-            lib.dc_chat_unref
-        )
+        return ffi.gc(lib.dc_get_chat(self.account._dc_context, self.id), lib.dc_chat_unref)
 
     def delete(self) -> None:
         """Delete this chat and all its messages.
@@ -62,28 +67,66 @@ class Chat(object):
     # ------  chat status/metadata API ------------------------------
 
     def is_group(self) -> bool:
-        """ return true if this chat is a group chat.
+        """Return True if this chat is a group chat.
 
-        :returns: True if chat is a group-chat, false otherwise
+        :returns: True if chat is a group-chat, False otherwise
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
+    def is_single(self) -> bool:
+        """Return True if this chat is a single/direct chat, False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_SINGLE
+
+    def is_mailinglist(self) -> bool:
+        """Return True if this chat is a mailing list, False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_MAILINGLIST
+
+    def is_broadcast(self) -> bool:
+        """Return True if this chat is a broadcast list, False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_BROADCAST
+
+    def is_multiuser(self) -> bool:
+        """Return True if this chat is a multi-user chat (group, mailing list or broadcast), False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_MAILINGLIST,
+            const.DC_CHAT_TYPE_BROADCAST,
+        )
+
+    def is_self_talk(self) -> bool:
+        """Return True if this chat is the self-chat (a.k.a. "Saved Messages"), False otherwise"""
+        return bool(lib.dc_chat_is_self_talk(self._dc_chat))
+
+    def is_device_talk(self) -> bool:
+        """Returns True if this chat is the "Device Messages" chat, False otherwise"""
+        return bool(lib.dc_chat_is_device_talk(self._dc_chat))
+
     def is_muted(self) -> bool:
-        """ return true if this chat is muted.
+        """return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
         """
-        return lib.dc_chat_is_muted(self._dc_chat)
+        return bool(lib.dc_chat_is_muted(self._dc_chat))
 
-    def is_contact_request(self):
-        """ return True if this chat is a contact request chat.
+    def is_pinned(self) -> bool:
+        """Return True if this chat is pinned, False otherwise"""
+        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_PINNED
+
+    def is_archived(self) -> bool:
+        """Return True if this chat is archived, False otherwise.
+        :returns: True if archived, False otherwise
+        """
+        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
+
+    def is_contact_request(self) -> bool:
+        """return True if this chat is a contact request chat.
 
         :returns: True if chat is a contact request chat, False otherwise.
         """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
+        return bool(lib.dc_chat_is_contact_request(self._dc_chat))
 
-    def is_promoted(self):
-        """ return True if this chat is promoted, i.e.
+    def is_promoted(self) -> bool:
+        """return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
         have been sent messages.
 
@@ -97,24 +140,24 @@ class Chat(object):
 
         :returns: True if the chat is writable, False otherwise
         """
-        return lib.dc_chat_can_send(self._dc_chat)
+        return bool(lib.dc_chat_can_send(self._dc_chat))
 
     def is_protected(self) -> bool:
-        """ return True if this chat is a protected chat.
+        """return True if this chat is a protected chat.
 
         :returns: True if chat is protected, False otherwise.
         """
-        return lib.dc_chat_is_protected(self._dc_chat)
+        return bool(lib.dc_chat_is_protected(self._dc_chat))
 
     def get_name(self) -> Optional[str]:
-        """ return name of this chat.
+        """return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
     def set_name(self, name: str) -> bool:
-        """ set name of this chat.
+        """set name of this chat.
 
         :param name: as a unicode string.
         :returns: True on success, False otherwise
@@ -122,8 +165,20 @@ class Chat(object):
         name = as_dc_charpointer(name)
         return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
 
+    def get_color(self):
+        """return the color of the chat.
+        :returns: color as 0x00rrggbb
+        """
+        return lib.dc_chat_get_color(self._dc_chat)
+
+    def get_summary(self):
+        """return dictionary with summary information."""
+        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
+        s = from_dc_charpointer(dc_res)
+        return json.loads(s)
+
     def mute(self, duration: Optional[int] = None) -> None:
-        """ mutes the chat
+        """mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
         :returns: None
@@ -137,7 +192,7 @@ class Chat(object):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
     def unmute(self) -> None:
-        """ unmutes the chat
+        """unmutes the chat
 
         :returns: None
         """
@@ -145,8 +200,26 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
+    def pin(self) -> None:
+        """Pin the chat."""
+        lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_PINNED)
+
+    def unpin(self) -> None:
+        """Unpin the chat."""
+        if self.is_pinned():
+            lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_NORMAL)
+
+    def archive(self) -> None:
+        """Archive the chat."""
+        lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_ARCHIVED)
+
+    def unarchive(self) -> None:
+        """Unarchive the chat."""
+        if self.is_archived():
+            lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_NORMAL)
+
     def get_mute_duration(self) -> int:
-        """ Returns the number of seconds until the mute of this chat is lifted.
+        """Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
@@ -154,14 +227,14 @@ class Chat(object):
         return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
 
     def get_ephemeral_timer(self) -> int:
-        """ get ephemeral timer.
+        """get ephemeral timer.
 
         :returns: ephemeral timer value in seconds
         """
         return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
 
     def set_ephemeral_timer(self, timer: int) -> bool:
-        """ set ephemeral timer.
+        """set ephemeral timer.
 
         :param: timer value in seconds
 
@@ -170,7 +243,7 @@ class Chat(object):
         return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
 
     def get_type(self) -> int:
-        """ (deprecated) return type of this chat.
+        """(deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
@@ -184,7 +257,7 @@ class Chat(object):
         return from_dc_charpointer(res)
 
     def get_join_qr(self) -> Optional[str]:
-        """ get/create Join-Group QR Code as ascii-string.
+        """get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
         in a second channel (typically used by mobiles with QRcode-show + scan UX)
@@ -220,7 +293,7 @@ class Chat(object):
         return msg
 
     def send_text(self, text):
-        """ send a text message and return the resulting Message instance.
+        """send a text message and return the resulting Message instance.
 
         :param msg: unicode text
         :raises ValueError: if message can not be send/chat does not exist.
@@ -233,7 +306,7 @@ class Chat(object):
         return Message.from_db(self.account, msg_id)
 
     def send_file(self, path, mime_type="application/octet-stream"):
-        """ send a file and return the resulting Message instance.
+        """send a file and return the resulting Message instance.
 
         :param path: path to the file.
         :param mime_type: the mime-type of this file, defaults to application/octet-stream.
@@ -248,7 +321,7 @@ class Chat(object):
         return Message.from_db(self.account, sent_id)
 
     def send_image(self, path):
-        """ send an image message and return the resulting Message instance.
+        """send an image message and return the resulting Message instance.
 
         :param path: path to an image file.
         :raises ValueError: if message can not be send/chat does not exist.
@@ -263,7 +336,7 @@ class Chat(object):
         return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
-        """ prepare a message for sending.
+        """prepare a message for sending.
 
         :param msg: the message to be prepared.
         :returns: a :class:`deltachat.message.Message` instance.
@@ -278,7 +351,7 @@ class Chat(object):
         return msg
 
     def prepare_message_file(self, path, mime_type=None, view_type="file"):
-        """ prepare a message for sending and return the resulting Message instance.
+        """prepare a message for sending and return the resulting Message instance.
 
         To actually send the message, call :meth:`send_prepared`.
         The file must be inside the blob directory.
@@ -294,7 +367,7 @@ class Chat(object):
         return self.prepare_message(msg)
 
     def send_prepared(self, message):
-        """ send a previously prepared message.
+        """send a previously prepared message.
 
         :param message: a :class:`Message` instance previously returned by
                         :meth:`prepare_file`.
@@ -314,7 +387,7 @@ class Chat(object):
         msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
 
     def set_draft(self, message):
-        """ set message as draft.
+        """set message as draft.
 
         :param message: a :class:`Message` instance
         :returns: None
@@ -325,7 +398,7 @@ class Chat(object):
             lib.dc_set_draft(self.account._dc_context, self.id, message._dc_msg)
 
     def get_draft(self):
-        """ get draft message for this chat.
+        """get draft message for this chat.
 
         :param message: a :class:`Message` instance
         :returns: Message object or None (if no draft available)
@@ -337,40 +410,34 @@ class Chat(object):
         return Message(self.account, dc_msg)
 
     def get_messages(self):
-        """ return list of messages in this chat.
+        """return list of messages in this chat.
 
         :returns: list of :class:`deltachat.message.Message` objects for this chat.
         """
         dc_array = ffi.gc(
             lib.dc_get_chat_msgs(self.account._dc_context, self.id, 0, 0),
-            lib.dc_array_unref
+            lib.dc_array_unref,
         )
         return list(iter_array(dc_array, lambda x: Message.from_db(self.account, x)))
 
     def count_fresh_messages(self):
-        """ return number of fresh messages in this chat.
+        """return number of fresh messages in this chat.
 
         :returns: number of fresh messages
         """
         return lib.dc_get_fresh_msg_cnt(self.account._dc_context, self.id)
 
     def mark_noticed(self):
-        """ mark all messages in this chat as noticed.
+        """mark all messages in this chat as noticed.
 
         Noticed messages are no longer fresh.
         """
         return lib.dc_marknoticed_chat(self.account._dc_context, self.id)
 
-    def get_summary(self):
-        """ return dictionary with summary information. """
-        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
-        s = from_dc_charpointer(dc_res)
-        return json.loads(s)
-
     # ------  group management API ------------------------------
 
     def add_contact(self, obj):
-        """ add a contact to this chat.
+        """add a contact to this chat.
 
         :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be added
@@ -383,7 +450,7 @@ class Chat(object):
         return contact
 
     def remove_contact(self, obj):
-        """ remove a contact from this chat.
+        """remove a contact from this chat.
 
         :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be removed
@@ -395,23 +462,22 @@ class Chat(object):
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
-        """ get all contacts for this chat.
+        """get all contacts for this chat.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
         """
         from .contact import Contact
+
         dc_array = ffi.gc(
             lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
-        )
-        return list(iter_array(
-            dc_array, lambda id: Contact(self.account, id))
+            lib.dc_array_unref,
         )
+        return list(iter_array(dc_array, lambda id: Contact(self.account, id)))
 
     def num_contacts(self):
-        """ return number of contacts in this chat. """
+        """return number of contacts in this chat."""
         dc_array = ffi.gc(
             lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
+            lib.dc_array_unref,
         )
         return lib.dc_array_get_cnt(dc_array)
 
@@ -458,12 +524,6 @@ class Chat(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_color(self):
-        """return the color of the chat.
-        :returns: color as 0x00rrggbb
-        """
-        return lib.dc_chat_get_color(self._dc_chat)
-
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
@@ -472,12 +532,6 @@ class Chat(object):
         """
         return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)
 
-    def is_archived(self):
-        """return True if this chat is archived.
-        :returns: True if archived.
-        """
-        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
-
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.
 
@@ -513,10 +567,7 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.fromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i),
-                    timezone.utc
-                ),
+                timestamp=datetime.fromtimestamp(lib.dc_array_get_timestamp(dc_array, i), timezone.utc),
                 marker=from_optional_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )
             for i in range(lib.dc_array_get_cnt(dc_array))

python/src/deltachat/contact.py

@@ -10,17 +10,21 @@ from .cutil import from_dc_charpointer, from_optional_dc_charpointer
 
 
 class Contact(object):
-    """ Delta-Chat Contact.
+    """Delta-Chat Contact.
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
+
     def __init__(self, account, id):
         from .account import Account
+
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
+        if other is None:
+            return False
         return self.account._dc_context == other.account._dc_context and self.id == other.id
 
     def __ne__(self, other):
@@ -31,19 +35,16 @@ class Contact(object):
 
     @property
     def _dc_contact(self):
-        return ffi.gc(
-            lib.dc_get_contact(self.account._dc_context, self.id),
-            lib.dc_contact_unref
-        )
+        return ffi.gc(lib.dc_get_contact(self.account._dc_context, self.id), lib.dc_contact_unref)
 
     @props.with_doc
     def addr(self) -> str:
-        """ normalized e-mail address for this account. """
+        """normalized e-mail address for this account."""
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
     def name(self) -> str:
-        """ display name for this contact. """
+        """display name for this contact."""
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
     # deprecated alias
@@ -52,28 +53,26 @@ class Contact(object):
     @props.with_doc
     def last_seen(self) -> date:
         """Last seen timestamp."""
-        return datetime.fromtimestamp(
-            lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc
-        )
+        return datetime.fromtimestamp(lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc)
 
     def is_blocked(self):
-        """ Return True if the contact is blocked. """
+        """Return True if the contact is blocked."""
         return lib.dc_contact_is_blocked(self._dc_contact)
 
     def set_blocked(self, block=True):
-        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
+        """[Deprecated, use block/unblock methods] Block or unblock a contact."""
         return lib.dc_block_contact(self.account._dc_context, self.id, block)
 
     def block(self):
-        """ Block this contact. Message will not be seen/retrieved from this contact. """
+        """Block this contact. Message will not be seen/retrieved from this contact."""
         return lib.dc_block_contact(self.account._dc_context, self.id, True)
 
     def unblock(self):
-        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
+        """Unblock this contact. Messages from this contact will be retrieved (again)."""
         return lib.dc_block_contact(self.account._dc_context, self.id, False)
 
     def is_verified(self):
-        """ Return True if the contact is verified. """
+        """Return True if the contact is verified."""
         return lib.dc_contact_is_verified(self._dc_contact)
 
     def get_profile_image(self) -> Optional[str]:
@@ -93,7 +92,7 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
 
     def create_chat(self):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+        """create or get an existing 1:1 chat object for the specified contact or contact id.
 
         :param contact: chat_id (int) or contact object.
         :returns: a :class:`deltachat.chat.Chat` object.

python/src/deltachat/cutil.py

@@ -1,9 +1,9 @@
-from .capi import lib
-from .capi import ffi
 from datetime import datetime, timezone
-from typing import Optional, TypeVar, Generator, Callable
+from typing import Callable, Generator, Optional, TypeVar
 
-T = TypeVar('T')
+from .capi import ffi, lib
+
+T = TypeVar("T")
 
 
 def as_dc_charpointer(obj):

python/src/deltachat/direct_imap.py

@@ -3,18 +3,27 @@ Internal Python-level IMAP handling used by the testplugin
 and for cleaning up inbox/mvbox for each test function run.
 """
 
-import io
-import ssl
-import pathlib
-from contextlib import contextmanager
-from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
 import imaplib
-from deltachat import const, Account
+import io
+import pathlib
+import ssl
+from contextlib import contextmanager
 from typing import List
 
+from imap_tools import (
+    AND,
+    Header,
+    MailBox,
+    MailBoxTls,
+    MailMessage,
+    MailMessageFlags,
+    errors,
+)
 
-FLAGS = b'FLAGS'
-FETCH = b'FETCH'
+from deltachat import Account, const
+
+FLAGS = b"FLAGS"
+FETCH = b"FETCH"
 ALL = "1:*"
 
 
@@ -69,8 +78,8 @@ class DirectImap:
         return self.conn.folder.set(foldername)
 
     def select_config_folder(self, config_name: str):
-        """ Return info about selected folder if it is
-        configured, otherwise None. """
+        """Return info about selected folder if it is
+        configured, otherwise None."""
         if "_" not in config_name:
             config_name = "configured_{}_folder".format(config_name)
         foldername = self.account.get_config(config_name)
@@ -78,17 +87,17 @@ class DirectImap:
             return self.select_folder(foldername)
 
     def list_folders(self) -> List[str]:
-        """ return list of all existing folder names"""
+        """return list of all existing folder names"""
         assert not self._idling
         return [folder.name for folder in self.conn.folder.list()]
 
     def delete(self, uid_list: str, expunge=True):
-        """ delete a range of messages (imap-syntax).
+        """delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
+        self.conn.client.uid("STORE", uid_list, "+FLAGS", r"(\Deleted)")
         if expunge:
             self.conn.expunge()
 
@@ -141,7 +150,13 @@ class DirectImap:
                 fn = path.joinpath(str(msg.uid))
                 fn.write_bytes(body)
                 log("Message", msg.uid, fn)
-                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
+                log(
+                    "Message",
+                    msg.uid,
+                    msg.flags,
+                    "Message-Id:",
+                    msg.obj.get("Message-Id"),
+                )
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
@@ -150,7 +165,7 @@ class DirectImap:
 
     @contextmanager
     def idle(self):
-        """ return Idle ContextManager. """
+        """return Idle ContextManager."""
         idle_manager = IdleManager(self)
         try:
             yield idle_manager
@@ -163,11 +178,11 @@ class DirectImap:
         """
         if msg.startswith("\n"):
             msg = msg[1:]
-        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(bytes(msg, encoding='ascii'), folder)
+        msg = "\n".join([s.lstrip() for s in msg.splitlines()])
+        self.conn.append(bytes(msg, encoding="ascii"), folder)
 
     def get_uid_by_message_id(self, message_id) -> str:
-        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
+        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header("MESSAGE-ID", message_id)))]
         if len(msgs) == 0:
             raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
         return msgs[0]
@@ -183,7 +198,7 @@ class IdleManager:
         self.direct_imap.conn.idle.start()
 
     def check(self, timeout=None) -> List[bytes]:
-        """ (blocking) wait for next idle message from server. """
+        """(blocking) wait for next idle message from server."""
         self.log("imap-direct: calling idle_check")
         res = self.direct_imap.conn.idle.poll(timeout=timeout)
         self.log("imap-direct: idle_check returned {!r}".format(res))
@@ -192,20 +207,19 @@ class IdleManager:
     def wait_for_new_message(self, timeout=None) -> bytes:
         while 1:
             for item in self.check(timeout=timeout):
-                if b'EXISTS' in item or b'RECENT' in item:
+                if b"EXISTS" in item or b"RECENT" in item:
                     return item
 
     def wait_for_seen(self, timeout=None) -> int:
-        """ Return first message with SEEN flag from a running idle-stream.
-        """
+        """Return first message with SEEN flag from a running idle-stream."""
         while 1:
             for item in self.check(timeout=timeout):
                 if FETCH in item:
                     self.log(str(item))
-                    if FLAGS in item and rb'\Seen' in item:
-                        return int(item.split(b' ')[1])
+                    if FLAGS in item and rb"\Seen" in item:
+                        return int(item.split(b" ")[1])
 
     def done(self):
-        """ send idle-done to server if we are currently in idle mode. """
+        """send idle-done to server if we are currently in idle mode."""
         res = self.direct_imap.conn.idle.stop()
         return res

python/src/deltachat/events.py

@@ -1,18 +1,19 @@
-import threading
-import sys
-import traceback
-import time
 import io
-import re
 import os
-from queue import Queue, Empty
+import re
+import sys
+import threading
+import time
+import traceback
+from contextlib import contextmanager
+from queue import Empty, Queue
 
 import deltachat
-from .hookspec import account_hookimpl
-from contextlib import contextmanager
+
 from .capi import ffi, lib
-from .message import map_system_message
 from .cutil import from_optional_dc_charpointer
+from .hookspec import account_hookimpl
+from .message import map_system_message
 
 
 class FFIEvent:
@@ -26,9 +27,10 @@ class FFIEvent:
 
 
 class FFIEventLogger:
-    """ If you register an instance of this logger with an Account
+    """If you register an instance of this logger with an Account
     you'll get all ffi-events printed.
     """
+
     # to prevent garbled logging
     _loglock = threading.RLock()
 
@@ -56,9 +58,9 @@ class FFIEventLogger:
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
 
         if os.name == "posix":
-            WARN = '\033[93m'
-            ERROR = '\033[91m'
-            ENDC = '\033[0m'
+            WARN = "\033[93m"
+            ERROR = "\033[91m"
+            ENDC = "\033[0m"
             if message.startswith("DC_EVENT_WARNING"):
                 s = WARN + s + ENDC
             if message.startswith("DC_EVENT_ERROR"):
@@ -171,12 +173,12 @@ class FFIEventTracker:
         self.get_info_contains("INBOX: Idle entering")
 
     def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
+        """wait for and return next incoming message."""
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")
         return self.account.get_message_by_id(ev.data2)
 
     def wait_next_messages_changed(self):
-        """ wait for and return next message-changed message or None
+        """wait for and return next message-changed message or None
         if the event contains no msgid"""
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
@@ -191,10 +193,11 @@ class FFIEventTracker:
 
 
 class EventThread(threading.Thread):
-    """ Event Thread for an account.
+    """Event Thread for an account.
 
     With each Account init this callback thread is started.
     """
+
     def __init__(self, account) -> None:
         self.account = account
         super(EventThread, self).__init__(name="events")
@@ -219,7 +222,7 @@ class EventThread(threading.Thread):
         self.join(timeout=timeout)
 
     def run(self) -> None:
-        """ get and run events until shutdown. """
+        """get and run events until shutdown."""
         with self.log_execution("EVENT THREAD"):
             self._inner_run()
 
@@ -259,8 +262,7 @@ class EventThread(threading.Thread):
         except Exception as ex:
             logfile = io.StringIO()
             traceback.print_exception(*sys.exc_info(), file=logfile)
-            self.account.log("{}\nException {}\nTraceback:\n{}"
-                             .format(info, ex, logfile.getvalue()))
+            self.account.log("{}\nException {}\nTraceback:\n{}".format(info, ex, logfile.getvalue()))
 
     def _map_ffi_event(self, ffi_event: FFIEvent):
         name = ffi_event.name
@@ -269,7 +271,8 @@ class EventThread(threading.Thread):
             data1 = ffi_event.data1
             if data1 == 0 or data1 == 1000:
                 success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
+                comment = ffi_event.data2
+                yield "ac_configure_completed", dict(success=success, comment=comment)
         elif name == "DC_EVENT_INCOMING_MSG":
             msg = account.get_message_by_id(ffi_event.data2)
             yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
@@ -282,7 +285,10 @@ class EventThread(threading.Thread):
                         yield res
                     yield "ac_outgoing_message", dict(message=msg)
                 elif msg.is_in_fresh():
-                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+                    yield map_system_message(msg) or (
+                        "ac_incoming_message",
+                        dict(message=msg),
+                    )
         elif name == "DC_EVENT_MSG_DELIVERED":
             msg = account.get_message_by_id(ffi_event.data2)
             yield "ac_message_delivered", dict(message=msg)

python/src/deltachat/hookspec.py

@@ -2,7 +2,6 @@
 
 import pluggy
 
-
 account_spec_name = "deltachat-account"
 account_hookspec = pluggy.HookspecMarker(account_spec_name)
 account_hookimpl = pluggy.HookimplMarker(account_spec_name)
@@ -13,12 +12,13 @@ global_hookimpl = pluggy.HookimplMarker(global_spec_name)
 
 
 class PerAccount:
-    """ per-Account-instance hook specifications.
+    """per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
     Hooks are generally not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
+
     @classmethod
     def _make_plugin_manager(cls):
         pm = pluggy.PluginManager(account_spec_name)
@@ -27,7 +27,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_process_ffi_event(self, ffi_event):
-        """ process a CFFI low level events for a given account.
+        """process a CFFI low level events for a given account.
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
@@ -35,37 +35,37 @@ class PerAccount:
 
     @account_hookspec
     def ac_log_line(self, message):
-        """ log a message related to the account. """
+        """log a message related to the account."""
 
     @account_hookspec
-    def ac_configure_completed(self, success):
-        """ Called after a configure process completed. """
+    def ac_configure_completed(self, success, comment):
+        """Called after a configure process completed."""
 
     @account_hookspec
     def ac_incoming_message(self, message):
-        """ Called on any incoming message (both existing chats and contact requests). """
+        """Called on any incoming message (both existing chats and contact requests)."""
 
     @account_hookspec
     def ac_outgoing_message(self, message):
-        """ Called on each outgoing message (both system and "normal")."""
+        """Called on each outgoing message (both system and "normal")."""
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP.
+        """Called when an outgoing message has been delivered to SMTP.
 
         :param message: Message that was just delivered.
         """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title.
+        """Chat was created or modified regarding membership, avatar, title.
 
         :param chat: Chat which was modified.
         """
 
     @account_hookspec
     def ac_member_added(self, chat, contact, actor, message):
-        """ Called for each contact added to an accepted chat.
+        """Called for each contact added to an accepted chat.
 
         :param chat: Chat where contact was added.
         :param contact: Contact that was added.
@@ -75,7 +75,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_member_removed(self, chat, contact, actor, message):
-        """ Called for each contact removed from a chat.
+        """Called for each contact removed from a chat.
 
         :param chat: Chat where contact was removed.
         :param contact: Contact that was removed.
@@ -85,10 +85,11 @@ class PerAccount:
 
 
 class Global:
-    """ global hook specifications using a per-process singleton
+    """global hook specifications using a per-process singleton
     plugin manager instance.
 
     """
+
     _plugin_manager = None
 
     @classmethod
@@ -100,11 +101,11 @@ class Global:
 
     @global_hookspec
     def dc_account_init(self, account):
-        """ called when `Account::__init__()` function starts executing. """
+        """called when `Account::__init__()` function starts executing."""
 
     @global_hookspec
     def dc_account_extra_configure(self, account):
-        """ Called when account configuration successfully finished.
+        """Called when account configuration successfully finished.
 
         This hook can be used to perform extra work before
         ac_configure_completed is called.
@@ -112,4 +113,4 @@ class Global:
 
     @global_hookspec
     def dc_account_after_shutdown(self, account):
-        """ Called after the account has been shutdown. """
+        """Called after the account has been shutdown."""

python/src/deltachat/message.py

@@ -1,21 +1,23 @@
 """ The Message object. """
 
+import json
 import os
 import re
-from . import props
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer, as_dc_charpointer
-from .capi import lib, ffi
-from . import const
 from datetime import datetime, timezone
-from typing import Optional
+from typing import Optional, Union
+
+from . import const, props
+from .capi import ffi, lib
+from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer
 
 
 class Message(object):
-    """ Message object.
+    """Message object.
 
     You obtain instances of it through :class:`deltachat.account.Account` or
     :class:`deltachat.chat.Chat`.
     """
+
     def __init__(self, account, dc_msg):
         self.account = account
         assert isinstance(self.account._dc_context, ffi.CData)
@@ -25,42 +27,48 @@ class Message(object):
         msg_id = self.id
         assert msg_id is not None and msg_id >= 0, repr(msg_id)
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
+        if other is None:
+            return False
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
         c = self.get_sender_contact()
         typ = "outgoing" if self.is_outgoing() else "incoming"
         return "<Message {} sys={} {} id={} sender={}/{} chat={}/{}>".format(
-            typ, self.is_system_message(), repr(self.text[:10]),
-            self.id, c.id, c.addr, self.chat.id, self.chat.get_name())
+            typ,
+            self.is_system_message(),
+            repr(self.text[:10]),
+            self.id,
+            c.id,
+            c.addr,
+            self.chat.id,
+            self.chat.get_name(),
+        )
 
     @classmethod
     def from_db(cls, account, id):
         assert id > 0
-        return cls(account, ffi.gc(
-                lib.dc_get_msg(account._dc_context, id),
-                lib.dc_msg_unref
-        ))
+        return cls(account, ffi.gc(lib.dc_get_msg(account._dc_context, id), lib.dc_msg_unref))
 
     @classmethod
     def new_empty(cls, account, view_type):
-        """ create a non-persistent message.
+        """create a non-persistent message.
 
-        :param: view_type is the message type code or one of the strings:
-           "text", "audio", "video", "file", "sticker"
+        :param view_type: the message type code or one of the strings:
+           "text", "audio", "video", "file", "sticker", "videochat", "webxdc"
         """
         if isinstance(view_type, int):
             view_type_code = view_type
         else:
             view_type_code = get_viewtype_code_from_name(view_type)
-        return Message(account, ffi.gc(
-            lib.dc_msg_new(account._dc_context, view_type_code),
-            lib.dc_msg_unref
-        ))
+        return Message(
+            account,
+            ffi.gc(lib.dc_msg_new(account._dc_context, view_type_code), lib.dc_msg_unref),
+        )
 
     def create_chat(self):
-        """ create or get an existing chat (group) object for this message.
+        """create or get an existing chat (group) object for this message.
 
         If the message is a contact request
         the sender will become an accepted contact.
@@ -72,23 +80,22 @@ class Message(object):
 
     @props.with_doc
     def id(self):
-        """id of this message. """
+        """id of this message."""
         return lib.dc_msg_get_id(self._dc_msg)
 
     @props.with_doc
     def text(self) -> str:
-        """unicode text of this messages (might be empty if not a text message). """
+        """unicode text of this messages (might be empty if not a text message)."""
         return from_dc_charpointer(lib.dc_msg_get_text(self._dc_msg))
 
     def set_text(self, text):
-        """set text of this message. """
+        """set text of this message."""
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
     @props.with_doc
     def html(self) -> str:
-        """html text of this messages (might be empty if not an html message). """
-        return from_optional_dc_charpointer(
-            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
+        """html text of this messages (might be empty if not an html message)."""
+        return from_optional_dc_charpointer(lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
 
     def has_html(self):
         """return True if this message has an html part, False otherwise."""
@@ -103,11 +110,11 @@ class Message(object):
 
     @props.with_doc
     def filename(self):
-        """filename if there was an attachment, otherwise empty string. """
+        """filename if there was an attachment, otherwise empty string."""
         return from_dc_charpointer(lib.dc_msg_get_file(self._dc_msg))
 
     def set_file(self, path, mime_type=None):
-        """set file for this message from path and mime_type. """
+        """set file for this message from path and mime_type."""
         mtype = ffi.NULL if mime_type is None else as_dc_charpointer(mime_type)
         if not os.path.exists(path):
             raise ValueError("path does not exist: {!r}".format(path))
@@ -115,7 +122,7 @@ class Message(object):
 
     @props.with_doc
     def basename(self) -> str:
-        """basename of the attachment if it exists, otherwise empty string. """
+        """basename of the attachment if it exists, otherwise empty string."""
         # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 
@@ -124,44 +131,74 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
+    def get_status_updates(self, serial: int = 0) -> list:
+        """Get the status updates of this webxdc message.
+
+        The status updates may be sent by yourself or by other members.
+        If this message doesn't have a webxdc instance, an empty list is returned.
+
+        :param serial: The last known serial. Pass 0 if there are no known serials to receive all updates.
+        """
+        return json.loads(
+            from_dc_charpointer(lib.dc_get_webxdc_status_updates(self.account._dc_context, self.id, serial))
+        )
+
+    def send_status_update(self, json_data: Union[str, dict], description: str) -> bool:
+        """Send an status update for the webxdc instance of this message.
+
+        If the webxdc instance is a draft, the update is not sent immediately.
+        Instead, the updates are collected and sent out in a batch when the instance is actually sent.
+
+        :param json_data: program-readable data, the actual payload.
+        :param description: The user-visible description of JSON data
+        :returns: True on success, False otherwise
+        """
+        if isinstance(json_data, dict):
+            json_data = json.dumps(json_data, default=str)
+        return bool(
+            lib.dc_send_webxdc_status_update(
+                self.account._dc_context, self.id, as_dc_charpointer(json_data), as_dc_charpointer(description)
+            )
+        )
+
     def is_system_message(self):
-        """ return True if this message is a system/info message. """
+        """return True if this message is a system/info message."""
         return bool(lib.dc_msg_is_info(self._dc_msg))
 
     def is_setup_message(self):
-        """ return True if this message is a setup message. """
+        """return True if this message is a setup message."""
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
     def get_setupcodebegin(self) -> str:
-        """ return the first characters of a setup code in a setup message. """
+        """return the first characters of a setup code in a setup message."""
         return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
 
     def is_encrypted(self):
-        """ return True if this message was encrypted. """
+        """return True if this message was encrypted."""
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
     def is_bot(self):
-        """ return True if this message is submitted automatically. """
+        """return True if this message is submitted automatically."""
         return bool(lib.dc_msg_is_bot(self._dc_msg))
 
     def is_forwarded(self):
-        """ return True if this message was forwarded. """
+        """return True if this message was forwarded."""
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
 
     def get_message_info(self) -> str:
-        """ Return informational text for a single message.
+        """Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
         """
         return from_dc_charpointer(lib.dc_get_msg_info(self.account._dc_context, self.id))
 
+    def get_summarytext(self, width: int) -> str:
+        """Get a message summary as a single line of text. Typically used for notifications."""
+        return from_dc_charpointer(lib.dc_msg_get_summarytext(self._dc_msg, width))
+
     def continue_key_transfer(self, setup_code):
-        """ extract key and use it as primary key for this account. """
-        res = lib.dc_continue_key_transfer(
-                self.account._dc_context,
-                self.id,
-                as_dc_charpointer(setup_code)
-        )
+        """extract key and use it as primary key for this account."""
+        res = lib.dc_continue_key_transfer(self.account._dc_context, self.id, as_dc_charpointer(setup_code))
         if res == 0:
             raise ValueError("could not decrypt")
 
@@ -230,7 +267,7 @@ class Message(object):
         lib.dc_msg_force_plaintext(self._dc_msg)
 
     def get_mime_headers(self):
-        """ return mime-header object for an incoming message.
+        """return mime-header object for an incoming message.
 
         This only returns a non-None object if ``save_mime_headers``
         config option was set and the message is incoming.
@@ -238,6 +275,7 @@ class Message(object):
         :returns: email-mime message object (with headers only, no body).
         """
         import email.parser
+
         mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id)
         if mime_headers:
             s = ffi.string(ffi.gc(mime_headers, lib.dc_str_unref))
@@ -257,6 +295,7 @@ class Message(object):
         :returns: :class:`deltachat.chat.Chat` object
         """
         from .chat import Chat
+
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
@@ -266,13 +305,11 @@ class Message(object):
 
         Usually used to impersonate someone else.
         """
-        return from_optional_dc_charpointer(
-            lib.dc_msg_get_override_sender_name(self._dc_msg))
+        return from_optional_dc_charpointer(lib.dc_msg_get_override_sender_name(self._dc_msg))
 
     def set_override_sender_name(self, name):
-        """set different sender name for a message. """
-        lib.dc_msg_set_override_sender_name(
-            self._dc_msg, as_dc_charpointer(name))
+        """set different sender name for a message."""
+        lib.dc_msg_set_override_sender_name(self._dc_msg, as_dc_charpointer(name))
 
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
@@ -287,6 +324,7 @@ class Message(object):
         :returns: :class:`deltachat.chat.Contact` instance
         """
         from .contact import Contact
+
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
         return Contact(self.account, contact_id)
 
@@ -299,14 +337,11 @@ class Message(object):
             dc_msg = self._dc_msg
         else:
             # load message from db to get a fresh/current state
-            dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
-                lib.dc_msg_unref
-            )
+            dc_msg = ffi.gc(lib.dc_get_msg(self.account._dc_context, self.id), lib.dc_msg_unref)
         return lib.dc_msg_get_state(dc_msg)
 
     def is_in_fresh(self):
-        """ return True if Message is incoming fresh message (un-noticed).
+        """return True if Message is incoming fresh message (un-noticed).
 
         Fresh messages are not noticed nor seen and are typically
         shown in notifications.
@@ -330,25 +365,25 @@ class Message(object):
         return self._msgstate == const.DC_STATE_IN_SEEN
 
     def is_outgoing(self):
-        """Return True if Message is outgoing. """
+        """Return True if Message is outgoing."""
         return self._msgstate in (
-            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
-            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
-            const.DC_STATE_OUT_DELIVERED)
+            const.DC_STATE_OUT_PREPARING,
+            const.DC_STATE_OUT_PENDING,
+            const.DC_STATE_OUT_FAILED,
+            const.DC_STATE_OUT_MDN_RCVD,
+            const.DC_STATE_OUT_DELIVERED,
+        )
 
     def is_out_preparing(self):
-        """Return True if Message is outgoing, but its file is being prepared.
-        """
+        """Return True if Message is outgoing, but its file is being prepared."""
         return self._msgstate == const.DC_STATE_OUT_PREPARING
 
     def is_out_pending(self):
-        """Return True if Message is outgoing, but is pending (no single checkmark).
-        """
+        """Return True if Message is outgoing, but is pending (no single checkmark)."""
         return self._msgstate == const.DC_STATE_OUT_PENDING
 
     def is_out_failed(self):
-        """Return True if Message is unrecoverably failed.
-        """
+        """Return True if Message is unrecoverably failed."""
         return self._msgstate == const.DC_STATE_OUT_FAILED
 
     def is_out_delivered(self):
@@ -375,48 +410,58 @@ class Message(object):
         return lib.dc_msg_get_viewtype(self._dc_msg)
 
     def is_text(self):
-        """ return True if it's a text message. """
+        """return True if it's a text message."""
         return self._view_type == const.DC_MSG_TEXT
 
     def is_image(self):
-        """ return True if it's an image message. """
+        """return True if it's an image message."""
         return self._view_type == const.DC_MSG_IMAGE
 
     def is_gif(self):
-        """ return True if it's a gif message. """
+        """return True if it's a gif message."""
         return self._view_type == const.DC_MSG_GIF
 
     def is_sticker(self):
-        """ return True if it's a sticker message. """
+        """return True if it's a sticker message."""
         return self._view_type == const.DC_MSG_STICKER
 
     def is_audio(self):
-        """ return True if it's an audio message. """
+        """return True if it's an audio message."""
         return self._view_type == const.DC_MSG_AUDIO
 
     def is_video(self):
-        """ return True if it's a video message. """
+        """return True if it's a video message."""
         return self._view_type == const.DC_MSG_VIDEO
 
+    def is_videochat_invitation(self):
+        """return True if it's a videochat invitation message."""
+        return self._view_type == const.DC_MSG_VIDEOCHAT_INVITATION
+
+    def is_webxdc(self):
+        """return True if it's a Webxdc message."""
+        return self._view_type == const.DC_MSG_WEBXDC
+
     def is_file(self):
-        """ return True if it's a file message. """
+        """return True if it's a file message."""
         return self._view_type == const.DC_MSG_FILE
 
     def mark_seen(self):
-        """ mark this message as seen. """
+        """mark this message as seen."""
         self.account.mark_seen_messages([self.id])
 
 
 # some code for handling DC_MSG_* view types
 
 _view_type_mapping = {
-    'text': const.DC_MSG_TEXT,
-    'image': const.DC_MSG_IMAGE,
-    'gif': const.DC_MSG_GIF,
-    'audio': const.DC_MSG_AUDIO,
-    'video': const.DC_MSG_VIDEO,
-    'file': const.DC_MSG_FILE,
-    'sticker': const.DC_MSG_STICKER,
+    "text": const.DC_MSG_TEXT,
+    "image": const.DC_MSG_IMAGE,
+    "gif": const.DC_MSG_GIF,
+    "audio": const.DC_MSG_AUDIO,
+    "video": const.DC_MSG_VIDEO,
+    "file": const.DC_MSG_FILE,
+    "sticker": const.DC_MSG_STICKER,
+    "videochat": const.DC_MSG_VIDEOCHAT_INVITATION,
+    "webxdc": const.DC_MSG_WEBXDC,
 }
 
 
@@ -424,14 +469,16 @@ def get_viewtype_code_from_name(view_type_name):
     code = _view_type_mapping.get(view_type_name)
     if code is not None:
         return code
-    raise ValueError("message typecode not found for {!r}, "
-                     "available {!r}".format(view_type_name, list(_view_type_mapping.keys())))
+    raise ValueError(
+        "message typecode not found for {!r}, " "available {!r}".format(view_type_name, list(_view_type_mapping.keys()))
+    )
 
 
 #
 # some helper code for turning system messages into hook events
 #
 
+
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
@@ -448,7 +495,7 @@ def map_system_message(msg):
 
 
 def extract_addr(text):
-    m = re.match(r'.*\((.+@.+)\)', text)
+    m = re.match(r".*\((.+@.+)\)", text)
     if m:
         text = m.group(1)
     text = text.rstrip(".")
@@ -456,9 +503,9 @@ def extract_addr(text):
 
 
 def parse_system_add_remove(text):
-    """ return add/remove info from parsing the given system message text.
+    """return add/remove info from parsing the given system message text.
 
-    returns a (action, affected, actor) triple """
+    returns a (action, affected, actor) triple"""
 
     # Member Me (x@y) removed by a@b.
     # Member x@y added by a@b
@@ -467,7 +514,7 @@ def parse_system_add_remove(text):
     # Group left by some one (tmp1@x.org).
     # Group left by tmp1@x.org.
     text = text.lower()
-    m = re.match(r'member (.+) (removed|added) by (.+)', text)
+    m = re.match(r"member (.+) (removed|added) by (.+)", text)
     if m:
         affected, action, actor = m.groups()
         return action, extract_addr(affected), extract_addr(actor)

python/src/deltachat/props.py

@@ -9,6 +9,7 @@ def with_doc(f):
 # https://github.com/devpi/devpi/blob/master/common/devpi_common/types.py
 def cached(f):
     """returns a cached property that is calculated by function f"""
+
     def get(self):
         try:
             return self._property_cache[f]

python/src/deltachat/provider.py

@@ -26,14 +26,12 @@ class Provider(object):
     @property
     def overview_page(self) -> str:
         """URL to the overview page of the provider on providers.delta.chat."""
-        return from_dc_charpointer(
-            lib.dc_provider_get_overview_page(self._provider))
+        return from_dc_charpointer(lib.dc_provider_get_overview_page(self._provider))
 
     @property
     def get_before_login_hints(self) -> str:
         """Should be shown to the user on login."""
-        return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hint(self._provider))
+        return from_dc_charpointer(lib.dc_provider_get_before_login_hint(self._provider))
 
     @property
     def status(self) -> int:

python/src/deltachat/testplugin.py

@@ -1,56 +1,62 @@
 from __future__ import print_function
-import os
-import sys
-import io
-import subprocess
-import queue
-import threading
+
 import fnmatch
+import io
+import os
+import pathlib
+import queue
+import subprocess
+import sys
+import threading
 import time
 import weakref
 from queue import Queue
-from typing import List, Callable
+from typing import Callable, List, Optional
 
 import pytest
 import requests
-import pathlib
-
-from . import Account, const, account_hookimpl, get_core_info
-from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
 
 import deltachat
 
+from . import Account, account_hookimpl, const, get_core_info
+from .events import FFIEventLogger, FFIEventTracker
+
 
 def pytest_addoption(parser):
     group = parser.getgroup("deltachat testplugin options")
     group.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
+        "--liveconfig",
+        action="store",
+        default=None,
+        help="a file with >=2 lines where each line " "contains NAME=VALUE config settings for one account",
     )
     group.addoption(
-        "--ignored", action="store_true",
+        "--ignored",
+        action="store_true",
         help="Also run tests marked with the ignored marker",
     )
     group.addoption(
-        "--strict-tls", action="store_true",
+        "--strict-tls",
+        action="store_true",
         help="Never accept invalid TLS certificates for test accounts",
     )
     group.addoption(
-        "--extra-info", action="store_true",
-        help="show more info on failures (imap server state, config)"
+        "--extra-info",
+        action="store_true",
+        help="show more info on failures (imap server state, config)",
     )
     group.addoption(
-        "--debug-setup", action="store_true",
-        help="show events during configure and start io phases of online accounts"
+        "--debug-setup",
+        action="store_true",
+        help="show events during configure and start io phases of online accounts",
     )
 
 
 def pytest_configure(config):
-    cfg = config.getoption('--liveconfig')
+    cfg = config.getoption("--liveconfig")
     if not cfg:
-        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
+        cfg = os.getenv("DCC_NEW_TMP_EMAIL")
         if cfg:
             config.option.liveconfig = cfg
 
@@ -113,19 +119,21 @@ def pytest_configure(config):
 
 def pytest_report_header(config, startdir):
     info = get_core_info()
-    summary = ['Deltachat core={} sqlite={} journal_mode={}'.format(
-        info['deltachat_core_version'],
-        info['sqlite_version'],
-        info['journal_mode'],
-    )]
+    summary = [
+        "Deltachat core={} sqlite={} journal_mode={}".format(
+            info["deltachat_core_version"],
+            info["sqlite_version"],
+            info["journal_mode"],
+        )
+    ]
 
     cfg = config.option.liveconfig
     if cfg:
         if "?" in cfg:
             url, token = cfg.split("?", 1)
-            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
+            summary.append("Liveconfig provider: {}?<token ommitted>".format(url))
         else:
-            summary.append('Liveconfig file: {}'.format(cfg))
+            summary.append("Liveconfig file: {}".format(cfg))
     return summary
 
 
@@ -135,15 +143,15 @@ def testprocess(request):
 
 
 class TestProcess:
-    """ A pytest session-scoped instance to help with managing "live" account configurations.
-    """
+    """A pytest session-scoped instance to help with managing "live" account configurations."""
+
     def __init__(self, pytestconfig):
         self.pytestconfig = pytestconfig
         self._addr2files = {}
         self._configlist = []
 
     def get_liveconfig_producer(self):
-        """ provide live account configs, cached on a per-test-process scope
+        """provide live account configs, cached on a per-test-process scope
         so that test functions can re-use already known live configs.
         Depending on the --liveconfig option this comes from
         a HTTP provider or a file with a line specifying each accounts config.
@@ -154,7 +162,7 @@ class TestProcess:
 
         if not liveconfig_opt.startswith("http"):
             for line in open(liveconfig_opt):
-                if line.strip() and not line.strip().startswith('#'):
+                if line.strip() and not line.strip().startswith("#"):
                     d = {}
                     for part in line.split():
                         name, value = part.split("=")
@@ -170,8 +178,7 @@ class TestProcess:
                 except IndexError:
                     res = requests.post(liveconfig_opt)
                     if res.status_code != 200:
-                        pytest.fail("newtmpuser count={} code={}: '{}'".format(
-                                    index, res.status_code, res.text))
+                        pytest.fail("newtmpuser count={} code={}: '{}'".format(index, res.status_code, res.text))
                     d = res.json()
                     config = dict(addr=d["email"], mail_pw=d["password"])
                     print("newtmpuser {}: addr={}".format(index, config["addr"]))
@@ -230,13 +237,17 @@ def data(request):
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
             # users like "deltabot".
-            self.paths = [os.path.normpath(x) for x in [
-                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
-                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
-            ]]
+            self.paths = [
+                os.path.normpath(x)
+                for x in [
+                    os.path.join(os.path.dirname(request.fspath.strpath), "data"),
+                    os.path.join(os.path.dirname(request.fspath.strpath), "..", "..", "test-data"),
+                    os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data"),
+                ]
+            ]
 
         def get_path(self, bn):
-            """ return path of file or None if it doesn't exist. """
+            """return path of file or None if it doesn't exist."""
             for path in self.paths:
                 fn = os.path.join(path, *bn.split("/"))
                 if os.path.exists(fn):
@@ -253,10 +264,11 @@ def data(request):
 
 
 class ACSetup:
-    """ accounts setup helper to deal with multiple configure-process
+    """accounts setup helper to deal with multiple configure-process
     and io & imap initialization phases. From tests, use the higher level
     public ACFactory methods instead of its private helper class.
     """
+
     CONFIGURING = "CONFIGURING"
     CONFIGURED = "CONFIGURED"
     IDLEREADY = "IDLEREADY"
@@ -272,17 +284,18 @@ class ACSetup:
         print("[acsetup]", "{:.3f}".format(time.time() - self.init_time), *args)
 
     def add_configured(self, account):
-        """ add an already configured account. """
+        """add an already configured account."""
         assert account.is_configured()
         self._account2state[account] = self.CONFIGURED
         self.log("added already configured account", account, account.get_config("addr"))
 
     def start_configure(self, account, reconfigure=False):
-        """ add an account and start its configure process. """
+        """add an account and start its configure process."""
+
         class PendingTracker:
             @account_hookimpl
-            def ac_configure_completed(this, success):
-                self._configured_events.put((account, success))
+            def ac_configure_completed(this, success: bool, comment: Optional[str]) -> None:
+                self._configured_events.put((account, success, comment))
 
         account.add_account_plugin(PendingTracker(), name="pending_tracker")
         self._account2state[account] = self.CONFIGURING
@@ -290,7 +303,7 @@ class ACSetup:
         self.log("started configure on", account)
 
     def wait_one_configured(self, account):
-        """ wait until this account has successfully configured. """
+        """wait until this account has successfully configured."""
         if self._account2state[account] == self.CONFIGURING:
             while 1:
                 acc = self._pop_config_success()
@@ -301,7 +314,7 @@ class ACSetup:
             acc._evtracker.consume_events()
 
     def bring_online(self):
-        """ Wait for all accounts to become ready to receive messages.
+        """Wait for all accounts to become ready to receive messages.
 
         This will initialize logging, start IO and the direct_imap attribute
         for each account which either is CONFIGURED already or which is CONFIGURING
@@ -320,9 +333,9 @@ class ACSetup:
         print("finished, account2state", self._account2state)
 
     def _pop_config_success(self):
-        acc, success = self._configured_events.get()
+        acc, success, comment = self._configured_events.get()
         if not success:
-            pytest.fail("configuring online account failed: {}".format(acc))
+            pytest.fail("configuring online account {} failed: {}".format(acc, comment))
         self._account2state[acc] = self.CONFIGURED
         return acc
 
@@ -336,12 +349,12 @@ class ACSetup:
         acc.log("inbox IDLE ready")
 
     def init_logging(self, acc):
-        """ idempotent function for initializing logging (will replace existing logger). """
+        """idempotent function for initializing logging (will replace existing logger)."""
         logger = FFIEventLogger(acc, logid=acc._logid, init_time=self.init_time)
         acc.add_account_plugin(logger, name="logger-" + acc._logid)
 
     def init_imap(self, acc):
-        """ initialize direct_imap and cleanup server state. """
+        """initialize direct_imap and cleanup server state."""
         from deltachat.direct_imap import DirectImap
 
         assert acc.is_configured()
@@ -375,8 +388,7 @@ class ACFactory:
         self._finalizers = []
         self._accounts = []
         self._acsetup = ACSetup(testprocess, self.init_time)
-        self._preconfigured_keys = ["alice", "bob", "charlie",
-                                    "dom", "elena", "fiona"]
+        self._preconfigured_keys = ["alice", "bob", "charlie", "dom", "elena", "fiona"]
         self.set_logging_default(False)
         request.addfinalizer(self.finalize)
 
@@ -399,7 +411,7 @@ class ACFactory:
                 acc.disable_logging()
 
     def get_next_liveconfig(self):
-        """ Base function to get functional online configurations
+        """Base function to get functional online configurations
         where we can make valid SMTP and IMAP connections with.
         """
         configdict = next(self._liveconfig_producer).copy()
@@ -418,16 +430,16 @@ class ACFactory:
         if addr in self.testprocess._addr2files:
             return self._getaccount(addr)
 
-    def get_unconfigured_account(self):
-        return self._getaccount()
+    def get_unconfigured_account(self, closed=False):
+        return self._getaccount(closed=closed)
 
-    def _getaccount(self, try_cache_addr=None):
+    def _getaccount(self, try_cache_addr=None, closed=False):
         logid = "ac{}".format(len(self._accounts) + 1)
         # we need to use fixed database basename for maybe_cache_* functions to work
         path = self.tmpdir.mkdir(logid).join("dc.db")
         if try_cache_addr:
             self.testprocess.cache_maybe_retrieve_configured_db_files(try_cache_addr, path)
-        ac = Account(path.strpath, logging=self._logging)
+        ac = Account(path.strpath, logging=self._logging, closed=closed)
         ac._logid = logid  # later instantiated FFIEventLogger needs this
         ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
         if self.pytestconfig.getoption("--debug-setup"):
@@ -456,16 +468,23 @@ class ACFactory:
             else:
                 print("WARN: could not use preconfigured keys for {!r}".format(addr))
 
-    def get_pseudo_configured_account(self):
+    def get_pseudo_configured_account(self, passphrase: Optional[str] = None) -> Account:
         # do a pseudo-configured account
-        ac = self.get_unconfigured_account()
+        ac = self.get_unconfigured_account(closed=bool(passphrase))
+        if passphrase:
+            ac.open(passphrase)
         acname = ac._logid
         addr = "{}@offline.org".format(acname)
-        ac.update_config(dict(
-            addr=addr, displayname=acname, mail_pw="123",
-            configured_addr=addr, configured_mail_pw="123",
-            configured="1",
-        ))
+        ac.update_config(
+            dict(
+                addr=addr,
+                displayname=acname,
+                mail_pw="123",
+                configured_addr=addr,
+                configured_mail_pw="123",
+                configured="1",
+            )
+        )
         self._preconfigure_key(ac, addr)
         self._acsetup.init_logging(ac)
         return ac
@@ -501,7 +520,7 @@ class ACFactory:
         return ac
 
     def wait_configured(self, account):
-        """ Wait until the specified account has successfully completed configure. """
+        """Wait until the specified account has successfully completed configure."""
         self._acsetup.wait_one_configured(account)
 
     def bring_accounts_online(self):
@@ -531,8 +550,10 @@ class ACFactory:
             sys.executable,
             "-u",
             fn,
-            "--email", bot_cfg["addr"],
-            "--password", bot_cfg["mail_pw"],
+            "--email",
+            bot_cfg["addr"],
+            "--password",
+            bot_cfg["mail_pw"],
             bot_ac.db_path,
         ]
         if ffi:
@@ -543,9 +564,9 @@ class ACFactory:
             stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-            bufsize=0,                 # line buffering
-            close_fds=True,            # close all FDs other than 0/1/2
-            universal_newlines=True    # give back text
+            bufsize=0,  # line buffering
+            close_fds=True,  # close all FDs other than 0/1/2
+            universal_newlines=True,  # give back text
         )
         bot = BotProcess(popen, addr=bot_cfg["addr"])
         self._finalizers.append(bot.kill)
@@ -565,7 +586,7 @@ class ACFactory:
     def introduce_each_other(self, accounts, sending=True):
         to_wait = []
         for i, acc in enumerate(accounts):
-            for acc2 in accounts[i + 1:]:
+            for acc2 in accounts[i + 1 :]:
                 chat = self.get_accepted_chat(acc, acc2)
                 if sending:
                     chat.send_text("hi")

python/src/deltachat/tracker.py

@@ -1,12 +1,11 @@
-
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl, Global
+from .hookspec import Global, account_hookimpl
 
 
 class ImexFailed(RuntimeError):
-    """ Exception for signalling that import/export operations failed."""
+    """Exception for signalling that import/export operations failed."""
 
 
 class ImexTracker:
@@ -24,14 +23,15 @@ class ImexTracker:
         while True:
             ev = self._imex_events.get(timeout=progress_timeout)
             if isinstance(ev, int) and ev >= target_progress:
-                assert ev <= progress_upper_limit, \
+                assert ev <= progress_upper_limit, (
                     str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
+                )
                 return ev
             if ev == 0:
                 return None
 
     def wait_finish(self, progress_timeout=60):
-        """ Return list of written files, raise ValueError if ExportFailed. """
+        """Return list of written files, raise ValueError if ExportFailed."""
         files_written = []
         while True:
             ev = self._imex_events.get(timeout=progress_timeout)
@@ -44,7 +44,7 @@ class ImexTracker:
 
 
 class ConfigureFailed(RuntimeError):
-    """ Exception for signalling that configuration failed."""
+    """Exception for signalling that configuration failed."""
 
 
 class ConfigureTracker:
@@ -77,11 +77,11 @@ class ConfigureTracker:
         self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
-        """ wait until smtp is configured. """
+        """wait until smtp is configured."""
         self._smtp_finished.wait()
 
     def wait_imap_connected(self):
-        """ wait until smtp is configured. """
+        """wait until smtp is configured."""
         self._imap_finished.wait()
 
     def wait_progress(self, data1=None):
@@ -91,7 +91,7 @@ class ConfigureTracker:
                 break
 
     def wait_finish(self, timeout=None):
-        """ wait until configure is completed.
+        """wait until configure is completed.
 
         Raise Exception if Configure failed
         """

python/tests/auditwheels.py

@@ -1,17 +1,19 @@
-
 import os
-import platform
 import subprocess
 import sys
 
-
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
-    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
             subprocess.check_call(
-                ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_" + arch])
+                [
+                    "auditwheel",
+                    "repair",
+                    p,
+                    "-w",
+                    workspacedir,
+                ]
+            )

python/tests/package_wheels.py

@@ -1,8 +1,6 @@
-
 import os
-import sys
 import subprocess
-
+import sys
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2

python/tests/stress_test_db.py

@@ -1,9 +1,9 @@
-
-import time
-import threading
-import pytest
 import os
-from queue import Queue, Empty
+import threading
+import time
+from queue import Empty, Queue
+
+import pytest
 
 import deltachat
 
@@ -24,7 +24,7 @@ def test_db_busy_error(acfactory, tmpdir):
     for acc in accounts:
         acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
         with open(acc.bigfile, "wb") as f:
-            f.write(b"01234567890"*1000_000)
+            f.write(b"01234567890" * 1000_000)
     log("created %s bigfiles" % len(accounts))
 
     contact_addrs = [acc.get_self_contact().addr for acc in accounts]
@@ -63,9 +63,7 @@ def test_db_busy_error(acfactory, tmpdir):
         elif report_type == ReportType.message_echo:
             continue
         else:
-            raise ValueError("{} unknown report type {}, args={}".format(
-                addr, report_type, report_args
-            ))
+            raise ValueError("{} unknown report type {}, args={}".format(addr, report_type, report_args))
         alive_count -= 1
         replier.log("shutting down")
         replier.account.shutdown()
@@ -88,10 +86,7 @@ class AutoReplier:
         self.current_sent = 0
         self.addr = self.account.get_self_contact().addr
 
-        self._thread = threading.Thread(
-            name="Stats{}".format(self.account),
-            target=self.thread_stats
-        )
+        self._thread = threading.Thread(name="Stats{}".format(self.account), target=self.thread_stats)
         self._thread.setDaemon(True)
         self._thread.start()
 

python/tests/test_0_complex_or_slow.py

@@ -1,4 +1,5 @@
 import sys
+
 import pytest
 
 
@@ -277,7 +278,7 @@ def test_fetch_existing_msgs_group_and_single(acfactory, lp):
     assert len(chats) == 4  # two newly created chats + self-chat + device-chat
     group_chat = [c for c in chats if c.get_name() == "group name"][0]
     assert group_chat.is_group()
-    private_chat, = [c for c in chats if c.get_name() == ac1_ac2_chat.get_name()]
+    (private_chat,) = [c for c in chats if c.get_name() == ac1_ac2_chat.get_name()]
     assert not private_chat.is_group()
 
     group_messages = group_chat.get_messages()
@@ -378,7 +379,7 @@ def test_ephemeral_timer(acfactory, lp):
     lp.sec("ac1: check that ephemeral timer is set for chat")
     assert chat1.get_ephemeral_timer() == 60
     chat1_summary = chat1.get_summary()
-    assert chat1_summary["ephemeral_timer"] == {'Enabled': {'duration': 60}}
+    assert chat1_summary["ephemeral_timer"] == {"Enabled": {"duration": 60}}
 
     lp.sec("ac2: receive system message about ephemeral timer modification")
     ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")

python/tests/test_1_online.py

@@ -1,10 +1,11 @@
 import os
-import sys
 import queue
+import sys
 from datetime import datetime, timezone
-from imap_tools import AND, U
 
 import pytest
+from imap_tools import AND, U
+
 from deltachat import const
 from deltachat.hookspec import account_hookimpl
 from deltachat.message import Message
@@ -78,7 +79,7 @@ def test_export_import_self_keys(acfactory, tmpdir, lp):
     assert len(export_files) == 2
     for x in export_files:
         assert x.startswith(dir.strpath)
-    key_id, = ac1._evtracker.get_info_regex_groups(r".*xporting.*KeyId\((.*)\).*")
+    (key_id,) = ac1._evtracker.get_info_regex_groups(r".*xporting.*KeyId\((.*)\).*")
     ac1._evtracker.consume_events()
 
     lp.sec("exported keys (private and public)")
@@ -86,8 +87,7 @@ def test_export_import_self_keys(acfactory, tmpdir, lp):
         lp.indent(dir.strpath + os.sep + name)
     lp.sec("importing into existing account")
     ac2.import_self_keys(dir.strpath)
-    key_id2, = ac2._evtracker.get_info_regex_groups(
-        r".*stored.*KeyId\((.*)\).*", check_error=False)
+    (key_id2,) = ac2._evtracker.get_info_regex_groups(r".*stored.*KeyId\((.*)\).*", check_error=False)
     assert key_id2 == key_id
 
 
@@ -238,6 +238,70 @@ def test_html_message(acfactory, lp):
     assert html_text in msg2.html
 
 
+def test_videochat_invitation_message(acfactory, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    chat = acfactory.get_accepted_chat(ac1, ac2)
+    text = "You are invited to a video chat, click https://meet.jit.si/WxEGad0gGzX to join."
+
+    lp.sec("ac1: prepare and send text message to ac2")
+    msg1 = chat.send_text("message0")
+    assert not msg1.is_videochat_invitation()
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == "message0"
+    assert not msg2.is_videochat_invitation()
+
+    lp.sec("ac1: prepare and send videochat invitation to ac2")
+    msg1 = Message.new_empty(ac1, "videochat")
+    msg1.set_text(text)
+    msg1 = chat.send_msg(msg1)
+    assert msg1.is_videochat_invitation()
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == text
+    assert msg2.is_videochat_invitation()
+
+
+def test_webxdc_message(acfactory, data, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    chat = acfactory.get_accepted_chat(ac1, ac2)
+
+    lp.sec("ac1: prepare and send text message to ac2")
+    msg1 = chat.send_text("message0")
+    assert not msg1.is_webxdc()
+    assert not msg1.send_status_update({"payload": "not an webxdc"}, "invalid")
+    assert not msg1.get_status_updates()
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == "message0"
+    assert not msg2.is_webxdc()
+    assert not msg1.get_status_updates()
+
+    lp.sec("ac1: prepare and send webxdc instance to ac2")
+    msg1 = Message.new_empty(ac1, "webxdc")
+    msg1.set_text("message1")
+    msg1.set_file(data.get_path("webxdc/minimal.xdc"))
+    msg1 = chat.send_msg(msg1)
+    assert msg1.is_webxdc()
+    assert msg1.filename
+
+    assert msg1.send_status_update({"payload": "test1"}, "some test data")
+    assert msg1.send_status_update({"payload": "test2"}, "more test data")
+    assert len(msg1.get_status_updates()) == 2
+    update1 = msg1.get_status_updates()[0]
+    assert update1["payload"] == "test1"
+    assert len(msg1.get_status_updates(update1["serial"])) == 1
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == "message1"
+    assert msg2.is_webxdc()
+    assert msg2.filename
+
+
 def test_mvbox_sentbox_threads(acfactory, lp):
     lp.sec("ac1: start with mvbox thread")
     ac1 = acfactory.new_online_configuring_account(mvbox_move=True, sentbox_watch=True)
@@ -689,7 +753,7 @@ def test_gossip_encryption_preference(acfactory, lp):
     msg = ac1._evtracker.wait_next_incoming_message()
     assert msg.text == "first message"
     assert not msg.is_encrypted()
-    res = "End-to-end encryption preferred:\n{}".format(ac2.get_config('addr'))
+    res = "End-to-end encryption preferred:\n{}".format(ac2.get_config("addr"))
     assert msg.chat.get_encryption_info() == res
     lp.sec("ac2 learns that ac3 prefers encryption")
     ac2.create_chat(ac3)
@@ -701,7 +765,7 @@ def test_gossip_encryption_preference(acfactory, lp):
     lp.sec("ac3 does not know that ac1 prefers encryption")
     ac1.create_chat(ac3)
     chat = ac3.create_chat(ac1)
-    res = "No encryption:\n{}".format(ac1.get_config('addr'))
+    res = "No encryption:\n{}".format(ac1.get_config("addr"))
     assert chat.get_encryption_info() == res
     msg = chat.send_text("not encrypted")
     msg = ac1._evtracker.wait_next_incoming_message()
@@ -766,7 +830,7 @@ def test_send_first_message_as_long_unicode_with_cr(acfactory, lp):
 
 def test_no_draft_if_cant_send(acfactory):
     """Tests that no quote can be set if the user can't send to this chat"""
-    ac1, = acfactory.get_online_accounts(1)
+    (ac1,) = acfactory.get_online_accounts(1)
     device_chat = ac1.get_device_chat()
     msg = Message.new_empty(ac1, "text")
     device_chat.set_draft(msg)
@@ -795,7 +859,9 @@ def test_dont_show_emails(acfactory, lp):
     acfactory.bring_accounts_online()
     ac1.stop_io()
 
-    ac1.direct_imap.append("Drafts", """
+    ac1.direct_imap.append(
+        "Drafts",
+        """
         From: ac1 <{}>
         Subject: subj
         To: alice@example.org
@@ -803,8 +869,13 @@ def test_dont_show_emails(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         message in Drafts that is moved to Sent later
-    """.format(ac1.get_config("configured_addr")))
-    ac1.direct_imap.append("Sent", """
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
+    ac1.direct_imap.append(
+        "Sent",
+        """
         From: ac1 <{}>
         Subject: subj
         To: alice@example.org
@@ -812,8 +883,13 @@ def test_dont_show_emails(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         message in Sent
-    """.format(ac1.get_config("configured_addr")))
-    ac1.direct_imap.append("Spam", """
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
+    ac1.direct_imap.append(
+        "Spam",
+        """
         From: unknown.address@junk.org
         Subject: subj
         To: {}
@@ -821,8 +897,13 @@ def test_dont_show_emails(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         Unknown message in Spam
-    """.format(ac1.get_config("configured_addr")))
-    ac1.direct_imap.append("Junk", """
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
+    ac1.direct_imap.append(
+        "Junk",
+        """
         From: unknown.address@junk.org
         Subject: subj
         To: {}
@@ -830,7 +911,10 @@ def test_dont_show_emails(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         Unknown message in Junk
-    """.format(ac1.get_config("configured_addr")))
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
 
     ac1.set_config("scan_all_folders_debounce_secs", "0")
     lp.sec("All prepared, now let DC find the message")
@@ -1154,7 +1238,7 @@ def test_send_and_receive_image(acfactory, lp, data):
 
 
 def test_import_export_online_all(acfactory, tmpdir, data, lp):
-    ac1, = acfactory.get_online_accounts(1)
+    (ac1,) = acfactory.get_online_accounts(1)
 
     lp.sec("create some chat content")
     chat1 = ac1.create_contact("some1@example.org", name="some1").create_chat()
@@ -1349,8 +1433,9 @@ def test_set_get_contact_avatar(acfactory, data, lp):
 
 
 def test_add_remove_member_remote_events(acfactory, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
     ac1_addr = ac1.get_config("addr")
+    ac3_addr = ac3.get_config("addr")
     # activate local plugin for ac2
     in_list = queue.Queue()
 
@@ -1387,13 +1472,12 @@ def test_add_remove_member_remote_events(acfactory, lp):
     ev = in_list.get()
     assert ev.action == "chat-modified"
     assert chat.is_promoted()
-    assert sorted(x.addr for x in chat.get_contacts()) == \
-        sorted(x.addr for x in ev.chat.get_contacts())
+    assert sorted(x.addr for x in chat.get_contacts()) == sorted(x.addr for x in ev.chat.get_contacts())
 
     lp.sec("ac1: add address2")
     # note that if the above create_chat() would not
     # happen we would not receive a proper member_added event
-    contact2 = chat.add_contact("devnull@testrun.org")
+    contact2 = chat.add_contact(ac3_addr)
     ev = in_list.get()
     assert ev.action == "chat-modified"
     ev = in_list.get()
@@ -1401,7 +1485,7 @@ def test_add_remove_member_remote_events(acfactory, lp):
     ev = in_list.get()
     assert ev.action == "added"
     assert ev.message.get_sender_contact().addr == ac1_addr
-    assert ev.contact.addr == "devnull@testrun.org"
+    assert ev.contact.addr == ac3_addr
 
     lp.sec("ac1: remove address2")
     chat.remove_contact(contact2)
@@ -1530,8 +1614,10 @@ def test_connectivity(acfactory, lp):
     ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTING)
     ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTING, const.DC_CONNECTIVITY_CONNECTED)
 
-    lp.sec("Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, " +
-           "all messages are fetched")
+    lp.sec(
+        "Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, "
+        + "all messages are fetched"
+    )
 
     ac1.direct_imap.select_config_folder("inbox")
     with ac1.direct_imap.idle() as idle1:
@@ -1594,10 +1680,12 @@ def test_fetch_deleted_msg(acfactory, lp):
 
     See https://github.com/deltachat/deltachat-core-rust/issues/2429.
     """
-    ac1, = acfactory.get_online_accounts(1)
+    (ac1,) = acfactory.get_online_accounts(1)
     ac1.stop_io()
 
-    ac1.direct_imap.append("INBOX", """
+    ac1.direct_imap.append(
+        "INBOX",
+        """
         From: alice <alice@example.org>
         Subject: subj
         To: bob@example.com
@@ -1606,7 +1694,8 @@ def test_fetch_deleted_msg(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         Deleted message
-    """)
+    """,
+    )
     ac1.direct_imap.delete("1:*", expunge=False)
     ac1.start_io()
 
@@ -1888,11 +1977,26 @@ def test_group_quote(acfactory, lp):
     assert received_reply.quote.id == out_msg.id
 
 
-@pytest.mark.parametrize("folder,move,expected_destination,", [
-    ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
-    ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
-    ("Spam", False, "INBOX"),  # ...emails are moved from the spam folder to the Inbox
-])
+@pytest.mark.parametrize(
+    "folder,move,expected_destination,",
+    [
+        (
+            "xyz",
+            False,
+            "xyz",
+        ),  # Test that emails are recognized in a random folder but not moved
+        (
+            "xyz",
+            True,
+            "DeltaChat",
+        ),  # ...emails are found in a random folder and moved to DeltaChat
+        (
+            "Spam",
+            False,
+            "INBOX",
+        ),  # ...emails are moved from the spam folder to the Inbox
+    ],
+)
 # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
 # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
 def test_scan_folders(acfactory, lp, folder, move, expected_destination):

python/tests/test_2_increation.py

@@ -2,13 +2,13 @@ from __future__ import print_function
 
 import os.path
 import shutil
+from filecmp import cmp
 
 import pytest
-from filecmp import cmp
 
 
 def wait_msg_delivered(account, msg_list):
-    """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
+    """wait for one or more MSG_DELIVERED events to match msg_list contents."""
     msg_list = list(msg_list)
     while msg_list:
         ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
@@ -16,7 +16,7 @@ def wait_msg_delivered(account, msg_list):
 
 
 def wait_msgs_changed(account, msgs_list):
-    """ wait for one or more MSGS_CHANGED events to match msgs_list contents. """
+    """wait for one or more MSGS_CHANGED events to match msgs_list contents."""
     account.log("waiting for msgs_list={}".format(msgs_list))
     msgs_list = list(msgs_list)
     while msgs_list:
@@ -38,7 +38,7 @@ class TestOnlineInCreation:
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
-        src = tmpdir.join('file.txt').ensure(file=1)
+        src = tmpdir.join("file.txt").ensure(file=1)
         with pytest.raises(Exception):
             chat.prepare_message_file(src.strpath)
 
@@ -48,11 +48,11 @@ class TestOnlineInCreation:
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
-        src = tmpdir.join('file.txt')
+        src = tmpdir.join("file.txt")
         src.write("hello there\n")
         chat.send_file(src.strpath)
 
-        blob_src = os.path.join(ac1.get_blobdir(), 'file.txt')
+        blob_src = os.path.join(ac1.get_blobdir(), "file.txt")
         assert os.path.exists(blob_src), "file.txt not copied to blobdir"
 
     def test_forward_increation(self, acfactory, data, lp):
@@ -63,7 +63,7 @@ class TestOnlineInCreation:
 
         lp.sec("create a message with a file in creation")
         orig = data.get_path("d.png")
-        path = os.path.join(ac1.get_blobdir(), 'd.png')
+        path = os.path.join(ac1.get_blobdir(), "d.png")
         with open(path, "x") as fp:
             fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
@@ -94,10 +94,7 @@ class TestOnlineInCreation:
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
         lp.sec("wait for both messages to be delivered to SMTP")
-        wait_msg_delivered(ac1, [
-            (chat2.id, forwarded_id),
-            (chat.id, prepared_original.id)
-        ])
+        wait_msg_delivered(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
 
         lp.sec("wait1 for original or forwarded messages to arrive")
         received_original = ac2._evtracker.wait_next_incoming_message()

python/tests/test_3_offline.py

@@ -1,32 +1,51 @@
 from __future__ import print_function
-import pytest
+
 import os
 import time
-from deltachat import const, Account
-from deltachat.message import Message
-from deltachat.hookspec import account_hookimpl
-from deltachat.capi import ffi, lib
-from deltachat.cutil import iter_array
 from datetime import datetime, timedelta, timezone
 
+import pytest
 
-@pytest.mark.parametrize("msgtext,res", [
-    ("Member Me (tmp1@x.org) removed by tmp2@x.org.",
-        ("removed", "tmp1@x.org", "tmp2@x.org")),
-    ("Member With space (tmp1@x.org) removed by tmp2@x.org.",
-        ("removed", "tmp1@x.org", "tmp2@x.org")),
-    ("Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
-        ("removed", "tmp1@x.org", "tmp2@x.org")),
-    ("Member With space (tmp1@x.org) removed by me",
-        ("removed", "tmp1@x.org", "me")),
-    ("Group left by some one (tmp1@x.org).",
-        ("removed", "tmp1@x.org", "tmp1@x.org")),
-    ("Group left by tmp1@x.org.",
-        ("removed", "tmp1@x.org", "tmp1@x.org")),
-    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org", "tmp2@x.org")),
-    ("Member nothing bla bla", None),
-    ("Another unknown system message", None),
-])
+from deltachat import Account, const
+from deltachat.capi import ffi, lib
+from deltachat.cutil import iter_array
+from deltachat.hookspec import account_hookimpl
+from deltachat.message import Message
+from deltachat.tracker import ImexFailed
+
+
+@pytest.mark.parametrize(
+    "msgtext,res",
+    [
+        (
+            "Member Me (tmp1@x.org) removed by tmp2@x.org.",
+            ("removed", "tmp1@x.org", "tmp2@x.org"),
+        ),
+        (
+            "Member With space (tmp1@x.org) removed by tmp2@x.org.",
+            ("removed", "tmp1@x.org", "tmp2@x.org"),
+        ),
+        (
+            "Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+            ("removed", "tmp1@x.org", "tmp2@x.org"),
+        ),
+        (
+            "Member With space (tmp1@x.org) removed by me",
+            ("removed", "tmp1@x.org", "me"),
+        ),
+        (
+            "Group left by some one (tmp1@x.org).",
+            ("removed", "tmp1@x.org", "tmp1@x.org"),
+        ),
+        ("Group left by tmp1@x.org.", ("removed", "tmp1@x.org", "tmp1@x.org")),
+        (
+            "Member tmp1@x.org added by tmp2@x.org.",
+            ("added", "tmp1@x.org", "tmp2@x.org"),
+        ),
+        ("Member nothing bla bla", None),
+        ("Another unknown system message", None),
+    ],
+)
 def test_parse_system_add_remove(msgtext, res):
     from deltachat.message import parse_system_add_remove
 
@@ -136,9 +155,11 @@ class TestOfflineContact:
         ac1 = acfactory.get_pseudo_configured_account()
         contact1 = ac1.create_contact("some1@example.org", name="some1")
         contact2 = ac1.create_contact("some1@example.org", name="some1")
+        contact3 = None
         str(contact1)
         repr(contact1)
         assert contact1 == contact2
+        assert contact1 != contact3
         assert contact1.id
         assert contact1.addr == "some1@example.org"
         assert contact1.display_name == "some1"
@@ -232,10 +253,12 @@ class TestOfflineChat:
     def test_chat_idempotent(self, chat1, ac1):
         contact1 = chat1.get_contacts()[0]
         chat2 = contact1.create_chat()
+        chat3 = None
         assert chat2.id == chat1.id
         assert chat2.get_name() == chat1.get_name()
         assert chat1 == chat2
         assert not (chat1 != chat2)
+        assert chat1 != chat3
 
         for ichat in ac1.get_chats():
             if ichat.id == chat1.id:
@@ -387,6 +410,8 @@ class TestOfflineChat:
 
     def test_message_eq_contains(self, chat1):
         msg = chat1.send_text("msg1")
+        msg2 = None
+        assert msg != msg2
         assert msg in chat1.get_messages()
         assert not (msg not in chat1.get_messages())
         str(msg)
@@ -424,11 +449,14 @@ class TestOfflineChat:
         assert os.path.exists(msg.filename)
         assert msg.filemime == "image/png"
 
-    @pytest.mark.parametrize("typein,typeout", [
+    @pytest.mark.parametrize(
+        "typein,typeout",
+        [
             (None, "application/octet-stream"),
             ("text/plain", "text/plain"),
             ("image/png", "image/png"),
-    ])
+        ],
+    )
     def test_message_file(self, ac1, chat1, data, lp, typein, typeout):
         lp.sec("sending file")
         fn = data.get_path("r.txt")
@@ -475,7 +503,7 @@ class TestOfflineChat:
         with pytest.raises(ValueError):
             ac1.set_config("addr", "123@example.org")
 
-    def test_import_export_one_contact(self, acfactory, tmpdir):
+    def test_import_export_on_unencrypted_acct(self, acfactory, tmpdir):
         backupdir = tmpdir.mkdir("backup")
         ac1 = acfactory.get_pseudo_configured_account()
         chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
@@ -504,6 +532,162 @@ class TestOfflineChat:
         assert messages[0].text == "msg1"
         assert os.path.exists(messages[1].filename)
 
+    def test_import_export_on_encrypted_acct(self, acfactory, tmpdir):
+        passphrase1 = "passphrase1"
+        passphrase2 = "passphrase2"
+        backupdir = tmpdir.mkdir("backup")
+        ac1 = acfactory.get_pseudo_configured_account(passphrase=passphrase1)
+
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+        assert not backupdir.listdir()
+        ac1.stop_io()
+
+        path = ac1.export_all(backupdir.strpath)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account(closed=True)
+        ac2.open(passphrase2)
+        ac2.import_all(path)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+        ac2.shutdown()
+
+        # check that passphrase is not lost after import:
+        ac2 = Account(ac2.db_path, logging=ac2._logging, closed=True)
+        ac2.open(passphrase2)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+    def test_import_export_with_passphrase(self, acfactory, tmpdir):
+        passphrase = "test_passphrase"
+        wrong_passphrase = "wrong_passprase"
+        backupdir = tmpdir.mkdir("backup")
+        ac1 = acfactory.get_pseudo_configured_account()
+
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+        assert not backupdir.listdir()
+        ac1.stop_io()
+
+        path = ac1.export_all(backupdir.strpath, passphrase)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account()
+        with pytest.raises(ImexFailed):
+            ac2.import_all(path, wrong_passphrase)
+        ac2.import_all(path, passphrase)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+    def test_import_encrypted_bak_into_encrypted_acct(self, acfactory, tmpdir):
+        """
+        Test that account passphrase isn't lost if backup failed to be imported.
+        See https://github.com/deltachat/deltachat-core-rust/issues/3379
+        """
+        acct_passphrase = "passphrase1"
+        bak_passphrase = "passphrase2"
+        wrong_passphrase = "wrong_passprase"
+        backupdir = tmpdir.mkdir("backup")
+
+        ac1 = acfactory.get_pseudo_configured_account()
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+        assert not backupdir.listdir()
+        ac1.stop_io()
+
+        path = ac1.export_all(backupdir.strpath, bak_passphrase)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account(closed=True)
+        ac2.open(acct_passphrase)
+        with pytest.raises(ImexFailed):
+            ac2.import_all(path, wrong_passphrase)
+        ac2.import_all(path, bak_passphrase)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+        ac2.shutdown()
+
+        # check that passphrase is not lost after import
+        ac2 = Account(ac2.db_path, logging=ac2._logging, closed=True)
+        ac2.open(acct_passphrase)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
     def test_set_get_draft(self, chat1):
         msg = Message.new_empty(chat1.account, "text")
         msg1 = chat1.prepare_message(msg)
@@ -629,6 +813,6 @@ class TestOfflineChat:
         lp.sec("check message count of only system messages (without daymarkers)")
         dc_array = ffi.gc(
             lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_INFO_ONLY, 0),
-            lib.dc_array_unref
+            lib.dc_array_unref,
         )
         assert len(list(iter_array(dc_array, lambda x: x))) == 2

python/tests/test_4_lowlevel.py

@@ -1,24 +1,25 @@
-
 import os
-
 from queue import Queue
-from deltachat import capi, cutil, const
-from deltachat import register_global_plugin
+
+from deltachat import capi, const, cutil, register_global_plugin
+from deltachat.capi import ffi, lib
 from deltachat.hookspec import global_hookimpl
-from deltachat.capi import ffi
-from deltachat.capi import lib
-from deltachat.testplugin import ACSetup, create_dict_from_files_in_path, write_dict_to_dir
+from deltachat.testplugin import (
+    ACSetup,
+    create_dict_from_files_in_path,
+    write_dict_to_dir,
+)
+
 # from deltachat.account import EventLogger
 
 
 class TestACSetup:
-
     def test_cache_writing(self, tmp_path):
         base = tmp_path.joinpath("hello")
         base.mkdir()
         d1 = base.joinpath("dir1")
         d1.mkdir()
-        d1.joinpath("file1").write_bytes(b'content1')
+        d1.joinpath("file1").write_bytes(b"content1")
         d2 = d1.joinpath("dir2")
         d2.mkdir()
         d2.joinpath("file2").write_bytes(b"123")
@@ -34,7 +35,7 @@ class TestACSetup:
         monkeypatch.setattr(acc, "configure", lambda **kwargs: None)
         pc.start_configure(acc)
         assert pc._account2state[acc] == pc.CONFIGURING
-        pc._configured_events.put((acc, True))
+        pc._configured_events.put((acc, True, None))
         monkeypatch.setattr(pc, "init_imap", lambda *args, **kwargs: None)
         pc.wait_one_configured(acc)
         assert pc._account2state[acc] == pc.CONFIGURED
@@ -54,11 +55,11 @@ class TestACSetup:
         pc.start_configure(ac2)
         assert pc._account2state[ac1] == pc.CONFIGURING
         assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac1, True))
+        pc._configured_events.put((ac1, True, None))
         pc.wait_one_configured(ac1)
         assert pc._account2state[ac1] == pc.CONFIGURED
         assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac2, True))
+        pc._configured_events.put((ac2, True, None))
         pc.bring_online()
         assert pc._account2state[ac1] == pc.IDLEREADY
         assert pc._account2state[ac2] == pc.IDLEREADY
@@ -103,6 +104,7 @@ def test_dc_close_events(tmpdir, acfactory):
         def dc_account_after_shutdown(self, account):
             assert account._dc_context is None
             shutdowns.put(account)
+
     register_global_plugin(ShutdownPlugin())
     assert hasattr(ac1, "_dc_context")
     ac1.shutdown()
@@ -178,8 +180,8 @@ def test_get_info_open(tmpdir):
         lib.dc_context_unref,
     )
     info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
-    assert 'deltachat_core_version' in info
-    assert 'database_dir' in info
+    assert "deltachat_core_version" in info
+    assert "database_dir" in info
 
 
 def test_logged_hook_failure(acfactory):
@@ -187,7 +189,7 @@ def test_logged_hook_failure(acfactory):
     cap = []
     ac1.log = cap.append
     with ac1._event_thread.swallow_and_log_exception("some"):
-        0/0
+        0 / 0
     assert cap
     assert "some" in str(cap)
     assert "ZeroDivisionError" in str(cap)
@@ -202,7 +204,7 @@ def test_logged_ac_process_ffi_failure(acfactory):
     class FailPlugin:
         @account_hookimpl
         def ac_process_ffi_event(ffi_event):
-            0/0
+            0 / 0
 
     cap = Queue()
     ac1.log = cap.put

python/tox.ini

@@ -28,6 +28,10 @@ deps =
 [testenv:auditwheels]
 skipsdist = True
 deps = auditwheel
+passenv =
+    AUDITWHEEL_ARCH
+    AUDITWHEEL_PLAT
+    AUDITWHEEL_POLICY
 commands = 
     python tests/auditwheels.py {toxworkdir}/wheelhouse
 
@@ -36,10 +40,14 @@ skipsdist = True
 skip_install = True
 deps =
     flake8
+    isort
+    black
     # pygments required by rst-lint
     pygments
     restructuredtext_lint 
 commands =
+    isort --check setup.py install_python_bindings.py src/deltachat examples/ tests/
+    black --check setup.py install_python_bindings.py src/deltachat examples/ tests/
     flake8 src/deltachat
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
@@ -85,3 +93,4 @@ markers =
 
 [flake8]
 max-line-length = 120
+ignore = E203, E266, E501, W503

rustfmt.toml

@@ -1 +0,0 @@
-edition = "2018"

scripts/README.md

@@ -25,7 +25,7 @@ and an own build machine.
 
 There is experimental support for triggering a remote Python or Rust test run 
 from your local checkout/branch. You will need to be authorized to login to 
-the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
+the build machine (ask your friendly sysadmin on #deltachat Libera Chat) to type::
 
     scripts/manual_remote_tests.sh rust
     scripts/manual_remote_tests.sh python

scripts/concourse/docs_wheels.yml

@@ -78,14 +78,15 @@ jobs:
         - name: deltachat-core-rust
       image_resource:
         source:
-          repository: vito/oci-build-task
+          repository: concourse/oci-build-task
         type: registry-image
       outputs:
         - name: coredeps-image
           path: image
       params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps
+        CONTEXT: deltachat-core-rust/scripts/coredeps
         UNPACK_ROOTFS: "true"
+        BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_x86_64
       platform: linux
       caches:
       - path: cache
@@ -177,14 +178,15 @@ jobs:
         - name: deltachat-core-rust
       image_resource:
         source:
-          repository: vito/oci-build-task
+          repository: concourse/oci-build-task
         type: registry-image
       outputs:
         - name: coredeps-image
           path: image
       params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
+        CONTEXT: deltachat-core-rust/scripts/coredeps
         UNPACK_ROOTFS: "true"
+        BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_aarch64
       platform: linux
       caches:
       - path: cache
@@ -230,3 +232,73 @@ jobs:
           devpi use https://m.devpi.net/dc/master
           devpi login ((devpi.login)) --password ((devpi.password))
           devpi upload py-wheels/*manylinux201*
+
+- name: python-musl-x86_64
+  plan:
+  - get: deltachat-core-rust
+  - get: deltachat-core-rust-release
+    trigger: true
+
+  # Build manylinux image with additional dependencies
+  - task: build-coredeps
+    privileged: true
+    config:
+      inputs:
+        # Building the latest, not tagged coredeps
+        - name: deltachat-core-rust
+      image_resource:
+        source:
+          repository: concourse/oci-build-task
+        type: registry-image
+      outputs:
+        - name: coredeps-image
+          path: image
+      params:
+        CONTEXT: deltachat-core-rust/scripts/coredeps
+        UNPACK_ROOTFS: "true"
+        BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_x86_64
+      platform: linux
+      caches:
+      - path: cache
+      run:
+        path: build
+
+  # Use built image to build python wheels
+  - task: build-wheels
+    image: coredeps-image
+    config:
+      inputs:
+        - name: deltachat-core-rust-release
+          path: .
+      outputs:
+        - name: py-wheels
+          path: ./python/.docker-tox/wheelhouse/
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          scripts/run_all.sh
+
+  # Upload musl x86_64 wheels
+  - task: upload-wheels
+    config:
+      inputs:
+        - name: py-wheels
+      image_resource:
+        type: registry-image
+        source:
+          repository: debian
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apt-get update -y
+          apt-get install -y --no-install-recommends python3-pip python3-setuptools
+          pip3 install devpi
+          devpi use https://m.devpi.net/dc/master
+          devpi login ((devpi.login)) --password ((devpi.password))
+          devpi upload py-wheels/*musllinux_1_1_x86_64*

scripts/coredeps/Dockerfile

@@ -0,0 +1,8 @@
+ARG BASEIMAGE=quay.io/pypa/manylinux2014_x86_64
+#ARG BASEIMAGE=quay.io/pypa/musllinux_1_1_x86_64
+#ARG BASEIMAGE=quay.io/pypa/manylinux2014_aarch64
+
+FROM $BASEIMAGE
+RUN pipx install tox
+COPY install-rust.sh /scripts/
+RUN /scripts/install-rust.sh

scripts/coredeps/install-rust.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Install Rust
+#
+# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
+#
+# Avoid using rustup here as it depends on reading /proc/self/exe and
+# has problems running under QEMU.
+RUST_VERSION=1.61.0
+
+ARCH="$(uname -m)"
+test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu
+
+curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$ARCH-unknown-linux-$LIBC.tar.gz" | tar xz
+cd "rust-${RUST_VERSION}-$ARCH-unknown-linux-$LIBC"
+./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$ARCH-unknown-linux-$LIBC"
+rustc --version
+cd ..
+rm -fr "rust-${RUST_VERSION}-$ARCH-unknown-linux-$LIBC"

scripts/docker-coredeps-arm64/Dockerfile

@@ -1,21 +0,0 @@
-FROM quay.io/pypa/manylinux2014_aarch64
-
-# Configure ld.so/ldconfig and pkg-config
-RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
-    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
-ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
-
-# Install a recent Perl, needed to install the openssl crate
-ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-
-ENV PIP_DISABLE_PIP_VERSION_CHECK 1
-
-# Install python tools (auditwheels,tox, ...)
-ADD deps/build_python.sh /builder/build_python.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
-
-# Install Rust
-ADD deps/build_rust.sh /builder/build_rust.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps-arm64/deps/build_openssl.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-OPENSSL_VERSION=1.1.1a
-OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
-
-curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf openssl-${OPENSSL_VERSION}.tar.gz
-cd openssl-${OPENSSL_VERSION} 
-./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
-
-sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
-sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
-sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
-
-make depend
-make 
-make install_sw install_ssldirs
-ldconfig -v | grep ssl 

scripts/docker-coredeps-arm64/deps/build_perl.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
-curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar -xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION}
-
-./Configure -de
-make
-make install

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.7 environment as the base environment 
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel 
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
-
-popd

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.61.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
-rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/Dockerfile

@@ -1,21 +0,0 @@
-FROM quay.io/pypa/manylinux2014_x86_64
-
-# Configure ld.so/ldconfig and pkg-config
-RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
-    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
-ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
-
-# Install a recent Perl, needed to install the openssl crate
-ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-
-ENV PIP_DISABLE_PIP_VERSION_CHECK 1
-
-# Install python tools (auditwheels,tox, ...)
-ADD deps/build_python.sh /builder/build_python.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
-
-# Install Rust
-ADD deps/build_rust.sh /builder/build_rust.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps/deps/build_openssl.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-OPENSSL_VERSION=1.1.1a
-OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
-
-curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf openssl-${OPENSSL_VERSION}.tar.gz
-cd openssl-${OPENSSL_VERSION} 
-./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
-
-sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
-sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
-sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
-
-make depend
-make 
-make install_sw install_ssldirs
-ldconfig -v | grep ssl 

scripts/docker-coredeps/deps/build_perl.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
-curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar -xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION}
-
-./Configure -de
-make
-make install

scripts/docker-coredeps/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.7 environment as the base environment
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
-
-popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.61.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
-rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/run-python-test.sh

@@ -22,4 +22,4 @@ export PYTHONDONTWRITEBYTECODE=1
 # run python tests (tox invokes pytest to run tests in python/tests)
 #TOX_PARALLEL_NO_SPINNER=1 tox -e lint,doc
 tox -e lint
-tox -e doc,py37
+tox -e doc,py3

scripts/run_all.sh

@@ -5,7 +5,6 @@
 set -e -x
 
 # Perform clean build of core and install.
-export TOXWORKDIR=.docker-tox
 
 # compile core lib
 
@@ -21,15 +20,8 @@ export DCC_RS_TARGET=release
 # needed by tox below.
 export PATH=$PATH:/opt/python/cp37-cp37m/bin
 export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-rm -f python3.7
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-rm -f python3.8
-ln -s /opt/python/cp38-cp38/bin/python3.8
-rm -f python3.9
-ln -s /opt/python/cp39-cp39/bin/python3.9
-popd
 
+TOXWORKDIR=.docker-tox
 pushd python
 # prepare a clean tox run
 rm -rf tests/__pycache__
@@ -41,7 +33,7 @@ mkdir -p $TOXWORKDIR
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,auditwheels
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,auditwheels
 popd
 
 

src/accounts.rs

@@ -1,19 +1,15 @@
 //! # Account manager module.
 
 use std::collections::BTreeMap;
-
-use async_std::channel::{self, Receiver, Sender};
-use async_std::fs;
-use async_std::path::PathBuf;
-use async_std::prelude::*;
-use async_std::sync::{Arc, RwLock};
-use uuid::Uuid;
+use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
+use tokio::fs;
+use uuid::Uuid;
 
 use crate::context::Context;
-use crate::events::{Event, EventType, Events};
+use crate::events::{Event, EventEmitter, EventType, Events};
 
 /// Account manager, that can handle multiple accounts in a single place.
 #[derive(Debug)]
@@ -21,7 +17,6 @@ pub struct Accounts {
     dir: PathBuf,
     config: Config,
     accounts: BTreeMap<u32, Context>,
-    emitter: EventEmitter,
 
     /// Event channel to emit account manager errors.
     events: Events,
@@ -30,7 +25,7 @@ pub struct Accounts {
 impl Accounts {
     /// Loads or creates an accounts folder at the given `dir`.
     pub async fn new(dir: PathBuf) -> Result<Self> {
-        if !dir.exists().await {
+        if !dir.exists() {
             Accounts::create(&dir).await?;
         }
 
@@ -51,40 +46,24 @@ impl Accounts {
     /// Opens an existing accounts structure. Will error if the folder doesn't exist,
     /// no account exists and no config exists.
     pub async fn open(dir: PathBuf) -> Result<Self> {
-        ensure!(dir.exists().await, "directory does not exist");
+        ensure!(dir.exists(), "directory does not exist");
 
         let config_file = dir.join(CONFIG_NAME);
-        ensure!(
-            config_file.exists().await,
-            "{:?} does not exist",
-            config_file
-        );
+        ensure!(config_file.exists(), "{:?} does not exist", config_file);
 
         let config = Config::from_file(config_file)
             .await
             .context("failed to load accounts config")?;
+        let events = Events::new();
         let accounts = config
-            .load_accounts()
+            .load_accounts(&events)
             .await
             .context("failed to load accounts")?;
 
-        let emitter = EventEmitter::new();
-
-        let events = Events::default();
-
-        emitter.sender.send(events.get_emitter()).await?;
-
-        for account in accounts.values() {
-            emitter.add_account(account).await.with_context(|| {
-                format!("failed to add account {} to event emitter", account.id)
-            })?;
-        }
-
         Ok(Self {
             dir,
             config,
             accounts,
-            emitter,
             events,
         })
     }
@@ -121,8 +100,12 @@ impl Accounts {
     pub async fn add_account(&mut self) -> Result<u32> {
         let account_config = self.config.new_account(&self.dir).await?;
 
-        let ctx = Context::new(account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
+        let ctx = Context::new(
+            &account_config.dbfile(),
+            account_config.id,
+            self.events.clone(),
+        )
+        .await?;
         self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -132,8 +115,12 @@ impl Accounts {
     pub async fn add_closed_account(&mut self) -> Result<u32> {
         let account_config = self.config.new_account(&self.dir).await?;
 
-        let ctx = Context::new_closed(account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
+        let ctx = Context::new_closed(
+            &account_config.dbfile(),
+            account_config.id,
+            self.events.clone(),
+        )
+        .await?;
         self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -156,7 +143,7 @@ impl Accounts {
             loop {
                 counter += 1;
 
-                if let Err(err) = fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                if let Err(err) = fs::remove_dir_all(&cfg.dir)
                     .await
                     .context("failed to remove account data")
                 {
@@ -165,7 +152,7 @@ impl Accounts {
                     }
 
                     // Wait 1 second and try again.
-                    async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
+                    tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
                 } else {
                     break;
                 }
@@ -181,16 +168,8 @@ impl Accounts {
         let blobdir = Context::derive_blobdir(&dbfile);
         let walfile = Context::derive_walfile(&dbfile);
 
-        ensure!(
-            dbfile.exists().await,
-            "no database found: {}",
-            dbfile.display()
-        );
-        ensure!(
-            blobdir.exists().await,
-            "no blobdir found: {}",
-            blobdir.display()
-        );
+        ensure!(dbfile.exists(), "no database found: {}", dbfile.display());
+        ensure!(blobdir.exists(), "no blobdir found: {}", blobdir.display());
 
         let old_id = self.config.get_selected_account().await;
 
@@ -201,7 +180,7 @@ impl Accounts {
             .await
             .context("failed to create new account")?;
 
-        let new_dbfile = account_config.dbfile().into();
+        let new_dbfile = account_config.dbfile();
         let new_blobdir = Context::derive_blobdir(&new_dbfile);
         let new_walfile = Context::derive_walfile(&new_dbfile);
 
@@ -215,7 +194,7 @@ impl Accounts {
             fs::rename(&blobdir, &new_blobdir)
                 .await
                 .context("failed to rename blobdir")?;
-            if walfile.exists().await {
+            if walfile.exists() {
                 fs::rename(&walfile, &new_walfile)
                     .await
                     .context("failed to rename walfile")?;
@@ -225,14 +204,13 @@ impl Accounts {
 
         match res {
             Ok(_) => {
-                let ctx = Context::new(new_dbfile, account_config.id).await?;
-                self.emitter.add_account(&ctx).await?;
+                let ctx = Context::new(&new_dbfile, account_config.id, self.events.clone()).await?;
                 self.accounts.insert(account_config.id, ctx);
                 Ok(account_config.id)
             }
             Err(err) => {
                 // remove temp account
-                fs::remove_dir_all(async_std::path::PathBuf::from(&account_config.dir))
+                fs::remove_dir_all(std::path::PathBuf::from(&account_config.dir))
                     .await
                     .context("failed to remove account data")?;
 
@@ -277,6 +255,9 @@ impl Accounts {
     }
 
     pub async fn stop_io(&self) {
+        // Sending an event here wakes up event loop even
+        // if there are no accounts.
+        info!(self, "Stopping IO for all accounts");
         for account in self.accounts.values() {
             account.stop_io().await;
         }
@@ -299,74 +280,9 @@ impl Accounts {
         self.events.emit(Event { id: 0, typ: event })
     }
 
-    /// Returns unified event emitter.
+    /// Returns event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
-        self.emitter.clone()
-    }
-}
-
-/// Unified event emitter for multiple accounts.
-#[derive(Debug, Clone)]
-pub struct EventEmitter {
-    /// Aggregate stream of events from all accounts.
-    stream: Arc<RwLock<futures::stream::SelectAll<crate::events::EventEmitter>>>,
-
-    /// Sender for the channel where new account emitters will be pushed.
-    sender: Sender<crate::events::EventEmitter>,
-
-    /// Receiver for the channel where new account emitters will be pushed.
-    receiver: Receiver<crate::events::EventEmitter>,
-}
-
-impl EventEmitter {
-    pub fn new() -> Self {
-        let (sender, receiver) = channel::unbounded();
-        Self {
-            stream: Arc::new(RwLock::new(futures::stream::SelectAll::new())),
-            sender,
-            receiver,
-        }
-    }
-
-    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
-    pub fn recv_sync(&mut self) -> Option<Event> {
-        async_std::task::block_on(self.recv()).unwrap_or_default()
-    }
-
-    /// Async recv of an event. Return `None` if all `Sender`s have been dropped.
-    pub async fn recv(&mut self) -> Result<Option<Event>> {
-        let mut stream = self.stream.write().await;
-        loop {
-            match futures::future::select(self.receiver.recv(), stream.next()).await {
-                futures::future::Either::Left((emitter, _)) => {
-                    stream.push(emitter?);
-                }
-                futures::future::Either::Right((ev, _)) => return Ok(ev),
-            }
-        }
-    }
-
-    /// Add event emitter of a new account to the aggregate event emitter.
-    pub async fn add_account(&self, context: &Context) -> Result<()> {
-        self.sender.send(context.get_event_emitter()).await?;
-        Ok(())
-    }
-}
-
-impl Default for EventEmitter {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl async_std::stream::Stream for EventEmitter {
-    type Item = Event;
-
-    fn poll_next(
-        mut self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Option<Self::Item>> {
-        std::pin::Pin::new(&mut self).poll_next(cx)
+        self.events.get_emitter()
     }
 }
 
@@ -392,7 +308,7 @@ struct InnerConfig {
 }
 
 impl Config {
-    pub async fn new(dir: &PathBuf) -> Result<Self> {
+    pub async fn new(dir: &Path) -> Result<Self> {
         let inner = InnerConfig {
             accounts: Vec::new(),
             selected_account: 0,
@@ -423,10 +339,10 @@ impl Config {
         Ok(Config { file, inner })
     }
 
-    pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
+    pub async fn load_accounts(&self, events: &Events) -> Result<BTreeMap<u32, Context>> {
         let mut accounts = BTreeMap::new();
         for account_config in &self.inner.accounts {
-            let ctx = Context::new(account_config.dbfile().into(), account_config.id)
+            let ctx = Context::new(&account_config.dbfile(), account_config.id, events.clone())
                 .await
                 .with_context(|| {
                     format!(
@@ -442,7 +358,7 @@ impl Config {
     }
 
     /// Create a new account in the given root directory.
-    async fn new_account(&mut self, dir: &PathBuf) -> Result<AccountConfig> {
+    async fn new_account(&mut self, dir: &Path) -> Result<AccountConfig> {
         let id = {
             let id = self.inner.next_id;
             let uuid = Uuid::new_v4();
@@ -450,7 +366,7 @@ impl Config {
 
             self.inner.accounts.push(AccountConfig {
                 id,
-                dir: target_dir.into(),
+                dir: target_dir,
                 uuid,
             });
             self.inner.next_id += 1;
@@ -531,10 +447,10 @@ impl AccountConfig {
 mod tests {
     use super::*;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_account_new_open() {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts1").into();
+        let p: PathBuf = dir.path().join("accounts1");
 
         let mut accounts1 = Accounts::new(p.clone()).await.unwrap();
         accounts1.add_account().await.unwrap();
@@ -549,10 +465,10 @@ mod tests {
         assert_eq!(accounts1.accounts.len(), accounts2.accounts.len());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_account_new_add_remove() {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone()).await.unwrap();
         assert_eq!(accounts.accounts.len(), 0);
@@ -576,10 +492,10 @@ mod tests {
         assert_eq!(accounts.accounts.len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_accounts_remove_last() -> Result<()> {
         let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone()).await?;
         assert!(accounts.get_selected_account().await.is_none());
@@ -597,17 +513,19 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_migrate_account() {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone()).await.unwrap();
         assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
-        let extern_dbfile: PathBuf = dir.path().join("other").into();
-        let ctx = Context::new(extern_dbfile.clone(), 0).await.unwrap();
+        let extern_dbfile: PathBuf = dir.path().join("other");
+        let ctx = Context::new(&extern_dbfile, 0, Events::new())
+            .await
+            .unwrap();
         ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
             .await
             .unwrap();
@@ -632,10 +550,10 @@ mod tests {
     }
 
     /// Tests that accounts are sorted by ID.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_accounts_sorted() {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone()).await.unwrap();
 
@@ -650,10 +568,10 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_accounts_ids_unique_increasing_and_persisted() -> Result<()> {
         let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
         let dummy_accounts = 10;
 
         let (id0, id1, id2) = {
@@ -732,10 +650,10 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_no_accounts_event_emitter() -> Result<()> {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let accounts = Accounts::new(p.clone()).await?;
 
@@ -743,25 +661,25 @@ mod tests {
         assert_eq!(accounts.accounts.len(), 0);
 
         // Create event emitter.
-        let mut event_emitter = accounts.get_event_emitter().await;
+        let event_emitter = accounts.get_event_emitter().await;
 
         // Test that event emitter does not return `None` immediately.
         let duration = std::time::Duration::from_millis(1);
-        assert!(async_std::future::timeout(duration, event_emitter.recv())
+        assert!(tokio::time::timeout(duration, event_emitter.recv())
             .await
             .is_err());
 
         // When account manager is dropped, event emitter is exhausted.
         drop(accounts);
-        assert_eq!(event_emitter.recv().await?, None);
+        assert_eq!(event_emitter.recv().await, None);
 
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_encrypted_account() -> Result<()> {
         let dir = tempfile::tempdir().context("failed to create tempdir")?;
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone())
             .await

src/blob.rs

@@ -4,15 +4,13 @@ use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
 use std::io::Cursor;
+use std::path::{Path, PathBuf};
 
-use async_std::path::{Path, PathBuf};
-use async_std::prelude::*;
-use async_std::{fs, io};
-
-use anyhow::{format_err, Context as _, Error};
+use anyhow::{format_err, Context as _, Error, Result};
 use image::{DynamicImage, ImageFormat};
 use num_traits::FromPrimitive;
-use thiserror::Error;
+use tokio::io::AsyncWriteExt;
+use tokio::{fs, io};
 
 use crate::config::Config;
 use crate::constants::{
@@ -44,31 +42,15 @@ impl<'a> BlobObject<'a> {
     /// name, followed by a random number and followed by a possible
     /// extension.  The `data` will be written into the file without
     /// race-conditions.
-    ///
-    /// # Errors
-    ///
-    /// [BlobError::CreateFailure] is used when the file could not
-    /// be created.  You can expect [BlobError.cause] to contain an
-    /// underlying error.
-    ///
-    /// [BlobError::WriteFailure] is used when the file could not
-    /// be written to.  You can expect [BlobError.cause] to contain an
-    /// underlying error.
     pub async fn create(
         context: &'a Context,
         suggested_name: &str,
         data: &[u8],
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+    ) -> Result<BlobObject<'a>> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name);
         let (name, mut file) = BlobObject::create_new_file(context, blobdir, &stem, &ext).await?;
-        file.write_all(data)
-            .await
-            .map_err(|err| BlobError::WriteFailure {
-                blobdir: blobdir.to_path_buf(),
-                blobname: name.clone(),
-                cause: err.into(),
-            })?;
+        file.write_all(data).await.context("file write failure")?;
 
         // workaround a bug in async-std
         // (the executor does not handle blocking operation in Drop correctly,
@@ -89,7 +71,7 @@ impl<'a> BlobObject<'a> {
         dir: &Path,
         stem: &str,
         ext: &str,
-    ) -> Result<(String, fs::File), BlobError> {
+    ) -> Result<(String, fs::File)> {
         const MAX_ATTEMPT: u32 = 16;
         let mut attempt = 0;
         let mut name = format!("{}{}", stem, ext);
@@ -105,12 +87,8 @@ impl<'a> BlobObject<'a> {
                 Ok(file) => return Ok((name, file)),
                 Err(err) => {
                     if attempt >= MAX_ATTEMPT {
-                        return Err(BlobError::CreateFailure {
-                            blobdir: dir.to_path_buf(),
-                            blobname: name,
-                            cause: err,
-                        });
-                    } else if attempt == 1 && !dir.exists().await {
+                        return Err(err).context("failed to create file");
+                    } else if attempt == 1 && !dir.exists() {
                         fs::create_dir_all(dir).await.ok_or_log(context);
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
@@ -126,40 +104,19 @@ impl<'a> BlobObject<'a> {
     /// but also copies an existing file into it.  This is done in a
     /// in way which avoids race-conditions when multiple files are
     /// concurrently created.
-    ///
-    /// # Errors
-    ///
-    /// In addition to the errors in [BlobObject::create] the
-    /// [BlobError::CopyFailure] is used when the data can not be
-    /// copied.
-    pub async fn create_and_copy(
-        context: &'a Context,
-        src: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+    pub async fn create_and_copy(context: &'a Context, src: &Path) -> Result<BlobObject<'a>> {
         let mut src_file = fs::File::open(src)
             .await
-            .map_err(|err| BlobError::CopyFailure {
-                blobdir: context.get_blobdir().to_path_buf(),
-                blobname: String::from(""),
-                src: src.to_path_buf(),
-                cause: err,
-            })?;
+            .with_context(|| format!("failed to open file {}", src.display()))?;
         let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
         let (name, mut dst_file) =
             BlobObject::create_new_file(context, context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
         if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
-            {
-                // Attempt to remove the failed file, swallow errors resulting from that.
-                let path = context.get_blobdir().join(&name_for_err);
-                fs::remove_file(path).await.ok();
-            }
-            return Err(BlobError::CopyFailure {
-                blobdir: context.get_blobdir().to_path_buf(),
-                blobname: name_for_err,
-                src: src.to_path_buf(),
-                cause: err,
-            });
+            // Attempt to remove the failed file, swallow errors resulting from that.
+            let path = context.get_blobdir().join(&name_for_err);
+            fs::remove_file(path).await.ok();
+            return Err(err).context("failed to copy file");
         }
 
         // workaround, see create() for details
@@ -184,16 +141,7 @@ impl<'a> BlobObject<'a> {
     ///
     /// Paths into the blob directory may be either defined by an absolute path
     /// or by the relative prefix `$BLOBDIR`.
-    ///
-    /// # Errors
-    ///
-    /// This merely delegates to the [BlobObject::create_and_copy] and
-    /// the [BlobObject::from_path] methods.  See those for possible
-    /// errors.
-    pub async fn new_from_path(
-        context: &'a Context,
-        src: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+    pub async fn new_from_path(context: &'a Context, src: &Path) -> Result<BlobObject<'a>> {
         if src.starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
         } else if src.starts_with("$BLOBDIR/") {
@@ -209,32 +157,14 @@ impl<'a> BlobObject<'a> {
     /// must use a valid blob name.  That is after sanitisation the
     /// name must still be the same, that means it must be valid UTF-8
     /// and not have any special characters in it.
-    ///
-    /// # Errors
-    ///
-    /// [BlobError::WrongBlobdir] is used if the path is not in
-    /// the blob directory.
-    ///
-    /// [BlobError::WrongName] is used if the file name does not
-    /// remain identical after sanitisation.
-    pub fn from_path(
-        context: &'a Context,
-        path: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let rel_path =
-            path.strip_prefix(context.get_blobdir())
-                .map_err(|_| BlobError::WrongBlobdir {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    src: path.to_path_buf(),
-                })?;
+    pub fn from_path(context: &'a Context, path: &Path) -> Result<BlobObject<'a>> {
+        let rel_path = path
+            .strip_prefix(context.get_blobdir())
+            .context("wrong blobdir")?;
         if !BlobObject::is_acceptible_blob_name(rel_path) {
-            return Err(BlobError::WrongName {
-                blobname: path.to_path_buf(),
-            });
+            return Err(format_err!("wrong name"));
         }
-        let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
-            blobname: path.to_path_buf(),
-        })?;
+        let name = rel_path.to_str().context("wrong name")?;
         BlobObject::from_name(context, name.to_string())
     }
 
@@ -244,24 +174,13 @@ impl<'a> BlobObject<'a> {
     /// prefixed, as returned by [BlobObject::as_name].  This is how
     /// you want to create a [BlobObject] for a filename read from the
     /// database.
-    ///
-    /// # Errors
-    ///
-    /// [BlobError::WrongName] is used if the name is not a valid
-    /// blobname, i.e. if [BlobObject::sanitise_name] does modify the
-    /// provided name.
-    pub fn from_name(
-        context: &'a Context,
-        name: String,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+    pub fn from_name(context: &'a Context, name: String) -> Result<BlobObject<'a>> {
         let name: String = match name.starts_with("$BLOBDIR/") {
             true => name.splitn(2, '/').last().unwrap().to_string(),
             false => name,
         };
         if !BlobObject::is_acceptible_blob_name(&name) {
-            return Err(BlobError::WrongName {
-                blobname: PathBuf::from(name),
-            });
+            return Err(format_err!("not an acceptable blob name: {}", &name));
         }
         Ok(BlobObject {
             blobdir: context.get_blobdir(),
@@ -394,7 +313,7 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<()> {
         let blob_abs = self.to_abs_path();
 
         let img_wh =
@@ -416,7 +335,7 @@ impl<'a> BlobObject<'a> {
         Ok(())
     }
 
-    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_image_size(&self, context: &Context) -> Result<()> {
         let blob_abs = self.to_abs_path();
         if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
             != Some((Viewtype::Image, "image/jpeg"))
@@ -439,8 +358,7 @@ impl<'a> BlobObject<'a> {
         {
             return Err(format_err!(
                 "Internal error: recode_to_size(..., None) shouldn't change the name of the image"
-            )
-            .into());
+            ));
         }
         Ok(())
     }
@@ -451,118 +369,82 @@ impl<'a> BlobObject<'a> {
         mut blob_abs: PathBuf,
         mut img_wh: u32,
         max_bytes: Option<usize>,
-    ) -> Result<Option<String>, BlobError> {
-        let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
-        })?;
-        let orientation = self.get_exif_orientation(context);
-        let mut encoded = Vec::new();
-        let mut changed_name = None;
+    ) -> Result<Option<String>> {
+        tokio::task::block_in_place(move || {
+            let mut img = image::open(&blob_abs).context("image recode failure")?;
+            let orientation = self.get_exif_orientation(context);
+            let mut encoded = Vec::new();
+            let mut changed_name = None;
 
-        fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
-            encoded.clear();
-            let mut buf = Cursor::new(encoded);
-            img.write_to(&mut buf, image::ImageFormat::Jpeg)?;
-            Ok(())
-        }
-        fn encoded_img_exceeds_bytes(
-            context: &Context,
-            img: &DynamicImage,
-            max_bytes: Option<usize>,
-            encoded: &mut Vec<u8>,
-        ) -> anyhow::Result<bool> {
-            if let Some(max_bytes) = max_bytes {
-                encode_img(img, encoded)?;
-                if encoded.len() > max_bytes {
-                    info!(
-                        context,
-                        "image size {}B ({}x{}px) exceeds {}B, need to scale down",
-                        encoded.len(),
-                        img.width(),
-                        img.height(),
-                        max_bytes,
-                    );
-                    return Ok(true);
-                }
-            }
-            Ok(false)
-        }
-        let exceeds_width = img.width() > img_wh || img.height() > img_wh;
+            let exceeds_width = img.width() > img_wh || img.height() > img_wh;
 
-        let do_scale =
-            exceeds_width || encoded_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
-        let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
+            let do_scale =
+                exceeds_width || encoded_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
+            let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
 
-        if do_scale || do_rotate {
-            if do_rotate {
-                img = match orientation {
-                    Ok(90) => img.rotate90(),
-                    Ok(180) => img.rotate180(),
-                    Ok(270) => img.rotate270(),
-                    _ => img,
-                }
-            }
-
-            if do_scale {
-                if !exceeds_width {
-                    // The image is already smaller than img_wh, but exceeds max_bytes
-                    // We can directly start with trying to scale down to 2/3 of its current width
-                    img_wh = max(img.width(), img.height()) * 2 / 3
-                }
-
-                loop {
-                    let new_img = img.thumbnail(img_wh, img_wh);
-
-                    if encoded_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
-                        if img_wh < 20 {
-                            return Err(format_err!(
-                                "Failed to scale image to below {}B",
-                                max_bytes.unwrap_or_default()
-                            )
-                            .into());
-                        }
-
-                        img_wh = img_wh * 2 / 3;
-                    } else {
-                        if encoded.is_empty() {
-                            encode_img(&new_img, &mut encoded)?;
-                        }
-
-                        info!(
-                            context,
-                            "Final scaled-down image size: {}B ({}px)",
-                            encoded.len(),
-                            img_wh
-                        );
-                        break;
+            if do_scale || do_rotate {
+                if do_rotate {
+                    img = match orientation {
+                        Ok(90) => img.rotate90(),
+                        Ok(180) => img.rotate180(),
+                        Ok(270) => img.rotate270(),
+                        _ => img,
                     }
                 }
+
+                if do_scale {
+                    if !exceeds_width {
+                        // The image is already smaller than img_wh, but exceeds max_bytes
+                        // We can directly start with trying to scale down to 2/3 of its current width
+                        img_wh = max(img.width(), img.height()) * 2 / 3
+                    }
+
+                    loop {
+                        let new_img = img.thumbnail(img_wh, img_wh);
+
+                        if encoded_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
+                            if img_wh < 20 {
+                                return Err(format_err!(
+                                    "Failed to scale image to below {}B",
+                                    max_bytes.unwrap_or_default()
+                                ));
+                            }
+
+                            img_wh = img_wh * 2 / 3;
+                        } else {
+                            if encoded.is_empty() {
+                                encode_img(&new_img, &mut encoded)?;
+                            }
+
+                            info!(
+                                context,
+                                "Final scaled-down image size: {}B ({}px)",
+                                encoded.len(),
+                                img_wh
+                            );
+                            break;
+                        }
+                    }
+                }
+
+                // The file format is JPEG now, we may have to change the file extension
+                if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
+                    blob_abs = blob_abs.with_extension("jpg");
+                    let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
+                    let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
+                    changed_name = Some(format!("$BLOBDIR/{}", file_name));
+                }
+
+                if encoded.is_empty() {
+                    encode_img(&img, &mut encoded)?;
+                }
+
+                std::fs::write(&blob_abs, &encoded)
+                    .context("failed to write recoded blob to file")?;
             }
 
-            // The file format is JPEG now, we may have to change the file extension
-            if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
-                blob_abs = blob_abs.with_extension("jpg");
-                let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
-                let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
-                changed_name = Some(format!("$BLOBDIR/{}", file_name));
-            }
-
-            if encoded.is_empty() {
-                encode_img(&img, &mut encoded)?;
-            }
-
-            fs::write(&blob_abs, &encoded)
-                .await
-                .map_err(|err| BlobError::WriteFailure {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-                    cause: err.into(),
-                })?;
-        }
-
-        Ok(changed_name)
+            Ok(changed_name)
+        })
     }
 
     pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
@@ -590,44 +472,33 @@ impl<'a> fmt::Display for BlobObject<'a> {
     }
 }
 
-/// Errors for the [BlobObject].
-#[derive(Debug, Error)]
-pub enum BlobError {
-    #[error("Failed to create blob {blobname} in {}", .blobdir.display())]
-    CreateFailure {
-        blobdir: PathBuf,
-        blobname: String,
-        #[source]
-        cause: std::io::Error,
-    },
-    #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())]
-    WriteFailure {
-        blobdir: PathBuf,
-        blobname: String,
-        #[source]
-        cause: anyhow::Error,
-    },
-    #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
-    CopyFailure {
-        blobdir: PathBuf,
-        blobname: String,
-        src: PathBuf,
-        #[source]
-        cause: std::io::Error,
-    },
-    #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())]
-    RecodeFailure {
-        blobdir: PathBuf,
-        blobname: String,
-        #[source]
-        cause: image::ImageError,
-    },
-    #[error("File path {} is not in {}", .src.display(), .blobdir.display())]
-    WrongBlobdir { blobdir: PathBuf, src: PathBuf },
-    #[error("Blob has a badname {}", .blobname.display())]
-    WrongName { blobname: PathBuf },
-    #[error("{0}")]
-    Other(#[from] anyhow::Error),
+fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
+    encoded.clear();
+    let mut buf = Cursor::new(encoded);
+    img.write_to(&mut buf, image::ImageFormat::Jpeg)?;
+    Ok(())
+}
+fn encoded_img_exceeds_bytes(
+    context: &Context,
+    img: &DynamicImage,
+    max_bytes: Option<usize>,
+    encoded: &mut Vec<u8>,
+) -> anyhow::Result<bool> {
+    if let Some(max_bytes) = max_bytes {
+        encode_img(img, encoded)?;
+        if encoded.len() > max_bytes {
+            info!(
+                context,
+                "image size {}B ({}x{}px) exceeds {}B, need to scale down",
+                encoded.len(),
+                img.width(),
+                img.height(),
+                max_bytes,
+            );
+            return Ok(true);
+        }
+    }
+    Ok(false)
 }
 
 #[cfg(test)]
@@ -643,7 +514,16 @@ mod tests {
 
     use super::*;
 
-    #[async_std::test]
+    fn check_image_size(path: impl AsRef<Path>, width: u32, height: u32) -> image::DynamicImage {
+        tokio::task::block_in_place(move || {
+            let img = image::open(path).expect("failed to open image");
+            assert_eq!(img.width(), width, "invalid width");
+            assert_eq!(img.height(), height, "invalid height");
+            img
+        })
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo", b"hello").await.unwrap();
@@ -654,28 +534,28 @@ mod tests {
         assert_eq!(blob.to_abs_path(), t.get_blobdir().join("foo"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_lowercase_ext() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo.TXT", b"hello").await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_as_file_name() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_as_rel_path() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_suffix() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
@@ -684,16 +564,16 @@ mod tests {
         assert_eq!(blob.suffix(), None);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_dup() {
         let t = TestContext::new().await;
         BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         let foo_path = t.get_blobdir().join("foo.txt");
-        assert!(foo_path.exists().await);
+        assert!(foo_path.exists());
         BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
         let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
-        while let Some(dirent) = dir.next().await {
-            let fname = dirent.unwrap().file_name();
+        while let Ok(Some(dirent)) = dir.next_entry().await {
+            let fname = dirent.file_name();
             if fname == foo_path.file_name().unwrap() {
                 assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
             } else {
@@ -704,20 +584,20 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_double_ext_preserved() {
         let t = TestContext::new().await;
         BlobObject::create(&t, "foo.tar.gz", b"hello")
             .await
             .unwrap();
         let foo_path = t.get_blobdir().join("foo.tar.gz");
-        assert!(foo_path.exists().await);
+        assert!(foo_path.exists());
         BlobObject::create(&t, "foo.tar.gz", b"world")
             .await
             .unwrap();
         let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
-        while let Some(dirent) = dir.next().await {
-            let fname = dirent.unwrap().file_name();
+        while let Ok(Some(dirent)) = dir.next_entry().await {
+            let fname = dirent.file_name();
             if fname == foo_path.file_name().unwrap() {
                 assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
             } else {
@@ -729,7 +609,7 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_long_names() {
         let t = TestContext::new().await;
         let s = "1".repeat(150);
@@ -738,7 +618,7 @@ mod tests {
         assert!(blobname.len() < 128);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_and_copy() {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
@@ -753,10 +633,10 @@ mod tests {
             .await
             .is_err());
         let whoops = t.get_blobdir().join("whoops");
-        assert!(!whoops.exists().await);
+        assert!(!whoops.exists());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_from_path() {
         let t = TestContext::new().await;
 
@@ -776,7 +656,7 @@ mod tests {
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
     }
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_from_name_long() {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
@@ -839,7 +719,7 @@ mod tests {
         assert!(!stem.contains('?'));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_selfavatar_outside_blobdir() {
         let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.jpg");
@@ -851,22 +731,17 @@ mod tests {
             .await
             .unwrap();
         let avatar_blob = t.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists().await);
+        assert!(!avatar_blob.exists());
         t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
+        assert!(avatar_blob.exists());
+        assert!(tokio::fs::metadata(&avatar_blob).await.unwrap().len() < avatar_bytes.len() as u64);
         let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
 
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(&avatar_blob).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+        check_image_size(avatar_src, 1000, 1000);
+        check_image_size(&avatar_blob, BALANCED_AVATAR_SIZE, BALANCED_AVATAR_SIZE);
 
         async fn file_size(path_buf: &PathBuf) -> u64 {
             let file = File::open(path_buf).await.unwrap();
@@ -880,12 +755,14 @@ mod tests {
             .unwrap();
         assert!(file_size(&avatar_blob).await <= 3000);
         assert!(file_size(&avatar_blob).await > 2000);
-        let img = image::open(&avatar_blob).unwrap();
-        assert!(img.width() > 130);
-        assert_eq!(img.width(), img.height());
+        tokio::task::block_in_place(move || {
+            let img = image::open(&avatar_blob).unwrap();
+            assert!(img.width() > 130);
+            assert_eq!(img.width(), img.height());
+        });
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_selfavatar_in_blobdir() {
         let t = TestContext::new().await;
         let avatar_src = t.get_blobdir().join("avatar.png");
@@ -896,9 +773,7 @@ mod tests {
             .await
             .unwrap();
 
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
+        check_image_size(&avatar_src, 900, 900);
 
         t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
@@ -909,12 +784,10 @@ mod tests {
             avatar_src.with_extension("jpg").to_str().unwrap()
         );
 
-        let img = image::open(avatar_cfg).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+        check_image_size(avatar_cfg, BALANCED_AVATAR_SIZE, BALANCED_AVATAR_SIZE);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_selfavatar_copy_without_recode() {
         let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.png");
@@ -926,20 +799,20 @@ mod tests {
             .await
             .unwrap();
         let avatar_blob = t.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists().await);
+        assert!(!avatar_blob.exists());
         t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
-        assert!(avatar_blob.exists().await);
+        assert!(avatar_blob.exists());
         assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
+            tokio::fs::metadata(&avatar_blob).await.unwrap().len(),
             avatar_bytes.len() as u64
         );
         let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_1() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
@@ -959,7 +832,7 @@ mod tests {
         .unwrap();
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
@@ -985,7 +858,7 @@ mod tests {
         // Do this in parallel to speed up the test a bit
         // (it still takes very long though)
         let bytes2 = bytes.clone();
-        let join_handle = async_std::task::spawn(async move {
+        let join_handle = tokio::task::spawn(async move {
             let img_rotated = send_image_check_mediaquality(
                 Some("0"),
                 &bytes2,
@@ -1013,10 +886,10 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
-        join_handle.await;
+        join_handle.await.unwrap();
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_3() {
         let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
@@ -1064,10 +937,10 @@ mod tests {
             .await?;
         let file = alice.get_blobdir().join("file.jpg");
 
-        File::create(&file).await?.write_all(bytes).await?;
-        let img = image::open(&file)?;
-        assert_eq!(img.width(), original_width);
-        assert_eq!(img.height(), original_height);
+        fs::write(&file, &bytes)
+            .await
+            .context("failed to write file")?;
+        check_image_size(&file, original_width, original_height);
 
         let blob = BlobObject::new_from_path(&alice, &file).await?;
         assert_eq!(blob.get_exif_orientation(&alice).unwrap_or(0), orientation);
@@ -1079,9 +952,11 @@ mod tests {
         let alice_msg = alice.get_last_msg().await;
         assert_eq!(alice_msg.get_width() as u32, compressed_width);
         assert_eq!(alice_msg.get_height() as u32, compressed_height);
-        let img = image::open(alice_msg.get_file(&alice).unwrap())?;
-        assert_eq!(img.width() as u32, compressed_width);
-        assert_eq!(img.height() as u32, compressed_height);
+        check_image_size(
+            alice_msg.get_file(&alice).unwrap(),
+            compressed_width,
+            compressed_height,
+        );
 
         let bob_msg = bob.recv_msg(&sent).await;
         assert_eq!(bob_msg.get_width() as u32, compressed_width);
@@ -1091,13 +966,11 @@ mod tests {
         let blob = BlobObject::new_from_path(&bob, &file).await?;
         assert_eq!(blob.get_exif_orientation(&bob).unwrap_or(0), 0);
 
-        let img = image::open(file)?;
-        assert_eq!(img.width() as u32, compressed_width);
-        assert_eq!(img.height() as u32, compressed_height);
+        let img = check_image_size(file, compressed_width, compressed_height);
         Ok(img)
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_increation_in_blobdir() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
@@ -1116,7 +989,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_increation_not_blobdir() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;

src/chat.rs

@@ -2,16 +2,16 @@
 
 use std::collections::HashMap;
 use std::convert::{TryFrom, TryInto};
+use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use anyhow::{bail, ensure, Context as _, Result};
-use async_std::path::{Path, PathBuf};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
 
 use crate::aheader::EncryptPreference;
-use crate::blob::{BlobError, BlobObject};
+use crate::blob::BlobObject;
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{
@@ -438,6 +438,7 @@ impl ChatId {
                 dc_create_smeared_timestamp(context).await,
                 None,
                 None,
+                None,
             )
             .await?;
         }
@@ -1390,11 +1391,7 @@ impl Chat {
             } else {
                 msg.param.get(Param::SendHtml).map(|s| s.to_string())
             };
-            if let Some(html) = html {
-                Some(new_html_mimepart(html).await.build().as_string())
-            } else {
-                None
-            }
+            html.map(|html| new_html_mimepart(html).build().as_string())
         } else {
             None
         };
@@ -1954,7 +1951,7 @@ pub async fn send_msg(context: &Context, chat_id: ChatId, msg: &mut Message) ->
                 }
             }
             msg.param.remove(Param::PrepForwards);
-            msg.update_param(context).await;
+            msg.update_param(context).await?;
         }
         return send_msg_inner(context, chat_id, msg).await;
     }
@@ -2071,7 +2068,7 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<
     let rendered_msg = match mimefactory.render(context).await {
         Ok(res) => Ok(res),
         Err(err) => {
-            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+            message::set_msg_failed(context, msg_id, &err.to_string()).await;
             Err(err)
         }
     }?;
@@ -2081,7 +2078,7 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<
         message::set_msg_failed(
             context,
             msg_id,
-            Some("End-to-end-encryption unavailable unexpectedly."),
+            "End-to-end-encryption unavailable unexpectedly.",
         )
         .await;
         bail!(
@@ -2123,7 +2120,7 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<
 
     if rendered_msg.is_encrypted && !needs_encryption {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
-        msg.update_param(context).await;
+        msg.update_param(context).await?;
     }
 
     ensure!(!recipients.is_empty(), "no recipients for smtp job set");
@@ -2131,7 +2128,7 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<
     let recipients = recipients.join(" ");
 
     msg.subject = rendered_msg.subject.clone();
-    msg.update_subject(context).await;
+    msg.update_subject(context).await?;
 
     let row_id = context
         .sql
@@ -3025,15 +3022,8 @@ pub async fn set_chat_profile_image(
         msg.param.remove(Param::Arg);
         msg.text = Some(stock_str::msg_grp_img_deleted(context, ContactId::SELF).await);
     } else {
-        let mut image_blob = match BlobObject::from_path(context, Path::new(new_image.as_ref())) {
-            Ok(blob) => Ok(blob),
-            Err(err) => match err {
-                BlobError::WrongBlobdir { .. } => {
-                    BlobObject::create_and_copy(context, Path::new(new_image.as_ref())).await
-                }
-                _ => Err(err),
-            },
-        }?;
+        let mut image_blob =
+            BlobObject::new_from_path(context, Path::new(new_image.as_ref())).await?;
         image_blob.recode_to_avatar_size(context).await?;
         chat.param.set(Param::ProfileImage, image_blob.as_name());
         msg.param.set(Param::Arg, image_blob.as_name());
@@ -3120,7 +3110,7 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId)
                         .set(Param::PrepForwards, new_msg_id.to_u32().to_string());
                 }
 
-                msg.update_param(context).await;
+                msg.update_param(context).await?;
                 msg.param = save_param;
             } else {
                 msg.state = MessageState::OutPending;
@@ -3168,7 +3158,7 @@ pub async fn resend_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
         for mut msg in msgs {
             if msg.get_showpadlock() && !chat.is_protected() {
                 msg.param.remove(Param::GuaranteeE2ee);
-                msg.update_param(context).await;
+                msg.update_param(context).await?;
             }
             match msg.get_state() {
                 MessageState::OutFailed | MessageState::OutDelivered | MessageState::OutMdnRcvd => {
@@ -3380,6 +3370,7 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul
 /// Adds an informational message to chat.
 ///
 /// For example, it can be a message showing that a member was added to a group.
+#[allow(clippy::too_many_arguments)]
 pub(crate) async fn add_info_msg_with_cmd(
     context: &Context,
     chat_id: ChatId,
@@ -3389,6 +3380,7 @@ pub(crate) async fn add_info_msg_with_cmd(
     // Timestamp to show to the user (if this is None, `timestamp_sort` will be shown to the user)
     timestamp_sent_rcvd: Option<i64>,
     parent: Option<&Message>,
+    from_id: Option<ContactId>,
 ) -> Result<MsgId> {
     let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
     let ephemeral_timer = chat_id.get_ephemeral_timer(context).await?;
@@ -3404,7 +3396,7 @@ pub(crate) async fn add_info_msg_with_cmd(
         VALUES (?,?,?, ?,?,?,?,?, ?,?,?, ?,?);",
         paramsv![
             chat_id,
-            ContactId::INFO,
+            from_id.unwrap_or(ContactId::INFO),
             ContactId::INFO,
             timestamp_sort,
             timestamp_sent_rcvd.unwrap_or(0),
@@ -3440,10 +3432,29 @@ pub(crate) async fn add_info_msg(
         timestamp,
         None,
         None,
+        None,
     )
     .await
 }
 
+pub(crate) async fn update_msg_text_and_timestamp(
+    context: &Context,
+    chat_id: ChatId,
+    msg_id: MsgId,
+    text: &str,
+    timestamp: i64,
+) -> Result<()> {
+    context
+        .sql
+        .execute(
+            "UPDATE msgs SET txt=?, timestamp=? WHERE id=?;",
+            paramsv![text, timestamp, msg_id],
+        )
+        .await?;
+    context.emit_msgs_changed(chat_id, msg_id);
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -3453,10 +3464,10 @@ mod tests {
     use crate::contact::Contact;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils::TestContext;
-    use async_std::fs::File;
-    use async_std::prelude::*;
+    use tokio::fs::File;
+    use tokio::io::AsyncWriteExt;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_chat_info() {
         let t = TestContext::new().await;
         let chat = t.create_chat_with_contact("bob", "bob@example.com").await;
@@ -3487,7 +3498,7 @@ mod tests {
         assert_eq!(info, loaded);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_draft_no_draft() {
         let t = TestContext::new().await;
         let chat = t.get_self_chat().await;
@@ -3495,14 +3506,14 @@ mod tests {
         assert!(draft.is_none());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_draft_special_chat_id() {
         let t = TestContext::new().await;
         let draft = DC_CHAT_ID_LAST_SPECIAL.get_draft(&t).await.unwrap();
         assert!(draft.is_none());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_draft_no_chat() {
         // This is a weird case, maybe this should be an error but we
         // do not get this info from the database currently.
@@ -3511,7 +3522,7 @@ mod tests {
         assert!(draft.is_none());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_draft() {
         let t = TestContext::new().await;
         let chat_id = &t.get_self_chat().await.id;
@@ -3525,7 +3536,7 @@ mod tests {
         assert_eq!(msg_text, draft_text);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_delete_draft() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
@@ -3546,7 +3557,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_forwarding_draft_failing() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = &t.get_self_chat().await.id;
@@ -3560,7 +3571,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_draft_stable_ids() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = &t.get_self_chat().await.id;
@@ -3605,7 +3616,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_change_quotes_on_reused_message_object() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "chat").await?;
@@ -3657,7 +3668,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_contact_to_chat_ex_add_self() {
         // Adding self to a contact should succeed, even though it's pointless.
         let t = TestContext::new_alice().await;
@@ -3670,7 +3681,7 @@ mod tests {
         assert_eq!(added, false);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_modify_chat_multi_device() -> Result<()> {
         let a1 = TestContext::new_alice().await;
         let a2 = TestContext::new_alice().await;
@@ -3745,7 +3756,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_modify_chat_disordered() -> Result<()> {
         // Alice creates a group with Bob, Claire and Daisy and then removes Claire and Daisy
         // (sleep() is needed as otherwise smeared time from Alice looks to Bob like messages from the future which are all set to "now" then)
@@ -3759,47 +3770,47 @@ mod tests {
         send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?;
 
         add_contact_to_chat(&alice, alice_chat_id, bob_id).await?;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
         let add1 = alice.pop_sent_msg().await;
 
         add_contact_to_chat(&alice, alice_chat_id, claire_id).await?;
         let add2 = alice.pop_sent_msg().await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         add_contact_to_chat(&alice, alice_chat_id, daisy_id).await?;
         let add3 = alice.pop_sent_msg().await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 4);
 
         remove_contact_from_chat(&alice, alice_chat_id, claire_id).await?;
         let remove1 = alice.pop_sent_msg().await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         remove_contact_from_chat(&alice, alice_chat_id, daisy_id).await?;
         let remove2 = alice.pop_sent_msg().await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 2);
 
         // Bob receives the add and deletion messages out of order
         let bob = TestContext::new_bob().await;
         bob.recv_msg(&add1).await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         bob.recv_msg(&add3).await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         let bob_chat_id = bob.recv_msg(&add2).await.chat_id;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 4);
 
         bob.recv_msg(&remove2).await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         bob.recv_msg(&remove1).await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 2);
 
@@ -3807,7 +3818,7 @@ mod tests {
     }
 
     /// Test that group updates are robust to lost messages and eventual out of order arrival.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_modify_chat_lost() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -3822,11 +3833,11 @@ mod tests {
 
         send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?;
         let add = alice.pop_sent_msg().await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         remove_contact_from_chat(&alice, alice_chat_id, claire_id).await?;
         let remove1 = alice.pop_sent_msg().await;
-        async_std::task::sleep(std::time::Duration::from_millis(1100)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
 
         remove_contact_from_chat(&alice, alice_chat_id, daisy_id).await?;
         let remove2 = alice.pop_sent_msg().await;
@@ -3849,7 +3860,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_leave_group() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -3878,7 +3889,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_remove_contact_for_single() {
         let ctx = TestContext::new_alice().await;
         let bob = Contact::create(&ctx, "", "bob@f.br").await.unwrap();
@@ -3902,7 +3913,7 @@ mod tests {
         assert_eq!(get_chat_contacts(&ctx, chat.id).await.unwrap().len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_self_talk() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat = &t.get_self_chat().await;
@@ -3933,7 +3944,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_device_msg_unlabelled() {
         let t = TestContext::new().await;
 
@@ -3968,7 +3979,7 @@ mod tests {
         assert_eq!(msg2.chat_id.get_msg_cnt(&t).await.unwrap(), 2);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_device_msg_labelled() -> Result<()> {
         let t = TestContext::new().await;
 
@@ -4018,7 +4029,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_device_msg_label_only() {
         let t = TestContext::new().await;
         let res = add_device_msg(&t, Some(""), None).await;
@@ -4038,7 +4049,7 @@ mod tests {
         assert!(!msg_id.as_ref().unwrap().is_unset());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_was_device_msg_ever_added() {
         let t = TestContext::new().await;
         add_device_msg(&t, Some("some-label"), None).await.ok();
@@ -4058,7 +4069,7 @@ mod tests {
         assert!(was_device_msg_ever_added(&t, "").await.is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_delete_device_chat() {
         let t = TestContext::new().await;
 
@@ -4078,7 +4089,7 @@ mod tests {
         assert_eq!(chatlist_len(&t, 0).await, 0)
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_device_chat_cannot_sent() {
         let t = TestContext::new().await;
         t.update_device_chats().await.unwrap();
@@ -4095,7 +4106,7 @@ mod tests {
         assert!(forward_msgs(&t, &[msg_id], device_chat_id).await.is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_delete_and_reset_all_device_msgs() {
         let t = TestContext::new().await;
         let mut msg = Message::new(Viewtype::Text);
@@ -4127,7 +4138,7 @@ mod tests {
             .len()
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_archive() {
         // create two chats
         let t = TestContext::new().await;
@@ -4230,7 +4241,7 @@ mod tests {
         assert_eq!(chatlist_len(&t, DC_GCL_ARCHIVED_ONLY).await, 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_unarchive_if_muted() -> Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -4324,7 +4335,7 @@ mod tests {
         result
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_pinned() {
         let t = TestContext::new().await;
 
@@ -4336,9 +4347,9 @@ mod tests {
             .await
             .unwrap()
             .chat_id;
-        async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
         let chat_id2 = t.get_self_chat().await.id;
-        async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
+        tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
         let chat_id3 = create_group_chat(&t, ProtectionStatus::Unprotected, "foo")
             .await
             .unwrap();
@@ -4381,7 +4392,7 @@ mod tests {
         assert_eq!(chatlist, vec![chat_id3, chat_id2, chat_id1]);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_chat_name() {
         let t = TestContext::new().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "foo")
@@ -4399,7 +4410,7 @@ mod tests {
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_same_chat_twice() {
         let context = TestContext::new().await;
         let contact1 = Contact::create(&context.ctx, "bob", "bob@mail.de")
@@ -4422,7 +4433,7 @@ mod tests {
         assert_eq!(chat2.name, chat.name);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_shall_attach_selfavatar() -> Result<()> {
         let t = TestContext::new().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "foo").await?;
@@ -4440,7 +4451,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_mute_duration() {
         let t = TestContext::new().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "foo")
@@ -4491,7 +4502,7 @@ mod tests {
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_info_msg() -> Result<()> {
         let t = TestContext::new().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "foo").await?;
@@ -4508,7 +4519,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_info_msg_with_cmd() -> Result<()> {
         let t = TestContext::new().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "foo").await?;
@@ -4520,6 +4531,7 @@ mod tests {
             10000,
             None,
             None,
+            None,
         )
         .await?;
 
@@ -4537,7 +4549,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_protection() {
         let t = TestContext::new_alice().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "foo")
@@ -4605,7 +4617,7 @@ mod tests {
         assert_eq!(msg.get_state(), MessageState::OutDelivered); // as bcc-self is disabled and there is nobody else in the chat
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_lookup_by_contact_id() {
         let ctx = TestContext::new_alice().await;
 
@@ -4648,7 +4660,7 @@ mod tests {
         assert!(found.is_none());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_lookup_self_by_contact_id() {
         let ctx = TestContext::new_alice().await;
 
@@ -4667,7 +4679,7 @@ mod tests {
         assert_eq!(chat.blocked, Blocked::Not);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_group_with_removed_message_id() -> Result<()> {
         // Alice creates a group with Bob, sends a message to bob
         let alice = TestContext::new_alice().await;
@@ -4721,7 +4733,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_marknoticed_chat() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
@@ -4766,7 +4778,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_contact_request_fresh_messages() -> Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -4816,7 +4828,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_contact_request_archive() -> Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -4856,7 +4868,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_classic_email_chat() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -4901,7 +4913,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_chat_get_color() -> Result<()> {
         let t = TestContext::new().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat").await?;
@@ -4944,7 +4956,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sticker_png() -> Result<()> {
         test_sticker(
             "sticker.png",
@@ -4955,7 +4967,7 @@ mod tests {
         .await
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sticker_jpeg() -> Result<()> {
         test_sticker(
             "sticker.jpg",
@@ -4966,7 +4978,7 @@ mod tests {
         .await
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sticker_gif() -> Result<()> {
         test_sticker(
             "sticker.gif",
@@ -4977,7 +4989,7 @@ mod tests {
         .await
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sticker_forward() -> Result<()> {
         // create chats
         let alice = TestContext::new_alice().await;
@@ -5008,7 +5020,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_forward() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -5029,7 +5041,33 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_forward_info_msg() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a").await?;
+        send_text_msg(&t, chat_id1, "msg one".to_string()).await?;
+        let bob_id = Contact::create(&t, "", "bob@example.net").await?;
+        add_contact_to_chat(&t, chat_id1, bob_id).await?;
+        let msg1 = t.get_last_msg_in(chat_id1).await;
+        assert!(msg1.is_info());
+        assert!(msg1.get_text().unwrap().contains("bob@example.net"));
+
+        let chat_id2 = ChatId::create_for_contact(&t, bob_id).await?;
+        assert_eq!(get_chat_msgs(&t, chat_id2, 0).await?.len(), 0);
+        forward_msgs(&t, &[msg1.id], chat_id2).await?;
+        let msg2 = t.get_last_msg_in(chat_id2).await;
+        assert!(!msg2.is_info()); // forwarded info-messages lose their info-state
+        assert_eq!(msg2.get_info_type(), SystemMessage::Unknown);
+        assert_ne!(msg2.from_id, ContactId::INFO);
+        assert_ne!(msg2.to_id, ContactId::INFO);
+        assert_eq!(msg2.get_text().unwrap(), msg1.get_text().unwrap());
+        assert!(msg2.is_forwarded());
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_forward_quote() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -5064,7 +5102,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_forward_group() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -5114,7 +5152,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_only_minimal_data_are_forwarded() -> Result<()> {
         // send a message from Alice to a group with Bob
         let alice = TestContext::new_alice().await;
@@ -5156,7 +5194,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_resend_own_message() -> Result<()> {
         // Alice creates group with Bob and sends an initial message
         let alice = TestContext::new_alice().await;
@@ -5209,7 +5247,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_resend_foreign_message_fails() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let alice_grp = create_group_chat(&alice, ProtectionStatus::Unprotected, "grp").await?;
@@ -5228,7 +5266,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_resend_opportunistically_encryption() -> Result<()> {
         // Alice creates group with Bob and sends an initial message
         let alice = TestContext::new_alice().await;
@@ -5265,7 +5303,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_resend_info_message_fails() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let alice_grp = create_group_chat(&alice, ProtectionStatus::Unprotected, "grp").await?;
@@ -5289,7 +5327,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_can_send_group() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = Contact::create(&alice, "", "bob@f.br").await?;
@@ -5315,7 +5353,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_broadcast() -> Result<()> {
         // create two context, send two messages so both know the other
         let alice = TestContext::new_alice().await;
@@ -5358,7 +5396,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_for_contact_with_blocked() -> Result<()> {
         let t = TestContext::new().await;
         let (contact_id, _) =
@@ -5392,7 +5430,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_chat_get_encryption_info() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;

src/chatlist.rs

@@ -376,7 +376,7 @@ mod tests {
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_try_load() {
         let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
@@ -432,7 +432,7 @@ mod tests {
         assert_eq!(chats.len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sort_self_talk_up_on_forward() {
         let t = TestContext::new().await;
         t.update_device_chats().await.unwrap();
@@ -457,7 +457,7 @@ mod tests {
             .is_self_talk());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_search_special_chat_names() {
         let t = TestContext::new().await;
         t.update_device_chats().await.unwrap();
@@ -488,7 +488,7 @@ mod tests {
         assert_eq!(chats.len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_search_single_chat() -> anyhow::Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -548,7 +548,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_search_single_chat_without_authname() -> anyhow::Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -610,7 +610,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_summary_unwrap() {
         let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")

src/config.rs

@@ -89,6 +89,11 @@ pub enum Config {
     #[strum(props(default = "1"))]
     FetchExistingMsgs,
 
+    /// If set to "1", then existing messages are considered to be already fetched.
+    /// This flag is reset after successful configuration.
+    #[strum(props(default = "1"))]
+    FetchedExistingMsgs,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -449,7 +454,7 @@ mod tests {
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;
         let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
@@ -466,7 +471,7 @@ mod tests {
         assert_eq!(media_quality, constants::MediaQuality::Worse);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ui_config() -> Result<()> {
         let t = TestContext::new().await;
 
@@ -488,7 +493,7 @@ mod tests {
     }
 
     /// Regression test for https://github.com/deltachat/deltachat-core-rust/issues/3012
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_config_bool() -> Result<()> {
         let t = TestContext::new().await;
 
@@ -500,7 +505,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_self_addrs() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -553,7 +558,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_change_primary_self_addr() -> Result<()> {
         let mut tcm = TestContextManager::new().await;
         let alice = tcm.alice().await;

src/configure.rs

@@ -6,13 +6,12 @@ mod read_url;
 mod server_params;
 
 use anyhow::{bail, ensure, Context as _, Result};
-use async_std::prelude::*;
-use async_std::task;
-use job::Action;
+use futures::FutureExt;
+use futures_lite::FutureExt as _;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
+use tokio::task;
 
 use crate::config::Config;
-use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2};
 use crate::context::Context;
 use crate::dc_tools::{time, EmailAddress};
 use crate::imap::Imap;
@@ -20,8 +19,8 @@ use crate::job;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
 use crate::message::{Message, Viewtype};
 use crate::oauth2::dc_get_oauth2_addr;
-use crate::param::Params;
 use crate::provider::{Protocol, Socket, UsernamePattern};
+use crate::scheduler::InterruptInfo;
 use crate::smtp::Smtp;
 use crate::stock_str;
 use crate::{chat, e2ee, provider};
@@ -57,8 +56,6 @@ impl Context {
 
     /// Configures this account with the currently set parameters.
     pub async fn configure(&self) -> Result<()> {
-        use futures::future::FutureExt;
-
         ensure!(
             self.scheduler.read().await.is_none(),
             "cannot configure, already running"
@@ -79,6 +76,24 @@ impl Context {
 
         self.free_ongoing().await;
 
+        if let Err(err) = res.as_ref() {
+            progress!(
+                self,
+                0,
+                Some(
+                    stock_str::configuration_failed(
+                        self,
+                        // We are using Anyhow's .context() and to show the
+                        // inner error, too, we need the {:#}:
+                        format!("{:#}", err),
+                    )
+                    .await
+                )
+            );
+        } else {
+            progress!(self, 1000);
+        }
+
         res
     }
 
@@ -116,58 +131,16 @@ impl Context {
             }
         }
 
-        match success {
-            Ok(_) => {
-                self.set_config(Config::NotifyAboutWrongPw, Some("1"))
-                    .await?;
-                progress!(self, 1000);
-                Ok(())
-            }
-            Err(err) => {
-                progress!(
-                    self,
-                    0,
-                    Some(
-                        stock_str::configuration_failed(
-                            self,
-                            // We are using Anyhow's .context() and to show the
-                            // inner error, too, we need the {:#}:
-                            format!("{:#}", err),
-                        )
-                        .await
-                    )
-                );
-                Err(err)
-            }
-        }
+        success?;
+        self.set_config(Config::NotifyAboutWrongPw, Some("1"))
+            .await?;
+        Ok(())
     }
 }
 
 async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 1);
 
-    // Check basic settings.
-    ensure!(!param.addr.is_empty(), "Please enter an email address.");
-
-    // Only check for IMAP password, SMTP password is an "advanced" setting.
-    ensure!(!param.imap.password.is_empty(), "Please enter a password.");
-    if param.smtp.password.is_empty() {
-        param.smtp.password = param.imap.password.clone()
-    }
-
-    // Normalize authentication flags.
-    let oauth2 = match param.server_flags & DC_LP_AUTH_FLAGS as i32 {
-        DC_LP_AUTH_OAUTH2 => true,
-        DC_LP_AUTH_NORMAL => false,
-        _ => false,
-    };
-    param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
-    param.server_flags |= if oauth2 {
-        DC_LP_AUTH_OAUTH2 as i32
-    } else {
-        DC_LP_AUTH_NORMAL as i32
-    };
-
     let socks5_config = param.socks5_config.clone();
     let socks5_enabled = socks5_config.is_some();
 
@@ -177,8 +150,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     // Step 1: Load the parameters and check email-address and password
 
     // Do oauth2 only if socks5 is disabled. As soon as we have a http library that can do
-    // socks5 requests, this can work with socks5 too
-    if oauth2 && !socks5_enabled {
+    // socks5 requests, this can work with socks5 too.  OAuth is always set either for both
+    // IMAP and SMTP or not at all.
+    if param.imap.oauth2 && !socks5_enabled {
         // the used oauth2 addr may differ, check this.
         // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
@@ -359,7 +333,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                 &smtp_param,
                 &socks5_config,
                 &smtp_addr,
-                oauth2,
                 provider_strict_tls,
                 &mut smtp,
             )
@@ -407,7 +380,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             &param.imap,
             &param.socks5_config,
             &param.addr,
-            oauth2,
             provider_strict_tls,
         )
         .await
@@ -431,7 +403,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 850);
 
     // Wait for SMTP configuration
-    match smtp_config_task.await {
+    match smtp_config_task.await.unwrap() {
         Ok(smtp_param) => {
             param.smtp = smtp_param;
         }
@@ -469,14 +441,12 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     e2ee::ensure_secret_key_exists(ctx).await?;
     info!(ctx, "key generation completed");
 
-    job::add(
-        ctx,
-        job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
-    )
-    .await?;
+    ctx.set_config_bool(Config::FetchedExistingMsgs, false)
+        .await?;
+    ctx.interrupt_inbox(InterruptInfo::new(false)).await;
 
     progress!(ctx, 940);
-    update_device_chats_handle.await?;
+    update_device_chats_handle.await??;
 
     ctx.sql.set_raw_config_bool("configured", true).await?;
 
@@ -565,26 +535,22 @@ async fn try_imap_one_param(
     param: &ServerLoginParam,
     socks5_config: &Option<Socks5Config>,
     addr: &str,
-    oauth2: bool,
     provider_strict_tls: bool,
 ) -> Result<Imap, ConfigurationError> {
     let inf = format!(
         "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
-        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
+        param.user,
+        param.server,
+        param.port,
+        param.security,
+        param.certificate_checks,
+        param.oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    let (_s, r) = async_std::channel::bounded(1);
+    let (_s, r) = async_channel::bounded(1);
 
-    let mut imap = match Imap::new(
-        param,
-        socks5_config.clone(),
-        addr,
-        oauth2,
-        provider_strict_tls,
-        r,
-    )
-    .await
+    let mut imap = match Imap::new(param, socks5_config.clone(), addr, provider_strict_tls, r).await
     {
         Err(err) => {
             info!(context, "failure: {}", err);
@@ -616,7 +582,6 @@ async fn try_smtp_one_param(
     param: &ServerLoginParam,
     socks5_config: &Option<Socks5Config>,
     addr: &str,
-    oauth2: bool,
     provider_strict_tls: bool,
     smtp: &mut Smtp,
 ) -> Result<(), ConfigurationError> {
@@ -627,7 +592,7 @@ async fn try_smtp_one_param(
         param.port,
         param.security,
         param.certificate_checks,
-        oauth2,
+        param.oauth2,
         if let Some(socks5_config) = socks5_config {
             socks5_config.to_string()
         } else {
@@ -637,14 +602,7 @@ async fn try_smtp_one_param(
     info!(context, "Trying: {}", inf);
 
     if let Err(err) = smtp
-        .connect(
-            context,
-            param,
-            socks5_config,
-            addr,
-            oauth2,
-            provider_strict_tls,
-        )
+        .connect(context, param, socks5_config, addr, provider_strict_tls)
         .await
     {
         info!(context, "failure: {}", err);
@@ -675,10 +633,13 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
         return "no error".to_string();
     };
 
-    if errors
-        .iter()
-        .all(|e| e.msg.to_lowercase().contains("could not resolve"))
-    {
+    if errors.iter().all(|e| {
+        e.msg.to_lowercase().contains("could not resolve")
+            || e.msg
+                .to_lowercase()
+                .contains("temporary failure in name resolution")
+            || e.msg.to_lowercase().contains("name or service not known")
+    }) {
         return stock_str::error_no_network(context).await;
     }
 
@@ -719,7 +680,7 @@ mod tests {
     use crate::config::Config;
     use crate::test_utils::TestContext;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_no_panic_on_bad_credentials() {
         let t = TestContext::new().await;
         t.set_config(Config::Addr, Some("probably@unexistant.addr"))

src/configure/read_url.rs

@@ -1,7 +1,6 @@
-use crate::context::Context;
+use anyhow::{anyhow, format_err};
 
-use anyhow::format_err;
-use anyhow::Context as _;
+use crate::context::Context;
 
 pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
     match read_url_inner(context, url).await {
@@ -16,24 +15,27 @@ pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
     }
 }
 
-pub async fn read_url_inner(context: &Context, mut url: &str) -> anyhow::Result<String> {
-    let mut _temp; // For the borrow checker
+pub async fn read_url_inner(context: &Context, url: &str) -> anyhow::Result<String> {
+    let client = reqwest::Client::new();
+    let mut url = url.to_string();
 
     // Follow up to 10 http-redirects
     for _i in 0..10 {
-        let mut response = surf::get(url).send().await.map_err(|e| e.into_inner())?;
+        let response = client.get(&url).send().await?;
         if response.status().is_redirection() {
-            _temp = response
-                .header("location")
-                .context("Redirection doesn't have a target location")?
+            let headers = response.headers();
+            let header = headers
+                .get_all("location")
+                .iter()
                 .last()
-                .to_string();
-            info!(context, "Following redirect to {}", _temp);
-            url = &_temp;
+                .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))?
+                .to_str()?;
+            info!(context, "Following redirect to {}", header);
+            url = header.to_string();
             continue;
         }
 
-        return response.body_string().await.map_err(|e| e.into_inner());
+        return response.text().await.map_err(Into::into);
     }
 
     Err(format_err!("Followed 10 redirections"))

src/contact.rs

@@ -2,9 +2,9 @@
 
 use std::convert::{TryFrom, TryInto};
 use std::fmt;
+use std::path::PathBuf;
 
 use anyhow::{bail, ensure, Context as _, Result};
-use async_std::path::PathBuf;
 use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use regex::Regex;
@@ -1438,8 +1438,8 @@ fn split_address_book(book: &str) -> Vec<(&str, &str)> {
 
 #[cfg(test)]
 mod tests {
-    use async_std::fs::File;
-    use async_std::io::WriteExt;
+    use tokio::fs::File;
+    use tokio::io::AsyncWriteExt;
 
     use super::*;
 
@@ -1508,7 +1508,7 @@ mod tests {
         )
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_contacts() -> Result<()> {
         let context = TestContext::new().await;
 
@@ -1572,7 +1572,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_is_self_addr() -> Result<()> {
         let t = TestContext::new().await;
         assert_eq!(t.is_self_addr("me@me.org").await?, false);
@@ -1584,7 +1584,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_or_lookup() {
         // add some contacts, this also tests add_address_book()
         let t = TestContext::new().await;
@@ -1685,7 +1685,7 @@ mod tests {
         assert!(!contact.is_blocked());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_contact_name_changes() -> Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -1797,7 +1797,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_delete() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -1825,7 +1825,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_remote_authnames() {
         let t = TestContext::new().await;
 
@@ -1876,7 +1876,7 @@ mod tests {
         assert_eq!(contact.get_display_name(), "bob3");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_remote_authnames_create_empty() {
         let t = TestContext::new().await;
 
@@ -1925,7 +1925,7 @@ mod tests {
     ///
     /// In the past, "Not Bob" name was stuck until "Bob" changed the name to "Not Bob" and back in
     /// the "From:" field or user set the name to empty string manually.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_remote_authnames_update_to() -> Result<()> {
         let t = TestContext::new().await;
 
@@ -1958,7 +1958,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_remote_authnames_edit_empty() {
         let t = TestContext::new().await;
 
@@ -1995,7 +1995,7 @@ mod tests {
         assert!(addr_cmp(" mailto:AA@AA.ORG", "Aa@Aa.orG"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_name_in_address() {
         let t = TestContext::new().await;
 
@@ -2034,7 +2034,7 @@ mod tests {
             .is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_lookup_id_by_addr() {
         let t = TestContext::new().await;
 
@@ -2059,7 +2059,7 @@ mod tests {
         assert_eq!(id, Some(ContactId::SELF));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_contact_get_color() -> Result<()> {
         let t = TestContext::new().await;
         let contact_id = Contact::create(&t, "name", "name@example.net").await?;
@@ -2078,7 +2078,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_contact_get_encrinfo() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -2123,7 +2123,7 @@ CCCB 5AA9 F6E1 141C 9431
 
     /// Tests that status is synchronized when sending encrypted BCC-self messages and not
     /// synchronized when the message is not encrypted.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_synchronize_status() -> Result<()> {
         // Alice has two devices.
         let alice1 = TestContext::new_alice().await;
@@ -2188,7 +2188,7 @@ CCCB 5AA9 F6E1 141C 9431
     }
 
     /// Tests that DC_EVENT_SELFAVATAR_CHANGED is emitted on avatar changes.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_selfavatar_changed_event() -> Result<()> {
         // Alice has two devices.
         let alice1 = TestContext::new_alice().await;
@@ -2247,7 +2247,7 @@ CCCB 5AA9 F6E1 141C 9431
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_last_seen() -> Result<()> {
         let alice = TestContext::new_alice().await;
 

src/context.rs

@@ -3,14 +3,13 @@
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
-use std::time::{Instant, SystemTime};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::{Duration, Instant, SystemTime};
 
 use anyhow::{ensure, Result};
-use async_std::{
-    channel::{self, Receiver, Sender},
-    path::{Path, PathBuf},
-    sync::{Arc, Mutex, RwLock},
-};
+use async_channel::{self as channel, Receiver, Sender};
+use tokio::sync::{Mutex, RwLock};
 
 use crate::chat::{get_chat_cnt, ChatId};
 use crate::config::Config;
@@ -22,6 +21,7 @@ use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
 use crate::quota::QuotaInfo;
+use crate::ratelimit::Ratelimit;
 use crate::scheduler::Scheduler;
 use crate::sql::Sql;
 
@@ -55,6 +55,7 @@ pub struct InnerContext {
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Option<Scheduler>>,
+    pub(crate) ratelimit: RwLock<Ratelimit>,
 
     /// Recently loaded quota information, if any.
     /// Set to `None` if quota was never tried to load.
@@ -73,7 +74,7 @@ pub struct InnerContext {
     /// The text of the last error logged and emitted as an event.
     /// If the ui wants to display an error after a failure,
     /// `last_error` should be used to avoid races with the event thread.
-    pub(crate) last_error: RwLock<String>,
+    pub(crate) last_error: std::sync::RwLock<String>,
 }
 
 /// The state of ongoing process.
@@ -113,8 +114,8 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 
 impl Context {
     /// Creates new context and opens the database.
-    pub async fn new(dbfile: PathBuf, id: u32) -> Result<Context> {
-        let context = Self::new_closed(dbfile, id).await?;
+    pub async fn new(dbfile: &Path, id: u32, events: Events) -> Result<Context> {
+        let context = Self::new_closed(dbfile, id, events).await?;
 
         // Open the database if is not encrypted.
         if context.check_passphrase("".to_string()).await? {
@@ -124,15 +125,15 @@ impl Context {
     }
 
     /// Creates new context without opening the database.
-    pub async fn new_closed(dbfile: PathBuf, id: u32) -> Result<Context> {
+    pub async fn new_closed(dbfile: &Path, id: u32, events: Events) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         let blobdir = dbfile.with_file_name(blob_fname);
-        if !blobdir.exists().await {
-            async_std::fs::create_dir_all(&blobdir).await?;
+        if !blobdir.exists() {
+            tokio::fs::create_dir_all(&blobdir).await?;
         }
-        let context = Context::with_blobdir(dbfile, blobdir, id).await?;
+        let context = Context::with_blobdir(dbfile.into(), blobdir, id, events).await?;
         Ok(context)
     }
 
@@ -167,9 +168,10 @@ impl Context {
         dbfile: PathBuf,
         blobdir: PathBuf,
         id: u32,
+        events: Events,
     ) -> Result<Context> {
         ensure!(
-            blobdir.is_dir().await,
+            blobdir.is_dir(),
             "Blobdir does not exist: {}",
             blobdir.display()
         );
@@ -184,12 +186,13 @@ impl Context {
             oauth2_mutex: Mutex::new(()),
             wrong_pw_warning_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
-            events: Events::default(),
+            events,
             scheduler: RwLock::new(None),
+            ratelimit: RwLock::new(Ratelimit::new(Duration::new(60, 0), 3.0)), // Allow to send 3 messages immediately, no more than once every 20 seconds.
             quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
-            last_error: RwLock::new("".to_string()),
+            last_error: std::sync::RwLock::new("".to_string()),
         };
 
         let ctx = Context {
@@ -339,7 +342,7 @@ impl Context {
 
     pub async fn get_info(&self) -> Result<BTreeMap<&'static str, String>> {
         let unset = "0";
-        let l = LoginParam::load_candidate_params(self).await?;
+        let l = LoginParam::load_candidate_params_unchecked(self).await?;
         let l2 = LoginParam::load_configured_params(self).await?;
         let secondary_addrs = self.get_secondary_self_addrs().await?.join(", ");
         let displayname = self.get_config(Config::Displayname).await?;
@@ -433,6 +436,12 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "fetched_existing_msgs",
+            self.get_config_bool(Config::FetchedExistingMsgs)
+                .await?
+                .to_string(),
+        );
         res.insert(
             "show_emails",
             self.get_config_int(Config::ShowEmails).await?.to_string(),
@@ -633,14 +642,14 @@ impl Context {
         Ok(mvbox.as_deref() == Some(folder_name))
     }
 
-    pub(crate) fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
+    pub(crate) fn derive_blobdir(dbfile: &Path) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         dbfile.with_file_name(blob_fname)
     }
 
-    pub(crate) fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
+    pub(crate) fn derive_walfile(dbfile: &Path) -> PathBuf {
         let mut wal_fname = OsString::new();
         wal_fname.push(dbfile.file_name().unwrap_or_default());
         wal_fname.push("-wal");
@@ -669,19 +678,19 @@ mod tests {
     use strum::IntoEnumIterator;
     use tempfile::tempdir;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_wrong_db() -> Result<()> {
         let tmp = tempfile::tempdir()?;
         let dbfile = tmp.path().join("db.sqlite");
-        std::fs::write(&dbfile, b"123")?;
-        let res = Context::new(dbfile.into(), 1).await?;
+        tokio::fs::write(&dbfile, b"123").await?;
+        let res = Context::new(&dbfile, 1, Events::new()).await?;
 
         // Broken database is indistinguishable from encrypted one.
         assert_eq!(res.is_open().await, false);
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_fresh_msgs() {
         let t = TestContext::new().await;
         let fresh = t.get_fresh_msgs().await.unwrap();
@@ -708,7 +717,7 @@ mod tests {
         dc_receive_imf(t, msg.as_bytes(), false).await.unwrap();
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_fresh_msgs_and_muted_chats() {
         // receive various mails in 3 chats
         let t = TestContext::new_alice().await;
@@ -758,7 +767,7 @@ mod tests {
         assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 9); // claire is counted again
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_fresh_msgs_and_muted_until() {
         let t = TestContext::new_alice().await;
         let bob = t.create_chat_with_contact("", "bob@g.it").await;
@@ -816,61 +825,61 @@ mod tests {
         assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new(dbfile.into(), 1).await.unwrap();
+        Context::new(&dbfile, 1, Events::new()).await.unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_wrong_blogdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
-        std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new(dbfile.into(), 1).await;
+        tokio::fs::write(&blobdir, b"123").await.unwrap();
+        let res = Context::new(&dbfile, 1, Events::new()).await;
         assert!(res.is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sqlite_parent_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new(dbfile.into(), 1).await.unwrap();
+        Context::new(&dbfile, 1, Events::new()).await.unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_with_empty_blobdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(dbfile.into(), blobdir, 1).await;
+        let res = Context::with_blobdir(dbfile, blobdir, 1, Events::new()).await;
         assert!(res.is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_with_blobdir_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(dbfile.into(), blobdir.into(), 1).await;
+        let res = Context::with_blobdir(dbfile, blobdir, 1, Events::new()).await;
         assert!(res.is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn no_crashes_on_context_deref() {
         let t = TestContext::new().await;
         std::mem::drop(t);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_info() {
         let t = TestContext::new().await;
 
@@ -886,7 +895,7 @@ mod tests {
         assert_eq!(info.get("level").unwrap(), "awesome");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_info_completeness() {
         // For easier debugging,
         // get_info() shall return all important information configurable by the Config-values.
@@ -934,7 +943,7 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let self_talk = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
@@ -990,7 +999,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_limit_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let chat = alice
@@ -1023,13 +1032,13 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_check_passphrase() -> Result<()> {
         let dir = tempdir()?;
         let dbfile = dir.path().join("db.sqlite");
 
         let id = 1;
-        let context = Context::new_closed(dbfile.clone().into(), id)
+        let context = Context::new_closed(&dbfile, id, Events::new())
             .await
             .context("failed to create context")?;
         assert_eq!(context.open("foo".to_string()).await?, true);
@@ -1037,7 +1046,7 @@ mod tests {
         drop(context);
 
         let id = 2;
-        let context = Context::new(dbfile.into(), id)
+        let context = Context::new(&dbfile, id, Events::new())
             .await
             .context("failed to create context")?;
         assert_eq!(context.is_open().await, false);
@@ -1048,7 +1057,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ongoing() -> Result<()> {
         let context = TestContext::new().await;
 

src/dc_receive_imf.rs

@@ -246,7 +246,7 @@ pub(crate) async fn dc_receive_imf_inner(
 
     if let Some(ref status_update) = mime_parser.webxdc_status_update {
         if let Err(err) = context
-            .receive_status_update(insert_msg_id, status_update)
+            .receive_status_update(from_id, insert_msg_id, status_update)
             .await
         {
             warn!(context, "receive_imf cannot update status: {}", err);
@@ -280,7 +280,9 @@ pub(crate) async fn dc_receive_imf_inner(
     // Always update the status, even if there is no footer, to allow removing the status.
     //
     // Ignore MDNs though, as they never contain the signature even if user has set it.
+    // Ignore footers from mailinglists as they are often created or modified by the mailinglist software.
     if mime_parser.mdn_reports.is_empty()
+        && !mime_parser.is_mailinglist_message()
         && is_partial_download.is_none()
         && from_id != ContactId::UNDEFINED
         && context
@@ -1739,13 +1741,14 @@ async fn create_or_lookup_mailinglist(
         }
     }
 
-    // if we have an additional name square brackets in the subject, we prefer that
+    // additional names in square brackets in the subject are preferred
     // (as that part is much more visible, we assume, that names is shorter and comes more to the point,
     // than the sometimes longer part from ListId)
     let subject = mime_parser.get_subject().unwrap_or_default();
-    static SUBJECT: Lazy<Regex> = Lazy::new(|| Regex::new(r"^.{0,5}\[(.*.)\]").unwrap());
+    static SUBJECT: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"^.{0,5}\[(.+?)\](\s*\[.+\])?").unwrap()); // remove square brackets around first name
     if let Some(cap) = SUBJECT.captures(&subject) {
-        name = cap[1].to_string();
+        name = cap[1].to_string() + cap.get(2).map_or("", |m| m.as_str());
     }
 
     // if we do not have a name yet and `From` indicates, that this is a notification list,
@@ -2203,8 +2206,8 @@ async fn add_or_lookup_contact_by_addr(
 #[cfg(test)]
 mod tests {
 
-    use async_std::fs::{self, File};
-    use async_std::io::WriteExt;
+    use tokio::fs::{self, File};
+    use tokio::io::AsyncWriteExt;
 
     use super::*;
 
@@ -2216,7 +2219,7 @@ mod tests {
     use crate::message::Message;
     use crate::test_utils::{get_chat_msg, TestContext, TestContextManager};
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_grpid_simple() {
         let context = TestContext::new().await;
         let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
@@ -2234,7 +2237,7 @@ mod tests {
         assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_grpid_from_multiple() {
         let context = TestContext::new().await;
         let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
@@ -2283,7 +2286,7 @@ mod tests {
                     \n\
                     hello\n";
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_adhoc_group_show_chats_only() {
         let t = TestContext::new_alice().await;
         assert_eq!(t.get_config_int(Config::ShowEmails).await.unwrap(), 0);
@@ -2306,7 +2309,7 @@ mod tests {
         assert_eq!(chats.len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_adhoc_group_show_accepted_contact_unknown() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("1")).await.unwrap();
@@ -2317,7 +2320,7 @@ mod tests {
         assert_eq!(chats.len(), 0);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_adhoc_group_show_accepted_contact_known() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("1")).await.unwrap();
@@ -2330,7 +2333,7 @@ mod tests {
         assert_eq!(chats.len(), 0);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_adhoc_group_show_accepted_contact_accepted() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("1")).await.unwrap();
@@ -2368,7 +2371,7 @@ mod tests {
         assert_eq!(chat::get_chat_contacts(&t, chat_id).await.unwrap().len(), 3);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_adhoc_group_show_all() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -2387,7 +2390,7 @@ mod tests {
         assert_eq!(chat::get_chat_contacts(&t, chat_id).await.unwrap().len(), 3);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_read_receipt_and_unarchive() -> Result<()> {
         // create alice's account
         let t = TestContext::new_alice().await;
@@ -2501,7 +2504,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_no_from() {
         // if there is no from given, from_id stays 0 which is just fine. These messages
         // are very rare, however, we have to add them to the database
@@ -2533,7 +2536,7 @@ mod tests {
         assert!(chats.get_msg_id(0).is_ok());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_escaped_from() {
         let t = TestContext::new_alice().await;
         let contact_id = Contact::create(&t, "foobar", "foobar@example.com")
@@ -2566,7 +2569,7 @@ mod tests {
         assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_escaped_recipients() {
         let t = TestContext::new_alice().await;
         Contact::create(&t, "foobar", "foobar@example.com")
@@ -2608,7 +2611,7 @@ mod tests {
         assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_cc_to_contact() {
         let t = TestContext::new_alice().await;
         Contact::create(&t, "foobar", "foobar@example.com")
@@ -2643,19 +2646,19 @@ mod tests {
         assert_eq!(contact.get_display_name(), "Carl");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_ndn_tiscali() {
         test_parse_ndn(
             "alice@tiscali.it",
             "shenauithz@testrun.org",
             "Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it",
             include_bytes!("../test-data/message/tiscali_ndn.eml"),
-            None,
+            Some("Non-Delivery-Notification without further details."),
         )
         .await;
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_ndn_testrun() {
         test_parse_ndn(
             "alice@testrun.org",
@@ -2667,7 +2670,7 @@ mod tests {
         .await;
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_ndn_yahoo() {
         test_parse_ndn(
             "alice@yahoo.com",
@@ -2679,7 +2682,7 @@ mod tests {
         .await;
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_ndn_gmail() {
         test_parse_ndn(
             "alice@gmail.com",
@@ -2691,7 +2694,7 @@ mod tests {
         .await;
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_ndn_gmx() {
         test_parse_ndn(
             "alice@gmx.com",
@@ -2703,7 +2706,7 @@ mod tests {
         .await;
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_ndn_posteo() {
         test_parse_ndn(
             "alice@posteo.org",
@@ -2715,7 +2718,7 @@ mod tests {
         .await;
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_ndn_testrun_2() {
         test_parse_ndn(
             "alice@example.org",
@@ -2781,7 +2784,7 @@ mod tests {
         assert_eq!(msg.error(), error_msg.map(|error| error.to_string()));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_ndn_group_msg() -> Result<()> {
         let t = TestContext::new().await;
         t.configure_addr("alice@gmail.com").await;
@@ -2841,7 +2844,7 @@ mod tests {
         Message::load_from_db(context, msg_id).await.unwrap()
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_html_only_mail() {
         let t = TestContext::new_alice().await;
         let msg = load_imf_email(&t, include_bytes!("../test-data/message/wrong-html.eml")).await;
@@ -2874,7 +2877,7 @@ mod tests {
     \n\
     hello back\n";
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_github_mailing_list() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.ctx.set_config(Config::ShowEmails, Some("2")).await?;
@@ -2941,7 +2944,7 @@ mod tests {
     \n\
     body 4\n";
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_classic_mailing_list() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.ctx
@@ -2987,7 +2990,7 @@ Hello mailinglist!\r\n"
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_other_device_writes_to_mailinglist() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await?;
@@ -3037,7 +3040,7 @@ Hello mailinglist!\r\n"
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_block_mailing_list() {
         let t = TestContext::new_alice().await;
         t.ctx
@@ -3071,7 +3074,7 @@ Hello mailinglist!\r\n"
         assert_eq!(msgs.len(), 2);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mailing_list_decide_block_then_unblock() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -3104,7 +3107,7 @@ Hello mailinglist!\r\n"
         assert_eq!(msgs.len(), 2);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mailing_list_decide_not_now() {
         let t = TestContext::new_alice().await;
         t.ctx
@@ -3137,7 +3140,7 @@ Hello mailinglist!\r\n"
         assert!(chat.is_contact_request());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mailing_list_decide_accept() {
         let t = TestContext::new_alice().await;
         t.ctx
@@ -3165,7 +3168,32 @@ Hello mailinglist!\r\n"
         assert!(chat.can_send(&t.ctx).await.unwrap());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_mailing_list_multiple_names_in_subject() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        t.set_config(Config::ShowEmails, Some("2")).await?;
+        dc_receive_imf(
+            &t,
+            b"From: Foo Bar <foo@bar.org>\n\
+    To: deltachat/deltachat-core-rust <deltachat-core-rust@noreply.github.com>\n\
+    Subject: [ola list] [foo][bar]  just a subject\n\
+    Message-ID: <3333@example.org>\n\
+    List-ID: \"looong description of 'ola list', with foo, bar\" <delta.codespeak.net>\n\
+    Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+    \n\
+    hello\n",
+            false,
+        )
+        .await
+        .unwrap();
+        let msg = t.get_last_msg().await;
+        let chat_id = msg.get_chat_id();
+        let chat = Chat::load_from_db(&t, chat_id).await?;
+        assert_eq!(chat.name, "ola list [foo][bar]");
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_majordomo_mailing_list() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -3214,7 +3242,7 @@ Hello mailinglist!\r\n"
         assert_eq!(chat::get_chat_msgs(&t, chat.id, 0).await.unwrap().len(), 2);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mailchimp_mailing_list() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -3244,7 +3272,7 @@ Hello mailinglist!\r\n"
         assert_eq!(chat.name, "Atlas Obscura");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_dhl_mailing_list() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -3269,7 +3297,7 @@ Hello mailinglist!\r\n"
         assert_eq!(chat.name, "DHL Paket");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_dpd_mailing_list() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -3294,7 +3322,7 @@ Hello mailinglist!\r\n"
         assert_eq!(chat.name, "DPD");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_xt_local_mailing_list() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await?;
@@ -3324,7 +3352,7 @@ Hello mailinglist!\r\n"
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_xing_mailing_list() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await?;
@@ -3345,7 +3373,7 @@ Hello mailinglist!\r\n"
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ttline_mailing_list() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await?;
@@ -3366,7 +3394,7 @@ Hello mailinglist!\r\n"
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mailing_list_with_mimepart_footer() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -3397,7 +3425,7 @@ Hello mailinglist!\r\n"
         assert_eq!(chat.name, "Intern");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mailing_list_with_mimepart_footer_signed() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -3422,7 +3450,7 @@ Hello mailinglist!\r\n"
 
     /// Test that the changes from apply_mailinglist_changes() are also applied
     /// if the message is assigned to the chat by In-Reply-To
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_apply_mailinglist_changes_assigned_by_reply() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -3464,7 +3492,7 @@ Hello mailinglist!\r\n"
         )
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_dont_show_tokens_in_contacts_list() {
         check_dont_show_in_contacts_list(
             "reply+OGHVYCLVBEGATYBICAXBIRQATABUOTUCERABERAHNO@reply.github.com",
@@ -3472,7 +3500,7 @@ Hello mailinglist!\r\n"
         .await;
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_dont_show_noreply_in_contacts_list() {
         check_dont_show_in_contacts_list("noreply@github.com").await;
     }
@@ -3510,7 +3538,7 @@ YEAAAAAA!.
         assert!(contacts.is_empty()); // The contact should not have been added to the db
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_pdf_filename_simple() {
         let t = TestContext::new_alice().await;
         let msg = load_imf_email(
@@ -3523,7 +3551,7 @@ YEAAAAAA!.
         assert_eq!(msg.param.get(Param::File).unwrap(), "$BLOBDIR/simple.pdf");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_pdf_filename_continuation() {
         // test filenames split across multiple header lines, see rfc 2231
         let t = TestContext::new_alice().await;
@@ -3550,7 +3578,7 @@ YEAAAAAA!.
     /// or mua may use multipart/related not correctly -
     /// so this test is in competition with parse_thunderbird_html_embedded_image()
     /// that wants the image to be kept in the chat.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_many_images() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -3571,7 +3599,7 @@ YEAAAAAA!.
 
     /// Test that classical MUA messages are assigned to group chats based on the `In-Reply-To`
     /// header.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_in_reply_to() {
         let t = TestContext::new().await;
         t.configure_addr("bob@example.com").await;
@@ -3636,7 +3664,7 @@ YEAAAAAA!.
 
     /// Test that classical MUA messages are assigned to group chats
     /// based on the `In-Reply-To` header for two-member groups.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_in_reply_to_two_member_group() {
         let t = TestContext::new().await;
         t.configure_addr("bob@example.com").await;
@@ -3738,7 +3766,7 @@ YEAAAAAA!.
         assert_eq!(msg.get_text().unwrap(), "private reply");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_save_mime_headers_off() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -3753,7 +3781,7 @@ YEAAAAAA!.
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_save_mime_headers_on() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         alice.set_config_bool(Config::SaveMimeHeaders, true).await?;
@@ -3924,7 +3952,7 @@ YEAAAAAA!.
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_alias_support_answer_from_nondc() {
         // Bob, the other supporter, answers with a classic MUA.
         let bob_answer = b"To: support@example.org, claire@example.org\n\
@@ -3944,7 +3972,7 @@ YEAAAAAA!.
         check_alias_reply(bob_answer, false, false).await;
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_alias_answer_from_dc() {
         // Bob, the other supporter, answers with Delta Chat.
         let bob_answer = b"To: support@example.org, claire@example.org\n\
@@ -3968,7 +3996,7 @@ YEAAAAAA!.
         check_alias_reply(bob_answer, false, false).await;
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_dont_assign_to_trash_by_parent() {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
@@ -4016,7 +4044,7 @@ YEAAAAAA!.
         assert_eq!(msg.text.unwrap(), "Reply");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_dont_show_all_outgoing_msgs_in_self_chat() {
         // Regression test for <https://github.com/deltachat/deltachat-android/issues/1940>:
         // Some servers add a `Bcc: <Self>` header, which caused all outgoing messages to
@@ -4043,7 +4071,7 @@ Message content",
         assert_ne!(msg.chat_id, t.get_self_chat().await.id);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_outgoing_classic_mail_creates_chat() {
         let alice = TestContext::new_alice().await;
 
@@ -4073,7 +4101,7 @@ Message content",
         assert_eq!(msg.get_text().unwrap(), "Subj – Message content");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_duplicate_message() -> Result<()> {
         // Test that duplicate messages are ignored based on the Message-ID
         let alice = TestContext::new_alice().await;
@@ -4132,7 +4160,80 @@ Second signature";
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_ignore_footer_status_from_mailinglist() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        t.set_config(Config::ShowEmails, Some("2")).await?;
+        let bob_id = Contact::add_or_lookup(&t, "", "bob@example.net", Origin::IncomingUnknownCc)
+            .await?
+            .0;
+        let bob = Contact::load_from_db(&t, bob_id).await?;
+        assert_eq!(bob.get_status(), "");
+        assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 0);
+
+        dc_receive_imf(
+            &t,
+            b"From: Bob <bob@example.net>
+To: Alice <alice@example.org>
+Message-ID: <1@example.org>
+Subject: first message
+
+body 1
+
+--
+Original signature",
+            false,
+        )
+        .await?;
+        let one2one_chat_id = t.get_last_msg().await.chat_id;
+        let bob = Contact::load_from_db(&t, bob_id).await?;
+        assert_eq!(bob.get_status(), "Original signature");
+
+        dc_receive_imf(
+            &t,
+            b"From: Bob <bob@example.net>
+Sender: ml@example.net
+To: Alice <alice@example.org>
+Message-ID: <2@example.net>
+Precedence: list
+Subject: second message
+
+body 2
+
+--
+The modified signature
+--
+Tap here to unsubscribe ...",
+            false,
+        )
+        .await?;
+        let ml_chat_id = t.get_last_msg().await.chat_id;
+        let bob = Contact::load_from_db(&t, bob_id).await?;
+        assert_eq!(bob.get_status(), "Original signature");
+
+        dc_receive_imf(
+            &t,
+            b"From: Bob <bob@example.net>
+To: Alice <alice@example.org>
+Message-ID: <3@example.org>
+Subject: third message
+
+body 3
+
+--
+Original signature updated",
+            false,
+        )
+        .await?;
+        let bob = Contact::load_from_db(&t, bob_id).await?;
+        assert_eq!(bob.get_status(), "Original signature updated");
+        assert_eq!(get_chat_msgs(&t, one2one_chat_id, 0).await?.len(), 2);
+        assert_eq!(get_chat_msgs(&t, ml_chat_id, 0).await?.len(), 1);
+        assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 2);
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_chat_assignment_private_classical_reply() {
         for outgoing_is_classical in &[true, false] {
             let t = TestContext::new_alice().await;
@@ -4216,7 +4317,7 @@ Private reply"#,
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_chat_assignment_private_chat_reply() {
         for (outgoing_is_classical, outgoing_has_multiple_recipients) in
             &[(true, true), (false, true), (false, false)]
@@ -4312,7 +4413,7 @@ Sent with my Delta Chat Messenger: https://delta.chat
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_chat_assignment_nonprivate_classical_reply() {
         for outgoing_is_classical in &[true, false] {
             let t = TestContext::new_alice().await;
@@ -4421,7 +4522,7 @@ Reply to all"#,
     /// messages have the same recipient lists and only differ in the subject and message contents.
     /// The messages can be properly assigned to chats only using the In-Reply-To or References
     /// headers.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_chat_assignment_adhoc() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_alice().await;
@@ -4501,7 +4602,7 @@ Second thread."#;
     }
 
     /// Test that read receipts don't create chats.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_read_receipts_dont_create_chats() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -4534,7 +4635,7 @@ Second thread."#;
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_gmx_forwarded_msg() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await?;
@@ -4554,7 +4655,7 @@ Second thread."#;
     }
 
     /// Tests that user is notified about new incoming contact requests.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_incoming_contact_request() -> Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -4579,7 +4680,7 @@ Second thread."#;
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_parent_message() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await?;
@@ -4642,7 +4743,7 @@ Message with references."#;
     }
 
     /// Test a message with RFC 1847 encapsulation as created by Thunderbird.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_rfc1847_encapsulation() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -4668,7 +4769,7 @@ Message with references."#;
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_invalid_to_address() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -4680,7 +4781,7 @@ Message with references."#;
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_reply_from_different_addr() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await?;
@@ -4745,7 +4846,7 @@ Reply from different address
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_long_filenames() -> Result<()> {
         let mut tcm = TestContextManager::new().await;
         let alice = tcm.alice().await;
@@ -4800,7 +4901,7 @@ Reply from different address
     }
 
     /// Tests that contact request is accepted automatically on outgoing message.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_accept_outgoing() -> Result<()> {
         let mut tcm = TestContextManager::new().await;
         let alice1 = tcm.alice().await;
@@ -4845,7 +4946,7 @@ Reply from different address
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_outgoing_private_reply_multidevice() -> Result<()> {
         let mut tcm = TestContextManager::new().await;
         let alice1 = tcm.alice().await;
@@ -4922,7 +5023,7 @@ Reply from different address
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_no_private_reply_to_blocked_account() -> Result<()> {
         let mut tcm = TestContextManager::new().await;
         let alice = tcm.alice().await;

src/dc_tools.rs

@@ -5,20 +5,19 @@ use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
 use std::io::Cursor;
+use std::path::{Path, PathBuf};
 use std::str::from_utf8;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
-use async_std::path::{Path, PathBuf};
-use async_std::prelude::*;
-use async_std::{fs, io};
-
 use anyhow::{bail, Error, Result};
 use chrono::{Local, TimeZone};
+use futures::StreamExt;
 use mailparse::dateparse;
 use mailparse::headers::Headers;
 use mailparse::MailHeaderMap;
 use rand::{thread_rng, Rng};
+use tokio::{fs, io};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
 use crate::constants::{DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
@@ -278,7 +277,7 @@ pub fn dc_get_filemeta(buf: &[u8]) -> Result<(u32, u32), Error> {
 ///
 /// If `path` starts with "$BLOBDIR", replaces it with the blobdir path.
 /// Otherwise, returns path as is.
-pub(crate) fn dc_get_abs_path<P: AsRef<Path>>(context: &Context, path: P) -> PathBuf {
+pub(crate) fn dc_get_abs_path(context: &Context, path: impl AsRef<Path>) -> PathBuf {
     let p: &Path = path.as_ref();
     if let Ok(p) = p.strip_prefix("$BLOBDIR") {
         context.get_blobdir().join(p)
@@ -297,10 +296,10 @@ pub(crate) async fn dc_get_filebytes(context: &Context, path: impl AsRef<Path>)
 
 pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
-    if !path_abs.exists().await {
+    if !path_abs.exists() {
         return false;
     }
-    if !path_abs.is_file().await {
+    if !path_abs.is_file() {
         warn!(
             context,
             "refusing to delete non-file \"{}\".",
@@ -323,8 +322,9 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
 }
 
 pub async fn dc_delete_files_in_dir(context: &Context, path: impl AsRef<Path>) {
-    match async_std::fs::read_dir(path).await {
-        Ok(mut read_dir) => {
+    match tokio::fs::read_dir(path).await {
+        Ok(read_dir) => {
+            let mut read_dir = tokio_stream::wrappers::ReadDirStream::new(read_dir);
             while let Some(entry) = read_dir.next().await {
                 match entry {
                     Ok(file) => {
@@ -344,7 +344,7 @@ pub(crate) async fn dc_create_folder(
     path: impl AsRef<Path>,
 ) -> Result<(), io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
-    if !path_abs.exists().await {
+    if !path_abs.exists() {
         match fs::create_dir_all(path_abs).await {
             Ok(_) => Ok(()),
             Err(err) => {
@@ -655,7 +655,7 @@ mod tests {
         assert_eq!(hop_info, expected)
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_receive_headers_integration() {
         let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
         let expected = r"State: Fresh
@@ -873,7 +873,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_file_handling() {
         let t = TestContext::new().await;
         let context = &t;
@@ -889,8 +889,8 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content")
             .await
             .is_ok());
-        assert!(dc_file_exist!(context, "$BLOBDIR/foobar").await);
-        assert!(!dc_file_exist!(context, "$BLOBDIR/foobarx").await);
+        assert!(dc_file_exist!(context, "$BLOBDIR/foobar"));
+        assert!(!dc_file_exist!(context, "$BLOBDIR/foobarx"));
         assert_eq!(dc_get_filebytes(context, "$BLOBDIR/foobar").await, 7);
 
         let abs_path = context
@@ -899,23 +899,23 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
             .to_string_lossy()
             .to_string();
 
-        assert!(dc_file_exist!(context, &abs_path).await);
+        assert!(dc_file_exist!(context, &abs_path));
 
         assert!(dc_delete_file(context, "$BLOBDIR/foobar").await);
         assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder")
             .await
             .is_ok());
-        assert!(dc_file_exist!(context, "$BLOBDIR/foobar-folder").await);
+        assert!(dc_file_exist!(context, "$BLOBDIR/foobar-folder"));
         assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder").await);
 
         let fn0 = "$BLOBDIR/data.data";
         assert!(dc_write_file(context, &fn0, b"content").await.is_ok());
 
         assert!(dc_delete_file(context, &fn0).await);
-        assert!(!dc_file_exist!(context, &fn0).await);
+        assert!(!dc_file_exist!(context, &fn0));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_smeared_timestamp() {
         let t = TestContext::new().await;
         assert_ne!(
@@ -931,7 +931,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_smeared_timestamps() {
         let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
@@ -1001,7 +1001,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         assert_eq!(improve_single_line_input("\r\nahte\n\r"), "ahte");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_maybe_warn_on_bad_time() {
         let t = TestContext::new().await;
         let timestamp_now = time();
@@ -1064,7 +1064,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         assert_eq!(msgs.len(), 2);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_maybe_warn_on_outdated() {
         let t = TestContext::new().await;
         let timestamp_now: i64 = time();

src/dehtml.rs

@@ -287,7 +287,7 @@ pub fn dehtml_manually(buf: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::simplify::simplify;
+    use crate::simplify::{simplify, SimplifiedText};
 
     #[test]
     fn test_dehtml() {
@@ -311,7 +311,7 @@ mod tests {
             ("<!some invalid html code>\n<b>some text</b>", "some text"),
         ];
         for (input, output) in cases {
-            assert_eq!(simplify(dehtml(input).unwrap(), true).0, output);
+            assert_eq!(simplify(dehtml(input).unwrap(), true).text, output);
         }
         let none_cases = vec!["<html> </html>", ""];
         for input in none_cases {
@@ -382,15 +382,21 @@ mod tests {
         assert_eq!(txt.trim(), "two\nlines");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_quote_div() {
         let input = include_str!("../test-data/message/gmx-quote-body.eml");
         let dehtml = dehtml(input).unwrap();
         println!("{}", dehtml);
-        let (msg, forwarded, cut, top_quote, footer) = simplify(dehtml, false);
-        assert_eq!(msg, "Test");
-        assert_eq!(forwarded, false);
-        assert_eq!(cut, false);
+        let SimplifiedText {
+            text,
+            is_forwarded,
+            is_cut,
+            top_quote,
+            footer,
+        } = simplify(dehtml, false);
+        assert_eq!(text, "Test");
+        assert_eq!(is_forwarded, false);
+        assert_eq!(is_cut, false);
         assert_eq!(top_quote.as_deref(), Some("test"));
         assert_eq!(footer, None);
     }

src/download.rs

@@ -280,7 +280,7 @@ mod tests {
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_download_limit() -> Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -303,7 +303,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_update_download_state() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat = t.create_chat_with_contact("Bob", "bob@example.org").await;
@@ -328,7 +328,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_partial_receive_imf() -> Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -376,7 +376,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_partial_download_and_ephemeral() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = t

src/e2ee.rs

@@ -315,7 +315,7 @@ async fn decrypt_if_autocrypt_message(
 ///
 /// Returns `None` if the part is not a Multipart/Signed part, otherwise retruns the set of key
 /// fingerprints for which there is a valid signature.
-async fn validate_detached_signature(
+fn validate_detached_signature(
     mail: &ParsedMail<'_>,
     public_keyring_for_validate: &Keyring<SignedPublicKey>,
 ) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
@@ -328,7 +328,7 @@ async fn validate_detached_signature(
         let content = first_part.raw_bytes;
         let signature = second_part.get_body_raw()?;
         let ret_valid_signatures =
-            pgp::pk_validate(content, &signature, public_keyring_for_validate).await?;
+            pgp::pk_validate(content, &signature, public_keyring_for_validate)?;
 
         Ok(Some((content.to_vec(), ret_valid_signatures)))
     } else {
@@ -352,7 +352,7 @@ async fn decrypt_part(
         // If decrypted part is a multipart/signed, then there is a detached signature.
         let decrypted_part = mailparse::parse_mail(&plain)?;
         if let Some((content, valid_detached_signatures)) =
-            validate_detached_signature(&decrypted_part, &public_keyring_for_validate).await?
+            validate_detached_signature(&decrypted_part, &public_keyring_for_validate)?
         {
             return Ok(Some((content, valid_detached_signatures)));
         } else {
@@ -422,7 +422,7 @@ mod tests {
     mod ensure_secret_key_exists {
         use super::*;
 
-        #[async_std::test]
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
         async fn test_prexisting() {
             let t = TestContext::new_alice().await;
             assert_eq!(
@@ -431,7 +431,7 @@ mod tests {
             );
         }
 
-        #[async_std::test]
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
         async fn test_not_configured() {
             let t = TestContext::new().await;
             assert!(ensure_secret_key_exists(&t).await.is_err());
@@ -480,7 +480,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert_eq!(has_decrypted_pgp_armor(data), false);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -588,7 +588,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         vec![(Some(peerstate), addr)]
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_should_encrypt() {
         let t = TestContext::new_alice().await;
         let encrypt_helper = EncryptHelper::new(&t).await.unwrap();
@@ -615,7 +615,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mixed_up_mime() -> Result<()> {
         // "Mixed Up" mail as received when sending an encrypted
         // message using Delta Chat Desktop via ProtonMail IMAP/SMTP

src/ephemeral.rs

@@ -62,9 +62,9 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{ensure, Context as _, Result};
-use async_std::channel::Receiver;
-use async_std::future::timeout;
+use async_channel::Receiver;
 use serde::{Deserialize, Serialize};
+use tokio::time::timeout;
 
 use crate::chat::{send_msg, ChatId};
 use crate::constants::{DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH};
@@ -581,7 +581,7 @@ mod tests {
         dc_tools::IsNoneOrEmpty,
     };
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_stock_ephemeral_messages() {
         let context = TestContext::new().await;
 
@@ -711,7 +711,7 @@ mod tests {
     }
 
     /// Test enabling and disabling ephemeral timer remotely.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_enable_disable() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -743,7 +743,7 @@ mod tests {
     }
 
     /// Test that timer is enabled even if the message explicitly enabling the timer is lost.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_enable_lost() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -785,7 +785,7 @@ mod tests {
 
     /// Test that Alice replying to the chat without a timer at the same time as Bob enables the
     /// timer does not result in disabling the timer on the Bob's side.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_timer_rollback() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -859,7 +859,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_delete_msgs() -> Result<()> {
         let t = TestContext::new_alice().await;
         let self_chat = t.get_self_chat().await;
@@ -985,7 +985,7 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_delete_expired_imap_messages() -> Result<()> {
         let t = TestContext::new_alice().await;
         const HOUR: i64 = 60 * 60;
@@ -1096,7 +1096,7 @@ mod tests {
     }
 
     // Regression test for a bug in the timer rollback protection.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_timer_references() -> Result<()> {
         let alice = TestContext::new_alice().await;
 

src/error.rs

@@ -1,31 +0,0 @@
-//! # Error handling
-
-#[macro_export]
-macro_rules! ensure_eq {
-    ($left:expr, $right:expr) => ({
-        match (&$left, &$right) {
-            (left_val, right_val) => {
-                if !(*left_val == *right_val) {
-                    bail!(r#"assertion failed: `(left == right)`
-  left: `{:?}`,
- right: `{:?}`"#, left_val, right_val)
-                }
-            }
-        }
-    });
-    ($left:expr, $right:expr,) => ({
-        ensure_eq!($left, $right)
-    });
-    ($left:expr, $right:expr, $($arg:tt)+) => ({
-        match (&($left), &($right)) {
-            (left_val, right_val) => {
-                if !(*left_val == *right_val) {
-                    bail!(r#"assertion failed: `(left == right)`
-  left: `{:?}`,
- right: `{:?}`: {}"#, left_val, right_val,
-                           format_args!($($arg)+))
-                }
-            }
-        }
-    });
-}

src/events.rs

@@ -1,7 +1,8 @@
 //! # Events specification.
 
-use async_std::channel::{self, Receiver, Sender, TrySendError};
-use async_std::path::PathBuf;
+use std::path::PathBuf;
+
+use async_channel::{self as channel, Receiver, Sender, TrySendError};
 
 use crate::chat::ChatId;
 use crate::contact::ContactId;
@@ -9,7 +10,8 @@ use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
 use crate::webxdc::StatusUpdateSerial;
 
-#[derive(Debug)]
+/// Event channel.
+#[derive(Debug, Clone)]
 pub struct Events {
     receiver: Receiver<Event>,
     sender: Sender<Event>,
@@ -17,13 +19,17 @@ pub struct Events {
 
 impl Default for Events {
     fn default() -> Self {
-        let (sender, receiver) = channel::bounded(1_000);
-
-        Self { receiver, sender }
+        Self::new()
     }
 }
 
 impl Events {
+    pub fn new() -> Self {
+        let (sender, receiver) = channel::bounded(1_000);
+
+        Self { receiver, sender }
+    }
+
     pub fn emit(&self, event: Event) {
         match self.sender.try_send(event) {
             Ok(()) => {}
@@ -56,23 +62,18 @@ impl Events {
 ///
 /// [`Context`]: crate::context::Context
 /// [`Context::get_event_emitter`]: crate::context::Context::get_event_emitter
-/// [`Stream`]: async_std::stream::Stream
+/// [`Stream`]: futures::stream::Stream
 #[derive(Debug, Clone)]
 pub struct EventEmitter(Receiver<Event>);
 
 impl EventEmitter {
-    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
-    pub fn recv_sync(&self) -> Option<Event> {
-        async_std::task::block_on(self.recv())
-    }
-
     /// Async recv of an event. Return `None` if the `Sender` has been droped.
     pub async fn recv(&self) -> Option<Event> {
         self.0.recv().await.ok()
     }
 }
 
-impl async_std::stream::Stream for EventEmitter {
+impl futures::stream::Stream for EventEmitter {
     type Item = Event;
 
     fn poll_next(

src/format_flowed.rs

@@ -51,14 +51,26 @@ fn format_line_flowed(line: &str, prefix: &str) -> String {
     result + &buffer
 }
 
-fn format_flowed_prefix(text: &str, prefix: &str) -> String {
+/// Returns text formatted according to RFC 3767 (format=flowed).
+///
+/// This function accepts text separated by LF, but returns text
+/// separated by CRLF.
+///
+/// RFC 2646 technique is used to insert soft line breaks, so DelSp
+/// SHOULD be set to "no" when sending.
+pub(crate) fn format_flowed(text: &str) -> String {
     let mut result = String::new();
 
     for line in text.split('\n') {
         if !result.is_empty() {
             result += "\r\n";
         }
-        let line = line.trim_end();
+
+        let line_no_prefix = line.strip_prefix('>');
+        let is_quote = line_no_prefix.is_some();
+        let line = line_no_prefix.unwrap_or(line).trim();
+        let prefix = if is_quote { "> " } else { "" };
+
         if prefix.len() + line.len() > 78 {
             result += &format_line_flowed(line, prefix);
         } else {
@@ -70,23 +82,23 @@ fn format_flowed_prefix(text: &str, prefix: &str) -> String {
             result += line;
         }
     }
-    result
-}
 
-/// Returns text formatted according to RFC 3767 (format=flowed).
-///
-/// This function accepts text separated by LF, but returns text
-/// separated by CRLF.
-///
-/// RFC 2646 technique is used to insert soft line breaks, so DelSp
-/// SHOULD be set to "no" when sending.
-pub fn format_flowed(text: &str) -> String {
-    format_flowed_prefix(text, "")
+    result
 }
 
 /// Same as format_flowed(), but adds "> " prefix to each line.
 pub fn format_flowed_quote(text: &str) -> String {
-    format_flowed_prefix(text, "> ")
+    let mut result = String::new();
+
+    for line in text.split('\n') {
+        if !result.is_empty() {
+            result += "\n";
+        }
+        result += "> ";
+        result += line;
+    }
+
+    format_flowed(&result)
 }
 
 /// Joins lines in format=flowed text.
@@ -129,6 +141,7 @@ pub fn unformat_flowed(text: &str, delsp: bool) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_format_flowed() {
@@ -144,18 +157,18 @@ mod tests {
                         client and enter the setup code presented on the generating device.";
         assert_eq!(format_flowed(text), expected);
 
-        let text = "> Not a quote";
-        assert_eq!(format_flowed(text), " > Not a quote");
+        let text = "> A quote";
+        assert_eq!(format_flowed(text), "> A quote");
 
         // Test space stuffing of wrapped lines
         let text = "> This is the Autocrypt Setup Message used to transfer your key between clients.\n\
                     >                               \n\
                     > To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
-        let expected = "\x20> This is the Autocrypt Setup Message used to transfer your key between \r\n\
-                        clients.\r\n\
-                        \x20>\r\n\
-                        \x20> To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
-                        client and enter the setup code presented on the generating device.";
+        let expected = "> This is the Autocrypt Setup Message used to transfer your key between \r\n\
+                        > clients.\r\n\
+                        > \r\n\
+                        > To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        > client and enter the setup code presented on the generating device.";
         assert_eq!(format_flowed(text), expected);
     }
 
@@ -175,6 +188,10 @@ mod tests {
         let expected = "> this is a quoted line";
         assert_eq!(format_flowed_quote(quote), expected);
 
+        let quote = "first quoted line\nsecond quoted line";
+        let expected = "> first quoted line\r\n> second quoted line";
+        assert_eq!(format_flowed_quote(quote), expected);
+
         let quote = "> foo bar baz";
         let expected = "> > foo bar baz";
         assert_eq!(format_flowed_quote(quote), expected);
@@ -185,4 +202,25 @@ mod tests {
             > unwrapped on the receiver";
         assert_eq!(format_flowed_quote(quote), expected);
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_send_quotes() -> anyhow::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat = alice.create_chat(&bob).await;
+
+        let sent = alice.send_text(chat.id, "> First quote").await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some("> First quote"));
+        assert!(received.quoted_text().is_none());
+        assert!(received.quoted_message(&bob).await?.is_none());
+
+        let sent = alice.send_text(chat.id, "> Second quote").await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some("> Second quote"));
+        assert!(received.quoted_text().is_none());
+        assert!(received.quoted_message(&bob).await?.is_none());
+
+        Ok(())
+    }
 }

src/html.rs

@@ -64,7 +64,7 @@ enum MimeMultipartType {
 
 /// Function takes a content type from a ParsedMail structure
 /// and checks and returns the rough mime-type.
-async fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
+fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
     let mimetype = ctype.mimetype.to_lowercase();
     if mimetype.starts_with("multipart") && ctype.params.get("boundary").is_some() {
         MimeMultipartType::Multiple
@@ -122,7 +122,7 @@ impl HtmlMsgParser {
     ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
         // Boxed future to deal with recursion
         async move {
-            match get_mime_multipart_type(&mail.ctype).await {
+            match get_mime_multipart_type(&mail.ctype) {
                 MimeMultipartType::Multiple => {
                     for cur_data in mail.subparts.iter() {
                         self.collect_texts_recursive(context, cur_data).await?
@@ -178,7 +178,7 @@ impl HtmlMsgParser {
     ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
         // Boxed future to deal with recursion
         async move {
-            match get_mime_multipart_type(&mail.ctype).await {
+            match get_mime_multipart_type(&mail.ctype) {
                 MimeMultipartType::Multiple => {
                     for cur_data in mail.subparts.iter() {
                         self.cid_to_data_recursive(context, cur_data).await?;
@@ -198,7 +198,7 @@ impl HtmlMsgParser {
                     if mimetype.type_() == mime::IMAGE {
                         if let Some(cid) = mail.headers.get_header_value(HeaderDef::ContentId) {
                             if let Ok(cid) = parse_message_id(&cid) {
-                                if let Ok(replacement) = mimepart_to_data_url(mail).await {
+                                if let Ok(replacement) = mimepart_to_data_url(mail) {
                                     let re_string = format!(
                                         "(<img[^>]*src[^>]*=[^>]*)(cid:{})([^>]*>)",
                                         regex::escape(&cid)
@@ -233,7 +233,7 @@ impl HtmlMsgParser {
 }
 
 /// Convert a mime part to a data: url as defined in [RFC 2397](https://tools.ietf.org/html/rfc2397).
-async fn mimepart_to_data_url(mail: &mailparse::ParsedMail<'_>) -> Result<String> {
+fn mimepart_to_data_url(mail: &mailparse::ParsedMail<'_>) -> Result<String> {
     let data = mail.get_body_raw()?;
     let data = base64::encode(&data);
     Ok(format!("data:{};base64,{}", mail.ctype.mimetype, data))
@@ -267,7 +267,7 @@ impl MsgId {
 ///
 /// Used on forwarding messages to avoid leaking the original mime structure
 /// and also to avoid sending too much, maybe large data.
-pub async fn new_html_mimepart(html: String) -> PartBuilder {
+pub fn new_html_mimepart(html: String) -> PartBuilder {
     PartBuilder::new()
         .content_type(&"text/html; charset=utf-8".parse::<mime::Mime>().unwrap())
         .body(html)
@@ -284,7 +284,7 @@ mod tests {
     use crate::message::{MessengerMessage, Viewtype};
     use crate::test_utils::TestContext;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_plain_unspecified() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
@@ -300,7 +300,7 @@ This message does not have Content-Type nor Subject.<br/>
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_plain_iso88591() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_plain_iso88591.eml");
@@ -316,7 +316,7 @@ message with a non-UTF-8 encoding: äöüßÄÖÜ<br/>
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_plain_flowed() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_plain_flowed.eml");
@@ -336,7 +336,7 @@ and will be wrapped as usual.<br/>
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_alt_plain() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
@@ -355,7 +355,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_html() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_html.eml");
@@ -373,7 +373,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_alt_html() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_alt_html.eml");
@@ -388,7 +388,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_alt_plain_html() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
@@ -405,7 +405,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_apple_cid_jpg() {
         // load raw mime html-data with related image-part (cid:)
         // and make sure, Content-Id has angle-brackets that are removed correctly.
@@ -424,14 +424,14 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         assert!(!parser.html.contains("cid:"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_html_invalid_msgid() {
         let t = TestContext::new().await;
         let msg_id = MsgId::new(100);
         assert!(msg_id.get_html(&t).await.is_err())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_html_forwarding() {
         // alice receives a non-delta html-message
         let alice = TestContext::new_alice().await;
@@ -478,7 +478,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         assert!(html.contains("this is <b>html</b>"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_html_forwarding_encrypted() {
         // Alice receives a non-delta html-message
         // (`ShowEmails=1` lets Alice actually receive non-delta messages for known contacts,
@@ -515,7 +515,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         assert!(html.contains("this is <b>html</b>"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_html() {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -547,7 +547,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         assert!(html.contains("<b>html</b> text"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_cp1252_html() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await?;

src/imap.rs

@@ -11,20 +11,19 @@ use std::{
 };
 
 use anyhow::{bail, format_err, Context as _, Result};
+use async_channel::Receiver;
 use async_imap::types::{
     Fetch, Flag, Mailbox, Name, NameAttribute, Quota, QuotaRoot, UnsolicitedResponse,
 };
-use async_std::channel::Receiver;
-use async_std::prelude::*;
+use futures::StreamExt;
 use num_traits::FromPrimitive;
 
 use crate::chat::{self, ChatId, ChatIdBlocked};
 use crate::config::Config;
 use crate::constants::{
     Blocked, Chattype, ShowEmails, DC_FETCH_EXISTING_MSGS_COUNT, DC_FOLDERS_CONFIGURED_VERSION,
-    DC_LP_AUTH_OAUTH2,
 };
-use crate::contact::ContactId;
+use crate::contact::{normalize_name, Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_receive_imf::{
     dc_receive_imf_inner, from_field_to_contact_id, get_prefetch_parent_message, ReceivedMsg,
@@ -154,7 +153,6 @@ struct ImapConfig {
     pub lp: ServerLoginParam,
     pub socks5_config: Option<Socks5Config>,
     pub strict_tls: bool,
-    pub oauth2: bool,
     pub selected_folder: Option<String>,
     pub selected_mailbox: Option<Mailbox>,
     pub selected_folder_needs_expunge: bool,
@@ -243,7 +241,6 @@ impl Imap {
         lp: &ServerLoginParam,
         socks5_config: Option<Socks5Config>,
         addr: &str,
-        oauth2: bool,
         provider_strict_tls: bool,
         idle_interrupt: Receiver<InterruptInfo>,
     ) -> Result<Self> {
@@ -262,7 +259,6 @@ impl Imap {
             lp: lp.clone(),
             socks5_config,
             strict_tls,
-            oauth2,
             selected_folder: None,
             selected_mailbox: None,
             selected_folder_needs_expunge: false,
@@ -301,7 +297,6 @@ impl Imap {
             &param.imap,
             param.socks5_config.clone(),
             &param.addr,
-            param.server_flags & DC_LP_AUTH_OAUTH2 != 0,
             param
                 .provider
                 .map_or(param.socks5_config.is_some(), |provider| {
@@ -334,7 +329,7 @@ impl Imap {
 
         self.connectivity.set_connecting(context).await;
 
-        let oauth2 = self.config.oauth2;
+        let oauth2 = self.config.lp.oauth2;
 
         let connection_res: Result<Client> = if self.config.lp.security == Socket::Starttls
             || self.config.lp.security == Socket::Plain
@@ -905,6 +900,42 @@ impl Imap {
         Ok(read_cnt > 0)
     }
 
+    /// Read the recipients from old emails sent by the user and add them as contacts.
+    /// This way, we can already offer them some email addresses they can write to.
+    ///
+    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
+    /// and show them in the chat list.
+    pub(crate) async fn fetch_existing_msgs(&mut self, context: &Context) -> Result<()> {
+        if context.get_config_bool(Config::Bot).await? {
+            return Ok(()); // Bots don't want those messages
+        }
+        self.prepare(context).await.context("could not connect")?;
+
+        add_all_recipients_as_contacts(context, self, Config::ConfiguredSentboxFolder).await;
+        add_all_recipients_as_contacts(context, self, Config::ConfiguredMvboxFolder).await;
+        add_all_recipients_as_contacts(context, self, Config::ConfiguredInboxFolder).await;
+
+        if context.get_config_bool(Config::FetchExistingMsgs).await? {
+            for config in &[
+                Config::ConfiguredMvboxFolder,
+                Config::ConfiguredInboxFolder,
+                Config::ConfiguredSentboxFolder,
+            ] {
+                if let Some(folder) = context.get_config(*config).await? {
+                    self.fetch_new_messages(context, &folder, false, true)
+                        .await
+                        .context("could not fetch messages")?;
+                }
+            }
+        }
+
+        info!(context, "Done fetching existing messages.");
+        context
+            .set_config_bool(Config::FetchedExistingMsgs, true)
+            .await?;
+        Ok(())
+    }
+
     /// Deletes batch of messages identified by their UID from the currently
     /// selected folder.
     async fn delete_message_batch(
@@ -1891,7 +1922,7 @@ fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
 
 fn get_folder_meaning(folder_name: &Name) -> FolderMeaning {
     for attr in folder_name.attributes() {
-        if let NameAttribute::Custom(ref label) = attr {
+        if let NameAttribute::Extension(ref label) = attr {
             match label.as_ref() {
                 "\\Trash" => return FolderMeaning::Other,
                 "\\Sent" => return FolderMeaning::Sent,
@@ -2287,6 +2318,50 @@ impl std::fmt::Display for UidRange {
         }
     }
 }
+async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
+    let mailbox = if let Ok(Some(m)) = context.get_config(folder).await {
+        m
+    } else {
+        return;
+    };
+    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
+        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+        warn!(context, "Could not select {}: {:#}", mailbox, e);
+        return;
+    }
+    match imap.get_all_recipients(context).await {
+        Ok(contacts) => {
+            let mut any_modified = false;
+            for contact in contacts {
+                let display_name_normalized = contact
+                    .display_name
+                    .as_ref()
+                    .map(|s| normalize_name(s))
+                    .unwrap_or_default();
+
+                match Contact::add_or_lookup(
+                    context,
+                    &display_name_normalized,
+                    &contact.addr,
+                    Origin::OutgoingTo,
+                )
+                .await
+                {
+                    Ok((_, modified)) => {
+                        if modified != Modifier::None {
+                            any_modified = true;
+                        }
+                    }
+                    Err(e) => warn!(context, "Could not add recipient: {}", e),
+                }
+            }
+            if any_modified {
+                context.emit_event(EventType::ContactsChanged(None));
+            }
+        }
+        Err(e) => warn!(context, "Could not add recipients: {}", e),
+    };
+}
 
 #[cfg(test)]
 mod tests {
@@ -2313,7 +2388,7 @@ mod tests {
         assert_eq!(get_folder_meaning_by_name("SPAM"), FolderMeaning::Spam);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_uid_next_validity() {
         let t = TestContext::new_alice().await;
         assert_eq!(get_uid_next(&t.ctx, "Inbox").await.unwrap(), 0);
@@ -2495,7 +2570,7 @@ mod tests {
         ("Spam", true, true, "DeltaChat"),
     ];
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_target_folder_incoming_accepted() -> Result<()> {
         for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
             check_target_folder_combination(
@@ -2512,7 +2587,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_target_folder_incoming_request() -> Result<()> {
         for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_REQUEST {
             check_target_folder_combination(
@@ -2529,7 +2604,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_target_folder_outgoing() -> Result<()> {
         // Test outgoing emails
         for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
@@ -2547,7 +2622,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_target_folder_setupmsg() -> Result<()> {
         // Test setupmessages
         for (folder, mvbox_move, chat_msg, _expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
@@ -2565,7 +2640,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_imap_search_command() -> Result<()> {
         let t = TestContext::new_alice().await;
         assert_eq!(

src/imap/client.rs

@@ -8,7 +8,7 @@ use anyhow::{Context as _, Result};
 use async_imap::Client as ImapClient;
 
 use async_smtp::ServerAddress;
-use async_std::net::{self, TcpStream};
+use tokio::net::{self, TcpStream};
 
 use super::session::Session;
 use crate::login_param::{dc_build_tls, Socks5Config};

src/imap/idle.rs

@@ -2,7 +2,7 @@ use super::Imap;
 
 use anyhow::{bail, Context as _, Result};
 use async_imap::extensions::idle::IdleResponse;
-use async_std::prelude::*;
+use futures_lite::FutureExt;
 use std::time::{Duration, SystemTime};
 
 use crate::{context::Context, scheduler::InterruptInfo};
@@ -87,9 +87,7 @@ impl Imap {
                 }
             }
 
-            let session = handle
-                .done()
-                .timeout(Duration::from_secs(15))
+            let session = tokio::time::timeout(Duration::from_secs(15), handle.done())
                 .await?
                 .context("IMAP IDLE protocol timed out")?;
             self.session = Some(Session { inner: session });
@@ -121,7 +119,7 @@ impl Imap {
 
         // check every minute if there are new messages
         // TODO: grow sleep durations / make them more flexible
-        let mut interval = async_std::stream::interval(Duration::from_secs(60));
+        let mut interval = tokio::time::interval(Duration::from_secs(60));
 
         enum Event {
             Tick,
@@ -131,7 +129,7 @@ impl Imap {
         let info = loop {
             use futures::future::FutureExt;
             match interval
-                .next()
+                .tick()
                 .map(|_| Event::Tick)
                 .race(
                     self.idle_interrupt

src/imap/scan_folders.rs

@@ -1,14 +1,13 @@
 use std::{collections::BTreeMap, time::Instant};
 
 use anyhow::{Context as _, Result};
+use futures::stream::StreamExt;
 
 use crate::config::Config;
 use crate::imap::Imap;
 use crate::log::LogExt;
 use crate::{context::Context, imap::FolderMeaning};
 
-use async_std::stream::StreamExt;
-
 use super::{get_folder_meaning, get_folder_meaning_by_name};
 
 impl Imap {
@@ -104,7 +103,7 @@ impl Imap {
         let list = session
             .list(Some(""), Some("*"))
             .await?
-            .filter_map(|f| f.ok_or_log_msg(context, "list_folders() can't get folder"));
+            .filter_map(|f| async { f.ok_or_log_msg(context, "list_folders() can't get folder") });
         Ok(list.collect().await)
     }
 }

src/imap/session.rs

@@ -2,8 +2,8 @@ use std::ops::{Deref, DerefMut};
 
 use async_imap::Session as ImapSession;
 use async_native_tls::TlsStream;
-use async_std::net::TcpStream;
 use fast_socks5::client::Socks5Stream;
+use tokio::net::TcpStream;
 
 #[derive(Debug)]
 pub(crate) struct Session {
@@ -11,7 +11,7 @@ pub(crate) struct Session {
 }
 
 pub(crate) trait SessionStream:
-    async_std::io::Read + async_std::io::Write + Unpin + Send + Sync + std::fmt::Debug
+    tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + Sync + std::fmt::Debug
 {
 }
 

src/imex.rs

@@ -2,16 +2,15 @@
 
 use std::any::Any;
 use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
 
 use ::pgp::types::KeyTrait;
 use anyhow::{bail, ensure, format_err, Context as _, Result};
-use async_std::{
-    fs::{self, File},
-    path::{Path, PathBuf},
-    prelude::*,
-};
-use async_tar::Archive;
+use futures::{StreamExt, TryStreamExt};
+use futures_lite::FutureExt;
 use rand::{thread_rng, Rng};
+use tokio::fs::{self, File};
+use tokio_tar::Archive;
 
 use crate::blob::BlobObject;
 use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
@@ -19,8 +18,8 @@ use crate::config::Config;
 use crate::contact::ContactId;
 use crate::context::Context;
 use crate::dc_tools::{
-    dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
-    dc_open_file_std, dc_read_file, dc_write_file, time, EmailAddress,
+    dc_create_folder, dc_delete_file, dc_get_filesuffix_lc, dc_open_file_std, dc_read_file,
+    dc_write_file, time, EmailAddress,
 };
 use crate::e2ee;
 use crate::events::EventType;
@@ -86,62 +85,45 @@ pub async fn imex(
 ) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
 
-    let res = async {
-        let success = imex_inner(context, what, path, passphrase).await;
-        match success {
-            Ok(()) => {
-                info!(context, "IMEX successfully completed");
-                context.emit_event(EventType::ImexProgress(1000));
-                Ok(())
-            }
-            Err(err) => {
-                cleanup_aborted_imex(context, what).await;
-                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-                error!(context, "{:#}", err);
-                context.emit_event(EventType::ImexProgress(0));
-                bail!("IMEX FAILED to complete: {}", err);
-            }
-        }
-    }
-    .race(async {
-        cancel.recv().await.ok();
-        cleanup_aborted_imex(context, what).await;
-        Err(format_err!("canceled"))
-    })
-    .await;
+    let res = imex_inner(context, what, path, passphrase)
+        .race(async {
+            cancel.recv().await.ok();
+            Err(format_err!("canceled"))
+        })
+        .await;
 
     context.free_ongoing().await;
 
+    if let Err(err) = res.as_ref() {
+        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+        error!(context, "IMEX failed to complete: {:#}", err);
+        context.emit_event(EventType::ImexProgress(0));
+    } else {
+        info!(context, "IMEX successfully completed");
+        context.emit_event(EventType::ImexProgress(1000));
+    }
+
     res
 }
 
-async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
-    if what == ImexMode::ImportBackup {
-        dc_delete_file(context, context.get_dbfile()).await;
-        dc_delete_files_in_dir(context, context.get_blobdir()).await;
-    }
-}
-
 /// Returns the filename of the backup found (otherwise an error)
 pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result<String> {
-    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
+    let mut dir_iter = tokio::fs::read_dir(dir_name).await?;
     let mut newest_backup_name = "".to_string();
     let mut newest_backup_path: Option<PathBuf> = None;
 
-    while let Some(dirent) = dir_iter.next().await {
-        if let Ok(dirent) = dirent {
-            let path = dirent.path();
-            let name = dirent.file_name();
-            let name: String = name.to_string_lossy().into();
-            if name.starts_with("delta-chat")
-                && name.ends_with(".tar")
-                && (newest_backup_name.is_empty() || name > newest_backup_name)
-            {
-                // We just use string comparison to determine which backup is newer.
-                // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
-                newest_backup_path = Some(path);
-                newest_backup_name = name;
-            }
+    while let Ok(Some(dirent)) = dir_iter.next_entry().await {
+        let path = dirent.path();
+        let name = dirent.file_name();
+        let name: String = name.to_string_lossy().into();
+        if name.starts_with("delta-chat")
+            && name.ends_with(".tar")
+            && (newest_backup_name.is_empty() || name > newest_backup_name)
+        {
+            // We just use string comparison to determine which backup is newer.
+            // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
+            newest_backup_path = Some(path);
+            newest_backup_name = name;
         }
     }
 
@@ -192,7 +174,7 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
     info!(context, "Wait for setup message being sent ...",);
     while !context.shall_stop_ongoing().await {
-        async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
         if let Ok(msg) = Message::load_from_db(context, msg_id).await {
             if msg.is_sent() {
                 info!(context, "... setup message sent.",);
@@ -461,7 +443,7 @@ async fn import_backup(
 
     context.sql.config_cache.write().await.clear();
 
-    let archive = Archive::new(backup_file);
+    let mut archive = Archive::new(backup_file);
 
     let mut entries = archive.entries()?;
     let mut last_progress = 0;
@@ -492,7 +474,7 @@ async fn import_backup(
             // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
             f.unpack_in(context.get_blobdir()).await?;
             let from_path = context.get_blobdir().join(f.path()?);
-            if from_path.is_file().await {
+            if from_path.is_file() {
                 if let Some(name) = from_path.file_name() {
                     fs::rename(&from_path, context.get_blobdir().join(name)).await?;
                 } else {
@@ -514,10 +496,7 @@ async fn import_backup(
 /// Returns Ok((temp_db_path, temp_path, dest_path)) on success. Unencrypted database can be
 /// written to temp_db_path. The backup can then be written to temp_path. If the backup succeeded,
 /// it can be renamed to dest_path. This guarantees that the backup is complete.
-async fn get_next_backup_path(
-    folder: &Path,
-    backup_time: i64,
-) -> Result<(PathBuf, PathBuf, PathBuf)> {
+fn get_next_backup_path(folder: &Path, backup_time: i64) -> Result<(PathBuf, PathBuf, PathBuf)> {
     let folder = PathBuf::from(folder);
     let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
         // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
@@ -535,7 +514,7 @@ async fn get_next_backup_path(
         let mut destfile = folder.clone();
         destfile.push(format!("{}-{:02}.tar", stem, i));
 
-        if !tempdbfile.exists().await && !tempfile.exists().await && !destfile.exists().await {
+        if !tempdbfile.exists() && !tempfile.exists() && !destfile.exists() {
             return Ok((tempdbfile, tempfile, destfile));
         }
     }
@@ -545,7 +524,7 @@ async fn get_next_backup_path(
 async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
-    let (temp_db_path, temp_path, dest_path) = get_next_backup_path(dir, now).await?;
+    let (temp_db_path, temp_path, dest_path) = get_next_backup_path(dir, now)?;
     let _d1 = DeleteOnDrop(temp_db_path.clone());
     let _d2 = DeleteOnDrop(temp_path.clone());
 
@@ -599,7 +578,8 @@ impl Drop for DeleteOnDrop {
     fn drop(&mut self) {
         let file = self.0.clone();
         // Not using dc_delete_file() here because it would send a DeletedBlobFile event
-        async_std::task::block_on(fs::remove_file(file)).ok();
+        // Hack to avoid panic in nested runtime calls of tokio
+        std::fs::remove_file(file).ok();
     }
 }
 
@@ -610,19 +590,21 @@ async fn export_backup_inner(
 ) -> Result<()> {
     let file = File::create(temp_path).await?;
 
-    let mut builder = async_tar::Builder::new(file);
+    let mut builder = tokio_tar::Builder::new(file);
 
     builder
         .append_path_with_name(temp_db_path, DBFILE_BACKUP_NAME)
         .await?;
 
-    let read_dir: Vec<_> = fs::read_dir(context.get_blobdir()).await?.collect().await;
+    let read_dir: Vec<_> =
+        tokio_stream::wrappers::ReadDirStream::new(fs::read_dir(context.get_blobdir()).await?)
+            .try_collect()
+            .await?;
     let count = read_dir.len();
     let mut written_files = 0;
 
     let mut last_progress = 0;
     for entry in read_dir.into_iter() {
-        let entry = entry?;
         let name = entry.file_name();
         if !entry.file_type().await?.is_file() {
             warn!(
@@ -663,9 +645,9 @@ async fn import_self_keys(context: &Context, dir: &Path) -> Result<()> {
     let mut imported_cnt = 0;
 
     let dir_name = dir.to_string_lossy();
-    let mut dir_handle = async_std::fs::read_dir(&dir).await?;
-    while let Some(entry) = dir_handle.next().await {
-        let entry_fn = entry?.file_name();
+    let mut dir_handle = tokio::fs::read_dir(&dir).await?;
+    while let Ok(Some(entry)) = dir_handle.next_entry().await {
+        let entry_fn = entry.file_name();
         let name_f = entry_fn.to_string_lossy();
         let path_plus_name = dir.join(&entry_fn);
         match dc_get_filesuffix_lc(&name_f) {
@@ -815,7 +797,7 @@ mod tests {
 
     use ::pgp::armor::BlockType;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_render_setup_file() {
         let t = TestContext::new_alice().await;
         let msg = render_setup_file(&t, "hello").await.unwrap();
@@ -832,7 +814,7 @@ mod tests {
         assert!(msg.contains("-----END PGP MESSAGE-----\n"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_render_setup_file_newline_replace() {
         let t = TestContext::new_alice().await;
         t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
@@ -843,7 +825,7 @@ mod tests {
         assert!(msg.contains("<p>hello<br>there</p>"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_setup_code() {
         let t = TestContext::new().await;
         let setupcode = create_setup_code(&t);
@@ -858,7 +840,7 @@ mod tests {
         assert_eq!(setupcode.chars().nth(39).unwrap(), '-');
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_public_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().public;
@@ -868,12 +850,12 @@ mod tests {
             .is_ok());
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
         let filename = format!("{}/public-key-default.asc", blobdir);
-        let bytes = async_std::fs::read(&filename).await.unwrap();
+        let bytes = tokio::fs::read(&filename).await.unwrap();
 
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_private_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().secret;
@@ -883,12 +865,12 @@ mod tests {
             .is_ok());
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
         let filename = format!("{}/private-key-default.asc", blobdir);
-        let bytes = async_std::fs::read(&filename).await.unwrap();
+        let bytes = tokio::fs::read(&filename).await.unwrap();
 
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_and_import_key() {
         let context = TestContext::new_alice().await;
         let blobdir = context.ctx.get_blobdir();
@@ -902,7 +884,7 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_and_import_backup() -> Result<()> {
         let backup_dir = tempfile::tempdir().unwrap();
 
@@ -911,26 +893,32 @@ mod tests {
 
         let context2 = TestContext::new().await;
         assert!(!context2.is_configured().await?);
-        assert!(has_backup(&context2, backup_dir.path().as_ref())
-            .await
-            .is_err());
+        assert!(has_backup(&context2, backup_dir.path()).await.is_err());
 
         // export from context1
-        assert!(imex(
-            &context1,
-            ImexMode::ExportBackup,
-            backup_dir.path().as_ref(),
-            None,
-        )
-        .await
-        .is_ok());
+        assert!(
+            imex(&context1, ImexMode::ExportBackup, backup_dir.path(), None)
+                .await
+                .is_ok()
+        );
         let _event = context1
             .evtracker
             .get_matching(|evt| matches!(evt, EventType::ImexProgress(1000)))
             .await;
 
         // import to context2
-        let backup = has_backup(&context2, backup_dir.path().as_ref()).await?;
+        let backup = has_backup(&context2, backup_dir.path()).await?;
+
+        // Import of unencrypted backup with incorrect "foobar" backup passphrase fails.
+        assert!(imex(
+            &context2,
+            ImexMode::ImportBackup,
+            backup.as_ref(),
+            Some("foobar".to_string())
+        )
+        .await
+        .is_err());
+
         assert!(
             imex(&context2, ImexMode::ImportBackup, backup.as_ref(), None)
                 .await
@@ -965,7 +953,7 @@ mod tests {
     const S_EM_SETUPCODE: &str = "1742-0185-6197-1303-7016-8412-3581-4441-0597";
     const S_EM_SETUPFILE: &str = include_str!("../test-data/message/stress.txt");
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_split_and_decrypt() {
         let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
         let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
@@ -988,20 +976,20 @@ mod tests {
         assert!(headers.get(HEADER_SETUPCODE).is_none());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_key_transfer() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
         let alice_clone = alice.clone();
-        let key_transfer_task = async_std::task::spawn(async move {
+        let key_transfer_task = tokio::task::spawn(async move {
             let ctx = alice_clone;
             initiate_key_transfer(&ctx).await
         });
 
         // Wait for the message to be added to the queue.
-        async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
         let sent = alice.pop_sent_msg().await;
-        let setup_code = key_transfer_task.await?;
+        let setup_code = key_transfer_task.await??;
 
         // Alice sets up a second device.
         let alice2 = TestContext::new().await;

src/job.rs

@@ -8,11 +8,8 @@ use anyhow::{Context as _, Result};
 use deltachat_derive::{FromSql, ToSql};
 use rand::{thread_rng, Rng};
 
-use crate::config::Config;
-use crate::contact::{normalize_name, Contact, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_tools::time;
-use crate::events::EventType;
 use crate::imap::Imap;
 use crate::param::Params;
 use crate::scheduler::InterruptInfo;
@@ -58,8 +55,6 @@ macro_rules! job_try {
 )]
 #[repr(u32)]
 pub enum Action {
-    FetchExistingMsgs = 110,
-
     // this is user initiated so it should have a fairly high priority
     UpdateRecentQuota = 140,
 
@@ -156,45 +151,6 @@ impl Job {
 
         Ok(())
     }
-
-    /// Read the recipients from old emails sent by the user and add them as contacts.
-    /// This way, we can already offer them some email addresses they can write to.
-    ///
-    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
-    /// and show them in the chat list.
-    async fn fetch_existing_msgs(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if job_try!(context.get_config_bool(Config::Bot).await) {
-            return Status::Finished(Ok(())); // Bots don't want those messages
-        }
-        if let Err(err) = imap.prepare(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
-        add_all_recipients_as_contacts(context, imap, Config::ConfiguredSentboxFolder).await;
-        add_all_recipients_as_contacts(context, imap, Config::ConfiguredMvboxFolder).await;
-        add_all_recipients_as_contacts(context, imap, Config::ConfiguredInboxFolder).await;
-
-        if job_try!(context.get_config_bool(Config::FetchExistingMsgs).await) {
-            for config in &[
-                Config::ConfiguredMvboxFolder,
-                Config::ConfiguredInboxFolder,
-                Config::ConfiguredSentboxFolder,
-            ] {
-                if let Some(folder) = job_try!(context.get_config(*config).await) {
-                    if let Err(e) = imap.fetch_new_messages(context, &folder, false, true).await {
-                        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-                        warn!(context, "Could not fetch messages, retrying: {:#}", e);
-                        return Status::RetryLater;
-                    };
-                }
-            }
-        }
-
-        info!(context, "Done fetching existing messages.");
-        Status::Finished(Ok(()))
-    }
-
     /// Synchronizes UIDs for all folders.
     async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.prepare(context).await {
@@ -250,51 +206,6 @@ pub async fn action_exists(context: &Context, action: Action) -> Result<bool> {
     Ok(exists)
 }
 
-async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
-    let mailbox = if let Ok(Some(m)) = context.get_config(folder).await {
-        m
-    } else {
-        return;
-    };
-    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
-        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-        warn!(context, "Could not select {}: {:#}", mailbox, e);
-        return;
-    }
-    match imap.get_all_recipients(context).await {
-        Ok(contacts) => {
-            let mut any_modified = false;
-            for contact in contacts {
-                let display_name_normalized = contact
-                    .display_name
-                    .as_ref()
-                    .map(|s| normalize_name(s))
-                    .unwrap_or_default();
-
-                match Contact::add_or_lookup(
-                    context,
-                    &display_name_normalized,
-                    &contact.addr,
-                    Origin::OutgoingTo,
-                )
-                .await
-                {
-                    Ok((_, modified)) => {
-                        if modified != Modifier::None {
-                            any_modified = true;
-                        }
-                    }
-                    Err(e) => warn!(context, "Could not add recipient: {}", e),
-                }
-            }
-            if any_modified {
-                context.emit_event(EventType::ContactsChanged(None));
-            }
-        }
-        Err(e) => warn!(context, "Could not add recipients: {}", e),
-    };
-}
-
 pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
 }
@@ -371,7 +282,6 @@ async fn perform_job_action(
 
     let try_res = match job.action {
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
-        Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
             Ok(status) => status,
             Err(err) => Status::Finished(Err(err)),
@@ -422,10 +332,7 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
 
     if delay_seconds == 0 {
         match action {
-            Action::ResyncFolders
-            | Action::FetchExistingMsgs
-            | Action::UpdateRecentQuota
-            | Action::DownloadMsg => {
+            Action::ResyncFolders | Action::UpdateRecentQuota | Action::DownloadMsg => {
                 info!(context, "interrupt: imap");
                 context.interrupt_inbox(InterruptInfo::new(false)).await;
             }
@@ -541,7 +448,7 @@ mod tests {
             .unwrap();
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_next_job_two() -> Result<()> {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
@@ -557,7 +464,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_next_job_one() -> Result<()> {
         let t = TestContext::new().await;
 

src/key.rs

@@ -11,6 +11,7 @@ use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use tokio::runtime::Handle;
 
 use crate::config::Config;
 use crate::constants::KeyGenType;
@@ -219,9 +220,10 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await?)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
-            let keypair =
-                async_std::task::spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
-                    .await?;
+            let keypair = Handle::current()
+                .spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
+                .await??;
+
             store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
             info!(
                 context,
@@ -397,8 +399,8 @@ mod tests {
     use super::*;
     use crate::test_utils::{alice_keypair, TestContext};
 
-    use async_std::sync::Arc;
     use once_cell::sync::Lazy;
+    use std::sync::Arc;
 
     static KEYPAIR: Lazy<KeyPair> = Lazy::new(alice_keypair);
 
@@ -520,7 +522,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(key, key2);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
         let t = TestContext::new_alice().await;
@@ -530,7 +532,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(alice.secret, seckey);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_self_generate_public() {
         let t = TestContext::new().await;
         t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
@@ -540,7 +542,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert!(key.is_ok());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_self_generate_secret() {
         let t = TestContext::new().await;
         t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
@@ -550,7 +552,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert!(key.is_ok());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_self_generate_concurrent() {
         use std::thread;
 
@@ -560,11 +562,19 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
             .unwrap();
         let thr0 = {
             let ctx = t.clone();
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+            thread::spawn(move || {
+                tokio::runtime::Runtime::new()
+                    .unwrap()
+                    .block_on(SignedPublicKey::load_self(&ctx))
+            })
         };
         let thr1 = {
             let ctx = t.clone();
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+            thread::spawn(move || {
+                tokio::runtime::Runtime::new()
+                    .unwrap()
+                    .block_on(SignedPublicKey::load_self(&ctx))
+            })
         };
         let res0 = thr0.join().unwrap();
         let res1 = thr1.join().unwrap();
@@ -577,7 +587,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(pubkey.primary_key, KEYPAIR.public.primary_key);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_save_self_key_twice() {
         // Saving the same key twice should result in only one row in
         // the keypairs table.

src/keyring.rs

@@ -76,7 +76,7 @@ mod tests {
         assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_keyring_load_self() {
         // new_self() implies load_self()
         let t = TestContext::new_alice().await;