Add broadcast lists to chattypes

This way the time to wait until next message can
be sent is always limited.

.github/mergeable.yml

@@ -22,5 +22,5 @@ mergeable:
       - do: checks
         status: 'action_required' 
         payload:
-          title: Changlog might need an update
+          title: Changelog might need an update
           summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on:
 
 env:
   RUSTFLAGS: -Dwarnings
-
+   
 jobs:
 
   fmt:
@@ -83,13 +83,13 @@ jobs:
             rust: 1.61.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.56.0
+          # Minimum Supported Rust Version = 1.56.1
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.56.0
+            rust: 1.56.1
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:
@@ -118,7 +118,7 @@ jobs:
 
     - name: install python
       if: ${{ matrix.python }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python }}
 
@@ -141,3 +141,18 @@ jobs:
         DCC_RS_DEV: ${{ github.workspace }}
       working-directory: python
       run: tox -e lint,mypy,doc,py3
+
+    - name: install pypy
+      if: ${{ matrix.python }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: 'pypy${{ matrix.python }}'
+
+    - name: run pypy tests
+      if: ${{ matrix.python }}
+      env:
+        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+        DCC_RS_TARGET: debug
+        DCC_RS_DEV: ${{ github.workspace }}
+      working-directory: python
+      run: tox -e pypy3

.github/workflows/node-package.yml

@@ -120,6 +120,9 @@ jobs:
       - name: install dependencies without running scripts
         run: |
           npm install --ignore-scripts
+      - name: build constants
+        run: |
+          npm run build:core:constants
       - name: build typescript part
         run: |
           npm run build:bindings:ts

CHANGELOG.md

@@ -2,9 +2,104 @@
 
 ## Unreleased
 
-## Changes
+### Changes
+- handle drafts from mailto links in scanned QR #3492
+- do not overflow ratelimiter leaky bucket #3496
 
-## Fixes
+### Fixes
+- don't squash text parts of NDN into attachments #3497
+- do not treat non-failed DSNs as NDNs #3506
+
+
+## 1.89.0
+
+### Changes
+
+- (AEAP) When one of your contacts changed their address, they are
+  only replaced in the chat where you got a message from them
+  for now #3491
+
+### Fixes
+- replace musl libc name resolution errors with a better message #3485
+- handle updates for not yet downloaded webxdc instances #3487
+
+
+## 1.88.0
+
+### Changes
+- Implemented "Automatic e-mail address Porting" (AEAP). You can
+  configure a new address in DC now, and when receivers get messages
+  they will automatically recognize your moving to a new address. #3385
+- switch from `async-std` to `tokio` as the async runtime #3449
+- upgrade to `pgp@0.8.0` #3467
+- add IMAP ID extension support #3468
+- configure DeltaChat folder by selecting it, so it is configured even if not LISTed #3371
+- build PyPy wheels #6683
+- improve default error if NDN does not provide an error #3456
+- increase ratelimit from 3 to 6 messages per 60 seconds #3481
+
+### Fixes
+- mailing list: remove square-brackets only for first name #3452
+- do not use footers from mailinglists as the contact status #3460
+- don't ignore KML parsing errors #3473
+
+
+## 1.87.0
+
+### Changes
+- limit the rate of MDN sending #3402
+- ignore ratelimits for bots #3439
+- remove `msgs_mdns` references to deleted messages during housekeeping #3387
+- format message lines starting with `>` as quotes #3434
+- node: remove `split2` dependency #3418
+- node: add git installation info to readme #3418
+- limit the rate of webxdc update sending #3417
+
+### Fixes
+- set a default error if NDN does not provide an error #3410
+- python: avoid exceptions when messages/contacts/chats are compared with `None`
+- node: wait for the event loop to stop before destroying contexts #3431 #3451
+- emit configuration errors via event on failure #3433
+- report configure and imex success/failure after freeing ongoing process #3442
+
+### API-Changes
+- python: added `Message.get_status_updates()`  #3416
+- python: added `Message.send_status_update()`  #3416
+- python: added `Message.is_webxdc()`  #3416
+- python: added `Message.is_videochat_invitation()`  #3416
+- python: added support for "videochat" and "webxdc" view types to `Message.new_empty()`  #3416
+
+
+## 1.86.0
+
+### API-Changes
+- python: added optional `closed` parameter to `Account` constructor #3394
+- python: added optional `passphrase` parameter to `Account.export_all()` and `Account.import_all()` #3394
+- python: added `Account.open()` #3394
+- python: added `Chat.is_single()` #3394
+- python: added `Chat.is_mailinglist()` #3394
+- python: added `Chat.is_broadcast()` #3394
+- python: added `Chat.is_multiuser()` #3394
+- python: added `Chat.is_self_talk()` #3394
+- python: added `Chat.is_device_talk()` #3394
+- python: added `Chat.is_pinned()` #3394
+- python: added `Chat.pin()` #3394
+- python: added `Chat.unpin()` #3394
+- python: added `Chat.archive()` #3394
+- python: added `Chat.unarchive()` #3394
+- python: added `Message.get_summarytext()` #3394
+- python: added optional `closed` parameter to `ACFactory.get_unconfigured_account()` (pytest plugin) #3394
+- python: added optional `passphrase` parameter to `ACFactory.get_pseudo_configured_account()` (pytest plugin) #3394
+
+### Changes
+- clean up series of webxdc info messages;
+  `DC_EVENT_MSGS_CHANGED` is emitted on changes of existing info messages #3395
+- update provider database #3399
+- refactorings #3375 #3403 #3398 #3404
+
+### Fixes
+- do not reset our database if imported backup cannot be decrypted #3397
+- node: remove `npx` from build script, this broke flathub build #3396
 
 
 ## 1.85.0
@@ -15,7 +110,6 @@
 - python: build Python 3.10 wheels #3392
 - update Rust dependencies
 
-
 ### Fixes
 - delete outgoing MDNs found in the Sent folder on Gmail #3372
 - fix searching one-to-one chats #3377
@@ -43,10 +137,11 @@
 ### Fixes
 - fix node prebuild & package ci #3337
 
+
 ## 1.82.0
 
 ### API-Changes
-- re-add removed DC_MSG_ID_MARKER1 as in use on iOS #3330
+- re-add removed `DC_MSG_ID_MARKER1` as in use on iOS #3330
 
 ### Changes
 - refactorings #3328

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.85.0"
+version = "1.89.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -9,6 +9,7 @@ rust-version = "1.56"
 [profile.dev]
 debug = 0
 panic = 'abort'
+opt-level = 1
 
 [profile.release]
 lto = true
@@ -19,19 +20,19 @@ deltachat_derive = { path = "./deltachat_derive" }
 
 ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
-async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", default-features=false, features = ["smtp-transport", "socks5"] }
-async-std-resolver = "0.21"
-async-std = { version = "1" }
-async-tar = { version = "0.4", default-features=false }
+async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] }
+async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] }
+async-smtp = { version = "0.5", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] }
+trust-dns-resolver = "0.21"
+tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
+tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
 backtrace = "0.3"
 base64 = "0.13"
 bitflags = "1.3"
-chrono = "0.4"
+chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
 dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+encoded-words = "0.2"
 escaper = "0.1"
 futures = "0.3"
 hex = "0.4.0"
@@ -47,12 +48,12 @@ num-derive = "0.3"
 num-traits = "0.2"
 once_cell = "1.12.0"
 percent-encoding = "2.0"
-pgp = { version = "0.7", default-features = false }
+pgp = { version = "0.8", default-features = false }
 pretty_env_logger = { version = "0.4", optional = true }
 quick-xml = "0.23"
 r2d2 = "0.8"
 r2d2_sqlite = "0.20"
-rand = "0.7"
+rand = "0.8"
 regex = "1.5"
 rusqlite = { version = "0.27", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
@@ -65,27 +66,30 @@ sha2 = "0.10"
 smallvec = "1"
 strum = "0.24"
 strum_macros = "0.24"
-surf = { version = "2.3", default-features = false, features = ["h1-client"] }
 thiserror = "1"
 toml = "0.5"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
-fast-socks5 = "0.4"
+fast-socks5 = "0.8"
 humansize = "1"
 qrcodegen = "1.7.0"
 tagger = "4.3.3"
 textwrap = "0.15.0"
-zip = { version = "0.6.2", default-features = false, features = ["deflate"] }
+async-channel = "1.6.1"
+futures-lite = "1.12.0"
+tokio-stream = { version = "0.1.9", features = ["fs"] }
+reqwest = { version = "0.11.11", features = ["json"] }
+async_zip = { git = "https://github.com/dignifiedquire/rs-async-zip", branch = "main", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = { version = "0.3.4", features = ["async_std"] }
+criterion = { version = "0.3.4", features = ["async_tokio"] }
 futures-lite = "1.12"
 log = "0.4"
 pretty_env_logger = "0.4"
 proptest = { version = "1", default-features = false, features = ["std"] }
 tempfile = "3"
+tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] }
 
 [workspace]
 members = [
@@ -132,5 +136,10 @@ harness = false
 default = ["vendored"]
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl"]
+vendored = [
+  "async-native-tls/vendored",
+  "async-smtp/native-tls-vendored",
+  "rusqlite/bundled-sqlcipher-vendored-openssl",
+  "reqwest/native-tls-vendored"
+]
 nightly = ["pgp/nightly"]

README.md

@@ -102,9 +102,6 @@ $ cargo build -p deltachat_ffi --release
 
 ## Debugging environment variables 
 
-- `DCC_IMAP_DEBUG`: if set IMAP protocol commands and responses will be
-  printed
-
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
 - `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and

benches/contacts.rs

@@ -1,14 +1,14 @@
-use async_std::task::block_on;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::contact::Contact;
 use deltachat::context::Context;
+use deltachat::Events;
 use tempfile::tempdir;
 
 async fn address_book_benchmark(n: u32, read_count: u32) {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(&dbfile, id, Events::new()).await.unwrap();
 
     let book = (0..n)
         .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
@@ -24,12 +24,16 @@ async fn address_book_benchmark(n: u32, read_count: u32) {
 }
 
 fn criterion_benchmark(c: &mut Criterion) {
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
     c.bench_function("create 500 contacts", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
+        b.to_async(&rt)
+            .iter(|| async { address_book_benchmark(black_box(500), black_box(0)).await })
     });
 
     c.bench_function("create 100 contacts and read it 1000 times", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
+        b.to_async(&rt)
+            .iter(|| async { address_book_benchmark(black_box(100), black_box(1000)).await })
     });
 }
 

benches/create_account.rs

@@ -1,12 +1,11 @@
-use async_std::path::PathBuf;
-use async_std::task::block_on;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::accounts::Accounts;
+use std::path::PathBuf;
 use tempfile::tempdir;
 
 async fn create_accounts(n: u32) {
     let dir = tempdir().unwrap();
-    let p: PathBuf = dir.path().join("accounts").into();
+    let p: PathBuf = dir.path().join("accounts");
 
     let mut accounts = Accounts::new(p.clone()).await.unwrap();
 
@@ -18,7 +17,8 @@ async fn create_accounts(n: u32) {
 
 fn criterion_benchmark(c: &mut Criterion) {
     c.bench_function("create 1 account", |b| {
-        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        b.to_async(&rt).iter(|| create_accounts(black_box(1)))
     });
 }
 

benches/get_chat_msgs.rs

@@ -1,15 +1,15 @@
-use async_std::path::Path;
+use std::path::Path;
 
-use criterion::async_executor::AsyncStdExecutor;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 
 use deltachat::chat::{self, ChatId};
 use deltachat::chatlist::Chatlist;
 use deltachat::context::Context;
+use deltachat::Events;
 
 async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(dbfile, id, Events::new()).await.unwrap();
 
     for c in chats.iter().take(10) {
         black_box(chat::get_chat_msgs(&context, *c, 0).await.ok());
@@ -20,15 +20,19 @@ fn criterion_benchmark(c: &mut Criterion) {
     // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
     // messages, such as your primary account.
     if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let chats: Vec<_> = async_std::task::block_on(async {
-            let context = Context::new((&path).into(), 100).await.unwrap();
+        let rt = tokio::runtime::Runtime::new().unwrap();
+
+        let chats: Vec<_> = rt.block_on(async {
+            let context = Context::new(Path::new(&path), 100, Events::new())
+                .await
+                .unwrap();
             let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
             let len = chatlist.len();
             (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
         });
 
         c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
-            b.to_async(AsyncStdExecutor)
+            b.to_async(&rt)
                 .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
         });
     } else {

benches/get_chatlist.rs

@@ -1,8 +1,10 @@
-use criterion::async_executor::AsyncStdExecutor;
+use std::path::Path;
+
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 
 use deltachat::chatlist::Chatlist;
 use deltachat::context::Context;
+use deltachat::Events;
 
 async fn get_chat_list_benchmark(context: &Context) {
     Chatlist::try_load(context, 0, None, None).await.unwrap();
@@ -12,10 +14,14 @@ fn criterion_benchmark(c: &mut Criterion) {
     // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
     // messages, such as your primary account.
     if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let context =
-            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let context = rt.block_on(async {
+            Context::new(Path::new(&path), 100, Events::new())
+                .await
+                .unwrap()
+        });
         c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
-            b.to_async(AsyncStdExecutor)
+            b.to_async(&rt)
                 .iter(|| get_chat_list_benchmark(black_box(&context)))
         });
     } else {

benches/receive_emails.rs

@@ -1,13 +1,12 @@
-use async_std::{path::PathBuf, task::block_on};
-use criterion::{
-    async_executor::AsyncStdExecutor, black_box, criterion_group, criterion_main, BatchSize,
-    Criterion,
-};
+use std::path::PathBuf;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::{
     config::Config,
     context::Context,
-    dc_receive_imf::dc_receive_imf,
     imex::{imex, ImexMode},
+    receive_imf::receive_imf,
+    Events,
 };
 use tempfile::tempdir;
 
@@ -31,7 +30,7 @@ Hello {i}",
             i = i,
             i_dec = i - 1,
         );
-        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), false)
+        receive_imf(&context, black_box(imf_raw.as_bytes()), false)
             .await
             .unwrap();
     }
@@ -42,13 +41,13 @@ async fn create_context() -> Context {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(&dbfile, id, Events::new()).await.unwrap();
 
     let backup: PathBuf = std::env::current_dir()
         .unwrap()
-        .join("delta-chat-backup.tar")
-        .into();
-    if backup.exists().await {
+        .join("delta-chat-backup.tar");
+
+    if backup.exists() {
         println!("Importing backup");
         imex(&context, ImexMode::ImportBackup, &backup, None)
             .await
@@ -71,11 +70,15 @@ async fn create_context() -> Context {
 fn criterion_benchmark(c: &mut Criterion) {
     let mut group = c.benchmark_group("Receive messages");
     group.bench_function("Receive 100 simple text msgs", |b| {
-        b.to_async(AsyncStdExecutor).iter_batched(
-            || block_on(create_context()),
-            |context| recv_all_emails(black_box(context)),
-            BatchSize::LargeInput,
-        );
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let context = rt.block_on(create_context());
+
+        b.to_async(&rt).iter(|| {
+            let ctx = context.clone();
+            async move {
+                recv_all_emails(black_box(ctx)).await;
+            }
+        });
     });
     group.finish();
 }

benches/search_msgs.rs

@@ -1,12 +1,13 @@
-use async_std::task::block_on;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::context::Context;
+use deltachat::Events;
 use std::path::Path;
 
-async fn search_benchmark(path: impl AsRef<Path>) {
-    let dbfile = path.as_ref();
+async fn search_benchmark(dbfile: impl AsRef<Path>) {
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(dbfile.as_ref(), id, Events::new())
+        .await
+        .unwrap();
 
     for _ in 0..10u32 {
         context.search_msgs(None, "hello").await.unwrap();
@@ -17,8 +18,10 @@ fn criterion_benchmark(c: &mut Criterion) {
     // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
     // messages, such as your primary account.
     if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+
         c.bench_function("search hello", |b| {
-            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
+            b.to_async(&rt).iter(|| search_benchmark(black_box(&path)))
         });
     } else {
         println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.85.0"
+version = "1.89.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,10 +20,11 @@ libc = "0.2"
 human-panic = "1"
 num-traits = "0.2"
 serde_json = "1.0"
-async-std = "1"
+tokio = { version = "1", features = ["rt-multi-thread"] }
 anyhow = "1"
 thiserror = "1"
 rand = "0.7"
+once_cell = "1.12.0"
 
 [features]
 default = ["vendored"]

deltachat-ffi/build.rs

@@ -1,4 +1,3 @@
-use std::io::Write;
 use std::path::PathBuf;
 use std::{env, fs};
 
@@ -28,8 +27,9 @@ fn main() {
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
-    fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
-        .unwrap()
-        .write_all(pkg_config.as_bytes())
-        .unwrap();
+    fs::write(
+        target_path.join("pkgconfig").join("deltachat.pc"),
+        pkg_config.as_bytes(),
+    )
+    .unwrap();
 }

deltachat-ffi/deltachat.h

@@ -1791,6 +1791,11 @@ void            dc_delete_msgs               (dc_context_t* context, const uint3
 /**
  * Forward messages to another chat.
  *
+ * All types of messages can be forwarded,
+ * however, they will be flagged as such (dc_msg_is_forwarded() is set).
+ *
+ * Original sender, info-state and webxdc updates are not forwarded on purpose.
+ *
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_ids An array of uint32_t containing all message IDs that should be forwarded.
@@ -2287,7 +2292,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  * - DC_QR_FPR_MISMATCH with dc_lot_t::id=Contact ID:
  *   scanned fingerprint does not match last seen fingerprint.
  *
- * - DC_QR_FPR_WITHOUT_ADDR with dc_lot_t::test1=Formatted fingerprint
+ * - DC_QR_FPR_WITHOUT_ADDR with dc_lot_t::text1=Formatted fingerprint
  *   the scanned QR code contains a fingerprint but no e-mail address;
  *   suggest the user to establish an encrypted connection first.
  *
@@ -2300,7 +2305,8 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   if so, call dc_set_config_from_qr().
  *
  * - DC_QR_ADDR with dc_lot_t::id=Contact ID:
- *   e-mail address scanned,
+ *   e-mail address scanned, optionally, a draft message could be set in
+ *   dc_lot_t::text1 in which case dc_lot_t::text1_meaning will be DC_TEXT1_DRAFT;
  *   ask the user if they want to start chatting;
  *   if so, call dc_create_chat_by_contact_id().
  *
@@ -3979,7 +3985,7 @@ int             dc_msg_is_sent                (const dc_msg_t* msg);
  *
  * For privacy reasons, we do not provide the name or the e-mail address of the
  * original author (in a typical GUI, you select the messages text and click on
- * "forwared"; you won't expect other data to be send to the new recipient,
+ * "forwarded"; you won't expect other data to be send to the new recipient,
  * esp. as the new recipient may not be in any relationship to the original author)
  *
  * @memberof dc_msg_t

deltachat-ffi/src/lib.rs

@@ -1,4 +1,4 @@
-#![deny(clippy::all)]
+#![warn(unused, clippy::all)]
 #![allow(
     non_camel_case_types,
     non_snake_case,
@@ -11,23 +11,23 @@
 
 #[macro_use]
 extern crate human_panic;
-extern crate num_traits;
-extern crate serde_json;
 
 use std::collections::BTreeMap;
 use std::convert::TryFrom;
 use std::fmt::Write;
+use std::future::Future;
 use std::ops::Deref;
 use std::ptr;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use anyhow::Context as _;
-use async_std::sync::RwLock;
-use async_std::task::{block_on, spawn};
 use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use num_traits::{FromPrimitive, ToPrimitive};
+use once_cell::sync::Lazy;
 use rand::Rng;
+use tokio::runtime::Runtime;
+use tokio::sync::RwLock;
 
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
@@ -40,6 +40,7 @@ use deltachat::stock_str::StockMessage;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
+use tokio::task::JoinHandle;
 
 mod dc_array;
 mod lot;
@@ -63,6 +64,23 @@ use deltachat::chatlist::Chatlist;
 /// Struct representing the deltachat context.
 pub type dc_context_t = Context;
 
+static RT: Lazy<Runtime> = Lazy::new(|| Runtime::new().expect("unable to create tokio runtime"));
+
+fn block_on<T>(fut: T) -> T::Output
+where
+    T: Future,
+{
+    RT.block_on(fut)
+}
+
+fn spawn<T>(fut: T) -> JoinHandle<T::Output>
+where
+    T: Future + Send + 'static,
+    T::Output: Send + 'static,
+{
+    RT.spawn(fut)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_context_new(
     _os_name: *const libc::c_char,
@@ -79,7 +97,7 @@ pub unsafe extern "C" fn dc_context_new(
     let ctx = if blobdir.is_null() || *blobdir == 0 {
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
-        block_on(Context::new(as_path(dbfile).to_path_buf().into(), id))
+        block_on(Context::new(as_path(dbfile), id, Events::new()))
     } else {
         eprintln!("blobdir can not be defined explicitly anymore");
         return ptr::null_mut();
@@ -103,10 +121,7 @@ pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *
     }
 
     let id = rand::thread_rng().gen();
-    match block_on(Context::new_closed(
-        as_path(dbfile).to_path_buf().into(),
-        id,
-    )) {
+    match block_on(Context::new_closed(as_path(dbfile), id, Events::new())) {
         Ok(context) => Box::into_raw(Box::new(context)),
         Err(err) => {
             eprintln!("failed to create context: {:#}", err);
@@ -380,7 +395,7 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
     let redirect = to_string_lossy(redirect);
 
     block_on(async move {
-        match oauth2::dc_get_oauth2_url(ctx, &addr, &redirect)
+        match oauth2::get_oauth2_url(ctx, &addr, &redirect)
             .await
             .log_err(ctx, "dc_get_oauth2_url failed")
         {
@@ -678,10 +693,13 @@ pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *
     }
     let events = &*events;
 
-    events
-        .recv_sync()
-        .map(|ev| Box::into_raw(Box::new(ev)))
-        .unwrap_or_else(ptr::null_mut)
+    block_on(async move {
+        events
+            .recv()
+            .await
+            .map(|ev| Box::into_raw(Box::new(ev)))
+            .unwrap_or_else(ptr::null_mut)
+    })
 }
 
 #[no_mangle]
@@ -721,7 +739,7 @@ pub unsafe extern "C" fn dc_preconfigure_keypair(
     }
     let ctx = &*context;
     block_on(async move {
-        let addr = dc_tools::EmailAddress::new(&to_string_lossy(addr))?;
+        let addr = tools::EmailAddress::new(&to_string_lossy(addr))?;
         let public = key::SignedPublicKey::from_asc(&to_string_lossy(public_data))?.0;
         let secret = key::SignedSecretKey::from_asc(&to_string_lossy(secret_data))?.0;
         let keypair = key::KeyPair {
@@ -2226,7 +2244,7 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
         Some(ChatId::new(chat_id))
     };
 
-    block_on(securejoin::dc_get_securejoin_qr(ctx, chat_id))
+    block_on(securejoin::get_securejoin_qr(ctx, chat_id))
         .unwrap_or_else(|_| "".to_string())
         .strdup()
 }
@@ -2264,7 +2282,7 @@ pub unsafe extern "C" fn dc_join_securejoin(
     let ctx = &*context;
 
     block_on(async move {
-        securejoin::dc_join_securejoin(ctx, &to_string_lossy(qr))
+        securejoin::join_securejoin(ctx, &to_string_lossy(qr))
             .await
             .map(|chatid| chatid.to_u32())
             .log_err(ctx, "failed dc_join_securejoin() call")
@@ -2390,7 +2408,7 @@ pub unsafe extern "C" fn dc_get_last_error(context: *mut dc_context_t) -> *mut l
         return "".strdup();
     }
     let ctx = &*context;
-    block_on(ctx.get_last_error()).strdup()
+    ctx.get_last_error().strdup()
 }
 
 // dc_array_t
@@ -3576,7 +3594,8 @@ pub unsafe extern "C" fn dc_msg_latefiling_mediasize(
         ffi_msg
             .message
             .latefiling_mediasize(ctx, width, height, duration)
-    });
+    })
+    .ok_or_log_msg(ctx, "Cannot set media size");
 }
 
 #[no_mangle]
@@ -4122,7 +4141,7 @@ pub unsafe extern "C" fn dc_accounts_new(
         return ptr::null_mut();
     }
 
-    let accs = block_on(Accounts::new(as_path(dbfile).to_path_buf().into()));
+    let accs = block_on(Accounts::new(as_path(dbfile).into()));
 
     match accs {
         Ok(accs) => Box::into_raw(Box::new(AccountsWrapper::new(accs))),
@@ -4294,7 +4313,7 @@ pub unsafe extern "C" fn dc_accounts_migrate_account(
     block_on(async move {
         let mut accounts = accounts.write().await;
         match accounts
-            .migrate_account(async_std::path::PathBuf::from(dbfile))
+            .migrate_account(std::path::PathBuf::from(dbfile))
             .await
         {
             Ok(id) => id,
@@ -4377,7 +4396,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
-pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
+pub type dc_accounts_event_emitter_t = EventEmitter;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_get_event_emitter(
@@ -4414,9 +4433,7 @@ pub unsafe extern "C" fn dc_accounts_get_next_event(
         return ptr::null_mut();
     }
     let emitter = &mut *emitter;
-
-    emitter
-        .recv_sync()
+    block_on(emitter.recv())
         .map(|ev| Box::into_raw(Box::new(ev)))
         .unwrap_or_else(ptr::null_mut)
 }

deltachat-ffi/src/lot.rs

@@ -51,7 +51,7 @@ impl Lot {
                 Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
                 Qr::Account { domain } => Some(domain),
                 Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { .. } => None,
+                Qr::Addr { draft, .. } => draft.as_deref(),
                 Qr::Url { url } => Some(url),
                 Qr::Text { text } => Some(text),
                 Qr::WithdrawVerifyContact { .. } => None,
@@ -79,7 +79,13 @@ impl Lot {
                 Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
                 Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
             },
-            Self::Qr(_qr) => Meaning::None,
+            Self::Qr(qr) => match qr {
+                Qr::Addr {
+                    draft: Some(_draft),
+                    ..
+                } => Meaning::Text1Draft,
+                _ => Meaning::None,
+            },
             Self::Error(_err) => Meaning::None,
         }
     }
@@ -118,7 +124,7 @@ impl Lot {
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => contact_id.to_u32(),
+                Qr::Addr { contact_id, .. } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
                 Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),

deltachat-ffi/src/string.rs

@@ -55,7 +55,7 @@ pub(crate) enum CStringError {
 /// # Example
 ///
 /// ```
-/// use deltachat::dc_tools::{dc_strdup, OsStrExt};
+/// use deltachat::tools::{dc_strdup, OsStrExt};
 /// let path = std::path::Path::new("/some/path");
 /// let path_c = path.to_c_string().unwrap();
 /// unsafe {

draft/aeap-mvp.md

@@ -11,26 +11,49 @@ Changes to the UIs
 Changes in the core
 -------------------
 
-- DONE: We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
+- [x] We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
 
-- DONE: If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
+- [x] If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
 
   - don't forget to deduplicate secondary self addresses in case the user switches back and forth between addresses).
 
   - The key stays the same.
 
-- No changes for 1:1 chats, there simply is a new one. (This works since, contrary to group messages, messages sent to a 1:1 chat are not assigned to the group chat but always to the 1:1 chat with the sender. So it's not a problem that the new messages might be put into the old chat if they are a reply to a message there.)
+- [x] No changes for 1:1 chats, there simply is a new one. (This works since, contrary to group messages, messages sent to a 1:1 chat are not assigned to the group chat but always to the 1:1 chat with the sender. So it's not a problem that the new messages might be put into the old chat if they are a reply to a message there.)
 
-- When sending a message: If any of the secondary self addrs is in the chat's member list, remove it locally (because we just transitioned away from it). We add a log message for this (alternatively, a system message in the chat would be more visible).
+- [ ] When sending a message: If any of the secondary self addrs is in the chat's member list, remove it locally (because we just transitioned away from it). We add a log message for this (alternatively, a system message in the chat would be more visible).
 
-- When receiving a message: If the key exists, but belongs to another address (we may want to benchmark this)
+- [x] ([#3385](https://github.com/deltachat/deltachat-core-rust/pull/3385)) When receiving a message: If the key exists, but belongs to another address (we may want to benchmark this)
   AND there is a `Chat-Version` header\
+  AND the message is signed correctly
+  AND the From address is (also) in the encrypted (and therefore signed) headers <sup>[[1]](#myfootnote1)</sup>\
   AND the message timestamp is newer than the contact's `lastseen` (to prevent changing the address back when messages arrive out of order) (this condition is not that important since we will have eventual consistency even without it):
 
   Replace the contact in _all_ groups, possibly deduplicate the members list, and add a system message to all of these chats.
   
   - Note that we can't simply compare the keys byte-by-byte, since the UID may have changed, or the sender may have rotated the key and signed the new key with the old one.
 
+<a name="myfootnote1">[1]</a>: Without this check, an attacker could replay a message from Alice to Bob. Then Bob's device would do an AEAP transition from Alice's to the attacker's address, allowing for easier phishing.
+
+<details>
+<summary>More details about this</summary>
+Suppose Alice sends a message to Evil (or to a group with both Evil and Bob). Evil then forwards the message to Bob, changing the From and To headers (and if necessary Message-Id) and replacing `addr=alice@example.org;` in the autocrypt header with `addr=evil@example.org;`.
+
+Then Bob's device sees that there is a message which is signed by Alice's key and comes from Evil's address and would do the AEAP transition, i.e. replace Alice with Evil in all groups and show a message "Alice changed their address from alice@example.org to evil@example.org". Disadvantages for Evil are that Bob's message will be shown on Alice's device, possibly creating confusion/suspicion, and that the usual "Setup changed for..." message will be shown the next time Evil sends a message (because Evil doesn't know Alice's private key).
+
+Possible mitigations:
+- if we make the AEAP device message sth. like "Automatically removed alice@example.org and added evil@example.org", then this will create more suspicion, making the phishing harder (we didn't talk about what what the wording should be at all yet).
+- Add something similar to replay protection to our Autocrypt implementation. This could be done e.g. by adding a second `From` header to the protected headers. If it's present, the receiver then requires it to be the same as the outer `From`, and if it's not present, we don't do AEAP --> **That's what we implemented**
+
+Note that usually a mail is signed by a key that has a UID matching the from address.
+
+  That's not mandatory for Autocrypt (and in fact, we just keep the old UID when changing the self address, so with AEAP the UID will actually be different than the from address sometimes)
+
+  https://autocrypt.org/level1.html#openpgp-based-key-data says:
+  > The content of the user id packet is only decorative
+
+</details>
+
 ### Notes:
   
 - We treat protected and non-protected chats the same
@@ -97,3 +120,8 @@ Other
 -----
 
 - The user is responsible that messages to the old address arrive at the new address, for example by configuring the old provider to forward all emails to the new one.
+
+Notes during implementing
+========================
+
+- As far as I understand the code, unencrypted messages are unsigned. So, the transition only works if both sides have the other side's key.

draft/webxdc-dev-reference.md

@@ -1,6 +1,12 @@
 # Webxdc Developer Reference
 
-(This document may eventually be merged with the [webxdc guidebook](https://deltachat.github.io/webxdc_docs/), where you may currently find other useful information.)
+This document gives a quick overview about the Webxdc specification,
+It is meant for both, developing Webxdc apps
+and developing Webxdc implementations.
+
+The [Webxdc guidebook](https://deltachat.github.io/webxdc_docs/) shows more detailed information
+when developing Webxdc apps.
+
 
 ## Webxdc File Format
 
@@ -34,8 +40,9 @@ To get a shared state, the peers use `sendUpdate()` to send updates to each othe
 - `update`: an object with the following properties:  
     - `update.payload`: any javascript primitive, array or object.
     - `update.info`: optional, short, informational message that will be added to the chat,
-       eg. "Alice voted" or "Bob scored 123 in MyGame";
-       usually only one line of text is shown,
+       eg. "Alice voted" or "Bob scored 123 in MyGame".
+       usually only one line of text is shown
+       and if there are series of info messages, older ones may be dropped.
        use this option sparingly to not spam the chat.
     - `update.document`: optional, name of the document in edit,
        must not be used eg. in games where the Webxdc does not create documents
@@ -137,9 +144,37 @@ round corners etc. will be added by the implementations as needed.
 If no icon is set, a default icon will be used.
 
 
+## Other APIs and Tags Usage Hints
+
+- `localStorage`, `sessionStorage`, `indexedDB` are okay to be used
+- `visibilitychange`-events are okay to be used
+- `window.navigator.language` is okay to be used, on desktop it is the system language
+- `<a href="localfile.html">` and other internal links are okay to be used
+- `<a href="mailto:addr@example.org?body=...">`- mailto links are okay to be used
+- `<meta name="viewport" ...>` usage is okay to be used
+  and useful esp. different webviews have different defaults
+
+
+### Discouraged Things
+
+- `document.cookie` is known not to work on desktop and iOS
+  use `localStorage` instead
+- `unload`-, `beforeunload`- and `pagehide`-events are known not to work on iOS and are flaky on other systems
+  (also partly discouraged by [mozilla](https://developer.mozilla.org/en-US/docs/Web/API/Window/unload_event))
+  use `visibilitychange` instead
+- `<title>` and `document.title` is ignored by Webxdc;
+  use the `name` property from `manifest.toml` instead
+- newest js features may not work on all webviews,
+  you may want to transpile your code down to an older js version
+  eg. with <https://babeljs.io>
+- `<a href="https://example.org/foo">` and other external links are blocked by definition;
+  instead, embed content or use `mailto:` link to offer a way for contact
+- `<input type="file">` is discouraged currently; this may change in future
+
+
 ## Webxdc Examples
 
-The following example shows an input field and  every input is show on all peers.
+The following example shows an input field and every input is show on all peers.
 
 ```html
 <!DOCTYPE html>
@@ -169,30 +204,10 @@ The following example shows an input field and  every input is show on all peers
 </html>
 ```
 
-[Webxdc Development Tool](https://github.com/deltachat/webxdc-dev)
-offers an **Webxdc Simulator** that can be used in many browsers without any installation needed.
+More examples at [github.com/webxdc](https://github.com/webxdc) and
+[topic #webxdc](https://github.com/topics/webxdc)
+
+[github.com/webxdc/hello](https://github.com/webxdc/hello)
+offers an **Webxdc Tool** that can be used in many browsers without any installation needed.
 You can also use that repository as a template for your own Webxdc -
 just clone and start adapting things to your need.
-
-
-### Advanced Examples
-
-- [2048](https://github.com/adbenitez/2048.xdc)
-- [Draw](https://github.com/adbenitez/draw.xdc)
-- [Poll](https://github.com/r10s/webxdc-poll/)
-- [Tic Tac Toe](https://github.com/Simon-Laux/tictactoe.xdc)
-- Even more with [Topic #webxdc on Github](https://github.com/topics/webxdc) or in the [webxdc GitHub organization](https://github.com/webxdc)
-
-
-## Closing Remarks
-
-- older devices might not have the newest js features in their webview,
-  you may want to transpile your code down to an older js version eg. with https://babeljs.io
-- viewport and scaling features are implementation specific,
-  if you want to have an explicit behavior, you can add eg.
-  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
-- the `<title>` tag should not be used and its content is usually not displayed;
-  instead, use the `name` property from `manifest.toml`
-- there are tons of ideas for enhancements of the API and the file format,
-  eg. in the future, we will may define icon- and manifest-files,
-  allow to aggregate the state or add metadata.

examples/repl/cmdline.rs

@@ -1,9 +1,10 @@
 extern crate dirs;
 
+use std::path::Path;
 use std::str::FromStr;
+use std::time::{Duration, SystemTime};
 
 use anyhow::{bail, ensure, Result};
-use async_std::path::Path;
 use deltachat::chat::{
     self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
 };
@@ -11,8 +12,6 @@ use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::dc_receive_imf::*;
-use deltachat::dc_tools::*;
 use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
@@ -20,10 +19,11 @@ use deltachat::log::LogExt;
 use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
+use deltachat::receive_imf::*;
 use deltachat::sql;
+use deltachat::tools::*;
 use deltachat::{config, provider};
-use std::fs;
-use std::time::{Duration, SystemTime};
+use tokio::fs;
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
@@ -96,10 +96,10 @@ async fn reset_tables(context: &Context, bits: i32) {
 }
 
 async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
-    let data = dc_read_file(context, filename).await?;
+    let data = read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, false).await {
-        println!("dc_receive_imf errored: {:?}", err);
+    if let Err(err) = receive_imf(context, &data, false).await {
+        println!("receive_imf errored: {:?}", err);
     }
     Ok(())
 }
@@ -128,24 +128,20 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
         }
         real_spec = rs.unwrap();
     }
-    if let Some(suffix) = dc_get_filesuffix_lc(&real_spec) {
+    if let Some(suffix) = get_filesuffix_lc(&real_spec) {
         if suffix == "eml" && poke_eml_file(context, &real_spec).await.is_ok() {
             read_cnt += 1
         }
     } else {
         /* import a directory */
         let dir_name = std::path::Path::new(&real_spec);
-        let dir = std::fs::read_dir(dir_name);
+        let dir = fs::read_dir(dir_name).await;
         if dir.is_err() {
             error!(context, "Import: Cannot open directory \"{}\".", &real_spec,);
             return false;
         } else {
-            let dir = dir.unwrap();
-            for entry in dir {
-                if entry.is_err() {
-                    break;
-                }
-                let entry = entry.unwrap();
+            let mut dir = dir.unwrap();
+            while let Ok(Some(entry)) = dir.next_entry().await {
                 let name_f = entry.file_name();
                 let name = name_f.to_string_lossy();
                 if name.ends_with(".eml") {
@@ -191,7 +187,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         DownloadState::Failure => " [⬇ Download failed]",
     };
 
-    let temp2 = dc_timestamp_to_str(msg.get_timestamp());
+    let temp2 = timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
         "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{}{} [{}]",
@@ -219,6 +215,14 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
                 msg.get_videochat_url().unwrap_or_default(),
                 msg.get_videochat_type().unwrap_or_default()
             )
+        } else if msg.get_viewtype() == Viewtype::Webxdc {
+            match msg.get_webxdc_info(context).await {
+                Ok(info) => format!(
+                    "[WEBXDC: {}, icon={}, document={}, summary={}, source_code_url={}]",
+                    info.name, info.icon, info.document, info.summary, info.source_code_url
+                ),
+                Err(err) => format!("[get_webxdc_info() failed: {}]", err),
+            }
         } else {
             "".to_string()
         },
@@ -492,7 +496,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let setup_code = create_setup_code(&context);
             let file_name = blobdir.join("autocrypt-setup-message.html");
             let file_content = render_setup_file(&context, &setup_code).await?;
-            async_std::fs::write(&file_name, file_content).await?;
+            fs::write(&file_name, file_content).await?;
             println!(
                 "Setup message written to: {}\nSetup code: {}",
                 file_name.display(),
@@ -532,7 +536,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 .join("connectivity.html");
             match context.get_connectivity_html().await {
                 Ok(html) => {
-                    fs::write(&file, html)?;
+                    fs::write(&file, html).await?;
                     println!("Report written to: {:#?}", file);
                 }
                 Err(err) => {
@@ -597,7 +601,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             _ => "",
                         }
                     };
-                    let timestr = dc_timestamp_to_str(summary.timestamp);
+                    let timestr = timestamp_to_str(summary.timestamp);
                     println!(
                         "{}{}{} [{}]{}",
                         summary
@@ -810,7 +814,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 println!(
                     "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} {} {}",
                     location.location_id,
-                    dc_timestamp_to_str(location.timestamp),
+                    timestamp_to_str(location.timestamp),
                     location.latitude,
                     location.longitude,
                     location.accuracy,
@@ -892,7 +896,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "No html-file given.");
             let path: &Path = arg1.as_ref();
-            let html = &*fs::read(&path)?;
+            let html = &*fs::read(&path).await?;
             let html = String::from_utf8_lossy(html);
 
             let mut msg = Message::new(Viewtype::Text);
@@ -1079,7 +1083,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 .unwrap_or_default()
                 .join(format!("msg-{}.html", id.to_u32()));
             let html = id.get_html(&context).await?.unwrap_or_default();
-            fs::write(&file, html)?;
+            fs::write(&file, html).await?;
             println!("HTML written to: {:#?}", file);
         }
         "listfresh" => {
@@ -1233,8 +1237,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "fileinfo" => {
             ensure!(!arg1.is_empty(), "Argument <file> missing.");
 
-            if let Ok(buf) = dc_read_file(&context, &arg1).await {
-                let (width, height) = dc_get_filemeta(&buf)?;
+            if let Ok(buf) = read_file(&context, &arg1).await {
+                let (width, height) = get_filemeta(&buf)?;
                 println!("width={}, height={}", width, height);
             } else {
                 bail!("Command failed.");

examples/repl/main.rs

@@ -9,17 +9,18 @@ extern crate deltachat;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
+use std::path::Path;
 use std::process::Command;
 
 use ansi_term::Color;
 use anyhow::{bail, Error};
-use async_std::path::Path;
 use deltachat::chat::ChatId;
 use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
+use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use deltachat::securejoin::*;
-use deltachat::EventType;
+use deltachat::{EventType, Events};
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
@@ -30,11 +31,11 @@ use rustyline::validate::Validator;
 use rustyline::{
     Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
 };
+use tokio::fs;
+use tokio::runtime::Handle;
 
 mod cmdline;
 use self::cmdline::*;
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
-use std::fs;
 
 /// Event Handler
 fn receive_event(event: EventType) {
@@ -298,10 +299,10 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new(Path::new(&args[1]), 0, Events::new()).await?;
 
     let events = context.get_event_emitter();
-    async_std::task::spawn(async move {
+    tokio::task::spawn(async move {
         while let Some(event) = events.recv().await {
             receive_event(event.typ);
         }
@@ -316,8 +317,9 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::channel::bounded(100);
-    let input_loop = async_std::task::spawn_blocking(move || {
+
+    let ctx = context.clone();
+    let input_loop = tokio::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
             highlighter: MatchingBracketHighlighter::new(),
@@ -339,16 +341,30 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line)).unwrap();
+                    let contine = Handle::current().block_on(async {
+                        match handle_cmd(line.trim(), ctx.clone(), &mut selected_chat).await {
+                            Ok(ExitResult::Continue) => true,
+                            Ok(ExitResult::Exit) => {
+                                println!("Exiting ...");
+                                false
+                            }
+                            Err(err) => {
+                                println!("Error: {}", err);
+                                true
+                            }
+                        }
+                    });
+
+                    if !contine {
+                        break;
+                    }
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
-                    drop(reader_s);
                     break;
                 }
                 Err(err) => {
                     println!("Error: {}", err);
-                    drop(reader_s);
                     break;
                 }
             }
@@ -359,15 +375,8 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         Ok::<_, Error>(())
     });
 
-    while let Ok(line) = reader_r.recv().await {
-        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
-            Ok(ExitResult::Continue) => {}
-            Ok(ExitResult::Exit) => break,
-            Err(err) => println!("Error: {}", err),
-        }
-    }
     context.stop_io().await;
-    input_loop.await?;
+    input_loop.await??;
 
     Ok(())
 }
@@ -400,7 +409,7 @@ async fn handle_cmd(
         "oauth2" => {
             if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
                 let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
+                    get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -417,7 +426,7 @@ async fn handle_cmd(
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
             let group = arg1.parse::<u32>().ok().map(ChatId::new);
-            let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
+            let mut qr = get_securejoin_qr(&ctx, group).await?;
             if !qr.is_empty() {
                 if arg0 == "getbadqr" && qr.len() > 40 {
                     qr.replace_range(12..22, "0000000000")
@@ -437,7 +446,7 @@ async fn handle_cmd(
             let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
             match get_securejoin_qr_svg(&ctx, group).await {
                 Ok(svg) => {
-                    fs::write(&file, svg)?;
+                    fs::write(&file, svg).await?;
                     println!("QR code svg written to: {:#?}", file);
                 }
                 Err(err) => {
@@ -448,7 +457,7 @@ async fn handle_cmd(
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await?;
+                join_securejoin(&ctx, arg1).await?;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),
@@ -458,11 +467,12 @@ async fn handle_cmd(
     Ok(ExitResult::Continue)
 }
 
-fn main() -> Result<(), Error> {
+#[tokio::main]
+async fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
     let args = std::env::args().collect();
-    async_std::task::block_on(async move { start(args).await })?;
+    start(args).await?;
 
     Ok(())
 }

examples/simple.rs

@@ -6,7 +6,7 @@ use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::message::Message;
-use deltachat::EventType;
+use deltachat::{EventType, Events};
 
 fn cb(event: EventType) {
     match event {
@@ -29,21 +29,21 @@ fn cb(event: EventType) {
 }
 
 /// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
-#[async_std::main]
+#[tokio::main]
 async fn main() {
     pretty_env_logger::try_init_timed().ok();
 
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new(dbfile.into(), 0)
+    let ctx = Context::new(&dbfile, 0, Events::new())
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
     log::info!("info: {:#?}", info);
 
     let events = ctx.get_event_emitter();
-    let events_spawn = async_std::task::spawn(async move {
+    let events_spawn = tokio::task::spawn(async move {
         while let Some(event) = events.recv().await {
             cb(event.typ);
         }
@@ -80,7 +80,7 @@ async fn main() {
     }
 
     // wait for the message to be sent out
-    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
 
     log::info!("fetching chats..");
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
@@ -96,5 +96,5 @@ async fn main() {
     ctx.stop_io().await;
     log::info!("closing");
     drop(ctx);
-    events_spawn.await;
+    events_spawn.await.unwrap();
 }

node/README.md

@@ -60,6 +60,19 @@ building from source or clone this repository and follow this steps:
 > not inside this folder. (We need this in order to include the rust source
 > code in the npm package.)
 
+### Use a git branch in deltachat-desktop
+
+You can directly install a core branch, but make sure:
+- that you have typescript in your project dependencies, as it is likely required
+- you know that there are **no prebuilds** and so core is built during installation which is why it takes so long
+
+```
+npm install https://github.com/deltachat/deltachat-core-rust.git#branch
+```
+
+If you want prebuilds for a branch that has a core pr, you might find an npm tar.gz package for that branch at <https://download.delta.chat/node/preview/>.
+The github ci also posts a link to it in the checks for each pr.
+
 ### Use build-from-source in deltachat-desktop
 
 If you want to use the manually built node bindings in the desktop client (for
@@ -104,7 +117,7 @@ $ fnm install 17 --arch x64
 $ fnm use 17
 $ node -p process.arch
 # result should be x64
-$ cd deltachat-core-rust && rustup target add x86_64-apple-darwin && cd -
+$ rustup target add x86_64-apple-darwin
 $ git apply patches/m1_build_use_x86_64.patch
 $ CARGO_BUILD_TARGET=x86_64-apple-darwin npm run build
 $ npm run test
@@ -221,7 +234,7 @@ We have the following scripts for building, testing and coverage:
 The following steps are needed to make a release:
 
 1. Wait until `pack-module` github action is completed
-2. Run `npm publish https://download.delta.chat/node/deltachat-node-v1.x.x.tar.gz` to publish it to npm. You probably need write rights to npm.
+2. Run `npm publish https://download.delta.chat/node/deltachat-node-1.x.x.tar.gz` to publish it to npm. You probably need write rights to npm.
 
 ## License
 

node/constants.js

@@ -83,7 +83,6 @@ module.exports = {
   DC_MSG_GIF: 21,
   DC_MSG_ID_DAYMARKER: 9,
   DC_MSG_ID_LAST_SPECIAL: 9,
-  DC_MSG_ID_MARKER1: 1,
   DC_MSG_IMAGE: 20,
   DC_MSG_STICKER: 23,
   DC_MSG_TEXT: 10,

node/lib/chat.ts

@@ -79,6 +79,10 @@ export class Chat {
     return this.getType() === C.DC_CHAT_TYPE_GROUP
   }
 
+  isBroadcast(): boolean {
+    return this.getType() === C.DC_CHAT_TYPE_BROADCAST
+  }
+
   isMuted(): boolean {
     return Boolean(binding.dcn_chat_is_muted(this.dc_chat))
   }

node/lib/constants.ts

@@ -83,7 +83,6 @@ export enum C {
   DC_MSG_GIF = 21,
   DC_MSG_ID_DAYMARKER = 9,
   DC_MSG_ID_LAST_SPECIAL = 9,
-  DC_MSG_ID_MARKER1 = 1,
   DC_MSG_IMAGE = 20,
   DC_MSG_STICKER = 23,
   DC_MSG_TEXT = 10,

node/lib/types.ts

@@ -5,6 +5,7 @@ export type ChatTypes =
   | C.DC_CHAT_TYPE_MAILINGLIST
   | C.DC_CHAT_TYPE_SINGLE
   | C.DC_CHAT_TYPE_UNDEFINED
+  | C.DC_CHAT_TYPE_BROADCAST
 
 export interface ChatJSON {
   archived: boolean

node/scripts/generate-constants.js

@@ -1,73 +1,65 @@
 #!/usr/bin/env node
-
 const fs = require('fs')
 const path = require('path')
-const split = require('split2')
 
 const data = []
-const regex = /^#define\s+(\w+)\s+(\w+)/i
-const header = path.resolve(
-  __dirname,
-  '../../deltachat-ffi/deltachat.h'
-)
+const header = path.resolve(__dirname, '../../deltachat-ffi/deltachat.h')
 
 console.log('Generating constants...')
 
-fs.createReadStream(header)
-  .pipe(split())
-  .on('data', (line) => {
-    const match = regex.exec(line)
-    if (match) {
-      const key = match[1]
-      const value = parseInt(match[2])
-      if (isNaN(value)) return
+const header_data = fs.readFileSync(header, 'UTF-8')
+const regex = /^#define\s+(\w+)\s+(\w+)/gm
+while (null != (match = regex.exec(header_data))) {
+  const key = match[1]
+  const value = parseInt(match[2])
+  if (!isNaN(value)) {
+    data.push({ key, value })
+  }
+}
 
-      data.push({ key, value })
-    }
+delete header_data
+
+const constants = data
+  .filter(
+    ({ key }) => key.toUpperCase()[0] === key[0] // check if define name is uppercase
+  )
+  .sort((lhs, rhs) => {
+    if (lhs.key < rhs.key) return -1
+    else if (lhs.key > rhs.key) return 1
+    return 0
   })
-  .on('end', () => {
-    const constants = data
-      .filter(
-        ({ key }) => key.toUpperCase()[0] === key[0] // check if define name is uppercase
-      )
-      .sort((lhs, rhs) => {
-        if (lhs.key < rhs.key) return -1
-        else if (lhs.key > rhs.key) return 1
-        return 0
-      })
-      .map((row) => {
-        return `  ${row.key}: ${row.value}`
-      })
-      .join(',\n')
+  .map((row) => {
+    return `  ${row.key}: ${row.value}`
+  })
+  .join(',\n')
 
-    const events = data
-      .sort((lhs, rhs) => {
-        if (lhs.value < rhs.value) return -1
-        else if (lhs.value > rhs.value) return 1
-        return 0
-      })
-      .filter((i) => {
-        return i.key.startsWith('DC_EVENT_')
-      })
-      .map((i) => {
-        return `  ${i.value}: '${i.key}'`
-      })
-      .join(',\n')
+const events = data
+  .sort((lhs, rhs) => {
+    if (lhs.value < rhs.value) return -1
+    else if (lhs.value > rhs.value) return 1
+    return 0
+  })
+  .filter((i) => {
+    return i.key.startsWith('DC_EVENT_')
+  })
+  .map((i) => {
+    return `  ${i.value}: '${i.key}'`
+  })
+  .join(',\n')
 
-    // backwards compat
-    fs.writeFileSync(
-      path.resolve(__dirname, '../constants.js'),
-      `// Generated!\n\nmodule.exports = {\n${constants}\n}\n`
-    )
-    // backwards compat
-    fs.writeFileSync(
-      path.resolve(__dirname, '../events.js'),
-      `/* eslint-disable quotes */\n// Generated!\n\nmodule.exports = {\n${events}\n}\n`
-    )
+// backwards compat
+fs.writeFileSync(
+  path.resolve(__dirname, '../constants.js'),
+  `// Generated!\n\nmodule.exports = {\n${constants}\n}\n`
+)
+// backwards compat
+fs.writeFileSync(
+  path.resolve(__dirname, '../events.js'),
+  `/* eslint-disable quotes */\n// Generated!\n\nmodule.exports = {\n${events}\n}\n`
+)
 
-    fs.writeFileSync(
-      path.resolve(__dirname, '../lib/constants.ts'),
-      `// Generated!\n\nexport enum C {\n${constants.replace(/:/g, ' =')},\n}\n
+fs.writeFileSync(
+  path.resolve(__dirname, '../lib/constants.ts'),
+  `// Generated!\n\nexport enum C {\n${constants.replace(/:/g, ' =')},\n}\n
 // Generated!\n\nexport const EventId2EventName: { [key: number]: string } = {\n${events},\n}\n`
-    )
-  })
+)

node/src/module.c

@@ -98,14 +98,6 @@ static void finalize_provider(napi_env env, void* data, void* hint) {
   }
 }
 
-static void finalize_account(napi_env env, void* data, void* hint) {
-  if (data) {
-    dc_accounts_t* dcn_accounts = (dc_accounts_t*)data;
-    //TRACE("cleaning up provider");
-    dc_accounts_unref(dcn_accounts);
-  }
-}
-
 /**
  * Helpers.
  */
@@ -348,7 +340,7 @@ NAPI_METHOD(dcn_start_event_handler) {
     callback,
     0,
     async_resource_name,
-    1,
+    1000, // max_queue_size
     1,
     NULL,
     NULL,
@@ -371,6 +363,11 @@ NAPI_METHOD(dcn_context_unref) {
 
   TRACE("Unrefing dc_context");
   dcn_context->gc = 1;
+  if (dcn_context->event_handler_thread != 0) {
+    dc_stop_io(dcn_context->dc_context);
+    uv_thread_join(&dcn_context->event_handler_thread);
+    dcn_context->event_handler_thread = 0;
+  }
   dc_context_unref(dcn_context->dc_context);
   dcn_context->dc_context = NULL;
 
@@ -2922,6 +2919,11 @@ NAPI_METHOD(dcn_accounts_unref) {
 
   TRACE("Unrefing dc_accounts");
   dcn_accounts->gc = 1;
+  if (dcn_accounts->event_handler_thread != 0) {
+    dc_accounts_stop_io(dcn_accounts->dc_accounts);
+    uv_thread_join(&dcn_accounts->event_handler_thread);
+    dcn_accounts->event_handler_thread = 0;
+  }
   dc_accounts_unref(dcn_accounts->dc_accounts);
   dcn_accounts->dc_accounts = NULL;
 
@@ -3093,8 +3095,8 @@ static void accounts_event_handler_thread_func(void* arg)
     }
     event = dc_accounts_get_next_event(dc_accounts_event_emitter);
     if (event == NULL) {
-      //TRACE("received NULL event, skipping");
-      continue;
+      TRACE("no more events");
+      break;
     }
 
     if (!dcn_accounts->threadsafe_event_handler) {
@@ -3216,7 +3218,7 @@ NAPI_METHOD(dcn_accounts_start_event_handler) {
     callback,
     0,
     async_resource_name,
-    1,
+    1000, // max_queue_size
     1,
     NULL,
     NULL,

node/test/test.js

@@ -128,7 +128,7 @@ describe('Basic offline Tests', function () {
 
     await expect(
       context.configure({ addr: 'delta1@delta.localhost' })
-    ).to.eventually.be.rejectedWith('Please enter a password.')
+    ).to.eventually.be.rejectedWith('Missing (IMAP) password.')
     await expect(context.configure({ mailPw: 'delta1' })).to.eventually.be
       .rejected
 
@@ -601,8 +601,8 @@ describe('Offline Tests with unconfigured account', function () {
     strictEqual(chatList.getCount(), 1, 'only one archived')
   })
 
-  it('Remove qoute from (draft) message', function () {
-    context.addDeviceMessage('test_qoute', 'test')
+  it('Remove quote from (draft) message', function () {
+    context.addDeviceMessage('test_quote', 'test')
     const msgId = context.getChatMessages(10, 0, 0)[0]
     const msg = context.messageNew()
 

package.json

@@ -19,7 +19,6 @@
     "prebuildify": "^3.0.0",
     "prebuildify-ci": "^1.0.4",
     "prettier": "^2.0.5",
-    "split2": "^4.1.0",
     "typedoc": "^0.17.0",
     "typescript": "^3.9.10"
   },
@@ -42,7 +41,7 @@
     "build": "npm run build:core && npm run build:bindings",
     "build:bindings": "npm run build:bindings:c && npm run build:bindings:ts",
     "build:bindings:c": "npm run build:bindings:c:c && npm run build:bindings:c:postinstall",
-    "build:bindings:c:c": "cd node && npx node-gyp rebuild",
+    "build:bindings:c:c": "cd node && node-gyp rebuild",
     "build:bindings:c:postinstall": "node node/scripts/postinstall.js",
     "build:bindings:ts": "cd node && tsc",
     "build:core": "npm run build:core:rust && npm run build:core:constants",
@@ -58,8 +57,8 @@
     "prebuildify": "cd node && prebuildify -t 16.13.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "test": "npm run test:lint && npm run test:mocha",
     "test:lint": "npm run lint",
-    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec"
+    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.85.0"
-}
+  "version": "1.89.0"
+}

python/README.rst

@@ -1,84 +1,77 @@
 =========================
-deltachat python bindings
+DeltaChat Python bindings
 =========================
 
-This package provides bindings to the deltachat-core_ Rust -library
-which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
+This package provides `Python bindings`_ to the `deltachat-core library`_
+which implements IMAP/SMTP/MIME/OpenPGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
+.. _`deltachat-core library`: https://github.com/deltachat/deltachat-core-rust
+.. _`Python bindings`: https://py.delta.chat/
 
 Installing pre-built packages (Linux-only)
-========================================================
+==========================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
+Otherwise you need to `compile the Delta Chat bindings yourself`__.
+
+__ sourceinstall_
 
 We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
-        virtualenv venv  # or: python -m venv
-        source venv/bin/activate
+        virtualenv env  # or: python -m venv
+        source env/bin/activate
 
 Afterwards, invoking ``python`` or ``pip install`` only
-modifies files in your ``venv`` directory and leaves
+modifies files in your ``env`` directory and leaves
 your system installation alone.
 
-For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest
-github ``master`` branch::
+For Linux we build wheels for all releases and push them to a python package
+index. To install the latest release::
 
-    pip install --pre -i https://m.devpi.net/dc/master deltachat
+    pip install deltachat
 
 To verify it worked::
 
     python -c "import deltachat"
 
-.. note::
-
-    If you can help to automate the building of wheels for Mac or Windows,
-    that'd be much appreciated! please then get
-    `in contact with us <https://delta.chat/en/contribute>`_.
-
-
 Running tests
 =============
 
-After successful binding installation you can install a few more
-Python packages before running the tests::
+Recommended way to run tests is using `tox <https://tox.wiki>`_.
+After successful binding installation you can install tox
+and run the tests::
 
-    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
+    pip install tox
+    tox -e py3
 
 This will run all "offline" tests and skip all functional
 end-to-end tests that require accounts on real e-mail servers.
 
 .. _livetests:
 
-running "live" tests with temporary accounts
----------------------------------------------
+Running "live" tests with temporary accounts
+--------------------------------------------
 
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLs created and managed by `mailadm <https://mailadm.readthedocs.io/>`_.
 
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
+Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this::
 
     export DCC_NEW_TMP_EMAIL=<URL you got from us>
 
 With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
-One hour is enough to invoke pytest and run all offline and online tests:
+One hour is enough to invoke pytest and run all offline and online tests::
 
-    pytest
-
-    # or if you have installed pytest-xdist for parallel test execution
-    pytest -n6
+    tox -e py3
 
 Each test run creates new accounts.
 
-
 .. _sourceinstall:
 
-Installing bindings from source (Updated: July 2020)
-=========================================================
+Installing bindings from source
+===============================
 
 Install Rust and Cargo first.
 The easiest is probably to use `rustup <https://rustup.rs/>`_.
@@ -97,74 +90,42 @@ E.g. on Debian-based systems `apt install python3 python3-pip
 python3-venv` should give you a usable python installation.
 
 Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
+virtual environment with dependencies using tox
+and activate it in your shell::
 
    cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
+   tox --devenv env
+   source env/bin/activate
 
 You should now be able to build the python bindings using the supplied script::
 
-   python install_python_bindings.py
+   python3 install_python_bindings.py
 
 The core compilation and bindings building might take a while,
 depending on the speed of your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-
-Code examples
-=============
-
-You may look at `examples <https://py.delta.chat/examples.html>`_.
-
-
-.. _`deltachat-core-rust github repository`: https://github.com/deltachat/deltachat-core-rust
-.. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
-
 
 Building manylinux based wheels
-====================================
+===============================
 
 Building portable manylinux wheels which come with libdeltachat.so
-can be done with docker-tooling.
+can be done with Docker_ or Podman_.
 
-using docker pull / premade images
-------------------------------------
+.. _Docker: https://www.docker.com/
+.. _Podman: https://podman.io/
 
-We publish a build environment under the ``deltachat/coredeps`` tag so
-that you can pull it from the ``hub.docker.com`` site's "deltachat"
-organization::
+If you want to build your own wheels, build container image first::
 
-    $ docker pull deltachat/coredeps
+   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
+   $ docker build -t deltachat/coredeps scripts/coredeps
 
-This docker image can be used to run tests and build Python wheels for all interpreters::
-
-    $ docker run -e DCC_NEW_TMP_EMAIL \
-       --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
-
-
-Optionally build your own docker image
---------------------------------------
-
-If you want to build your own custom docker image you can do this::
-
-   $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps scripts/docker_coredeps
-
-This will use the ``scripts/docker_coredeps/Dockerfile`` to build
-up docker image called ``deltachat/coredeps``.  You can afterwards
+This will use the ``scripts/coredeps/Dockerfile`` to build
+container image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 
    $ docker images
 
+This docker image can be used to run tests and build Python wheels for all interpreters::
 
-Troubleshooting
----------------
-
-On more recent systems running the docker image may crash.  You can
-fix this by adding ``vsyscall=emulate`` to the Linux kernel boot
-arguments commandline.  E.g. on Debian you'd add this to
-``GRUB_CMDLINE_LINUX_DEFAULT`` in ``/etc/default/grub``.
+    $ docker run -e DCC_NEW_TMP_EMAIL \
+       --rm -it -v $(pwd):/mnt -w /mnt \
+       deltachat/coredeps scripts/run_all.sh

python/doc/_templates/globaltoc.html

@@ -11,8 +11,7 @@
 <ul>
   <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
+  <li><a href="https://web.libera.chat/#deltachat">#deltachat</a></li>
 </ul>
 
-<b>#deltachat [freenode]</b>
-
 </div>

python/fail_test.py

@@ -1,7 +0,0 @@
-from __future__ import print_function
-from deltachat import capi
-from deltachat.capi import ffi, lib
-
-if __name__ == "__main__":
-    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
-    lib.dc_stop_io(ctx)

python/install_python_bindings.py

@@ -18,15 +18,13 @@ if __name__ == "__main__":
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
 
-    if target == 'release':
+    if target == "release":
         os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
 
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll", shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
-        subprocess.check_call([
-            sys.executable, "-m", "pip", "install", "-e", "."
-        ])
+        subprocess.check_call([sys.executable, "-m", "pip", "install", "-e", "."])

python/pyproject.toml

@@ -1,7 +1,41 @@
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
+requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2", "cffi>=1.0.0", "pkgconfig"]
 build-backend = "setuptools.build_meta"
 
+[project]
+name = "deltachat"
+description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
+readme = "README.rst"
+requires-python = ">=3.7"
+authors = [
+    { name = "holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors" },
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    "Programming Language :: Python :: 3",
+    "Topic :: Communications :: Email",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = [
+    "cffi>=1.0.0",
+    "imap-tools",
+    "pluggy",
+    "requests",
+]
+dynamic = [
+    "version"
+]
+
+[project.urls]
+"Home" = "https://github.com/deltachat/deltachat-core-rust/"
+"Bug Tracker" = "https://github.com/deltachat/deltachat-core-rust/issues"
+"Documentation" = "https://py.delta.chat/"
+
+[project.entry-points.pytest11]
+"deltachat.testplugin" = "deltachat.testplugin"
+
 [tool.setuptools_scm]
 root = ".."
 tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
@@ -9,3 +43,6 @@ git_describe_command = "git describe --dirty --tags --long --match py-*.*"
 
 [tool.black]
 line-length = 120
+
+[tool.isort]
+profile = "black"

python/setup.py

@@ -1,39 +1,4 @@
-import setuptools
-import os
-import re
-
-
-def main():
-    with open("README.rst") as f:
-        long_description = f.read()
-    setuptools.setup(
-        name='deltachat',
-        description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
-        long_description=long_description,
-        author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
-        setup_requires=[
-            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
-            'pkgconfig',
-        ],
-        packages=setuptools.find_packages('src'),
-        package_dir={'': 'src'},
-        cffi_modules=['src/deltachat/_build.py:ffibuilder'],
-        entry_points = {
-            'pytest11': [
-                'deltachat.testplugin = deltachat.testplugin',
-            ],
-        },
-        classifiers=[
-            'Development Status :: 4 - Beta',
-            'Intended Audience :: Developers',
-            'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
-            'Programming Language :: Python :: 3',
-            'Topic :: Communications :: Email',
-            'Topic :: Software Development :: Libraries',
-        ],
-    )
-
+from setuptools import setup
 
 if __name__ == "__main__":
-    main()
+    setup(cffi_modules=["src/deltachat/_build.py:ffibuilder"])

python/src/deltachat/account.py

@@ -62,12 +62,15 @@ class Account(object):
 
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True) -> None:
+    def __init__(self, db_path, os_name=None, logging=True, closed=False) -> None:
         """initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param os_name: this will be put to the X-Mailer header in outgoing messages
+        :param os_name: [Deprecated]
+        :param logging: enable logging for this account
+        :param closed: set to True to avoid automatically opening the account
+                       after creation.
         """
         # initialize per-account plugin system
         self._pm = hookspec.PerAccount._make_plugin_manager()
@@ -80,7 +83,7 @@ class Account(object):
             db_path = db_path.encode("utf8")
 
         self._dc_context = ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(os_name), db_path, ffi.NULL),
+            lib.dc_context_new_closed(db_path) if closed else lib.dc_context_new(ffi.NULL, db_path, ffi.NULL),
             lib.dc_context_unref,
         )
         if self._dc_context == ffi.NULL:
@@ -92,6 +95,17 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
+    def open(self, passphrase: Optional[str] = None) -> bool:
+        """Open the account's database with the given passphrase.
+        This can only be used on a closed account. If the account is new, this
+        operation sets the database passphrase. For existing databases the passphrase
+        should be the one used to encrypt the database the first time.
+
+        :returns: True if the database is opened with this passphrase, False if the
+                  passphrase is incorrect or an error occurred.
+        """
+        return bool(lib.dc_context_open(self._dc_context, as_dc_charpointer(passphrase)))
+
     def disable_logging(self) -> None:
         """disable logging."""
         self._logging = False
@@ -209,13 +223,13 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return True if lib.dc_is_configured(self._dc_context) else False
+        return bool(lib.dc_is_configured(self._dc_context))
 
     def is_open(self) -> bool:
         """Determine if account is open
 
         :returns True if account is open."""
-        return True if lib.dc_context_is_open(self._dc_context) else False
+        return bool(lib.dc_context_is_open(self._dc_context))
 
     def set_avatar(self, img_path: Optional[str]) -> None:
         """Set self avatar.
@@ -461,21 +475,24 @@ class Account(object):
         """
         return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
 
-    def export_all(self, path):
-        """return new file containing a backup of all database state
-        (chats, contacts, keys, media, ...). The file is created in the
-        the `path` directory.
+    def export_all(self, path: str, passphrase: Optional[str] = None) -> str:
+        """Export a backup of all database state (chats, contacts, keys, media, ...).
+
+        :param path: the directory where the backup will be stored.
+        :param passphrase: the backup will be encrypted with the passphrase, if it is
+                           None or empty string, the backup is not encrypted.
+        :returns: path to the created backup.
 
         Note that the account has to be stopped; call stop_io() if necessary.
         """
-        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
+        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP, passphrase)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
-    def _export(self, path, imex_cmd):
+    def _export(self, path: str, imex_cmd: int, passphrase: Optional[str] = None) -> list:
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            self.imex(path, imex_cmd, passphrase)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -487,21 +504,23 @@ class Account(object):
         """
         self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
 
-    def import_all(self, path):
-        """import delta chat state from the specified backup `path` (a file).
-
+    def import_all(self, path: str, passphrase: Optional[str] = None) -> None:
+        """Import Delta Chat state from the specified backup file.
         The account must be in unconfigured state for import to attempted.
+
+        :param path: path to the backup file.
+        :param passphrase: if not None or empty, the backup will be opened with the passphrase.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
+        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP, passphrase=passphrase)
 
-    def _import(self, path, imex_cmd):
+    def _import(self, path: str, imex_cmd: int, passphrase: Optional[str] = None) -> None:
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            self.imex(path, imex_cmd, passphrase)
             imex_tracker.wait_finish()
 
-    def imex(self, path, imex_cmd):
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+    def imex(self, path: str, imex_cmd: int, passphrase: Optional[str] = None) -> None:
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), as_dc_charpointer(passphrase))
 
     def initiate_key_transfer(self) -> str:
         """return setup code after a Autocrypt setup message

python/src/deltachat/chat.py

@@ -32,6 +32,8 @@ class Chat(object):
         self.id = id
 
     def __eq__(self, other) -> bool:
+        if other is None:
+            return False
         return self.id == getattr(other, "id", None) and self.account._dc_context == other.account._dc_context
 
     def __ne__(self, other) -> bool:
@@ -65,27 +67,65 @@ class Chat(object):
     # ------  chat status/metadata API ------------------------------
 
     def is_group(self) -> bool:
-        """return true if this chat is a group chat.
+        """Return True if this chat is a group chat.
 
-        :returns: True if chat is a group-chat, false otherwise
+        :returns: True if chat is a group-chat, False otherwise
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
+    def is_single(self) -> bool:
+        """Return True if this chat is a single/direct chat, False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_SINGLE
+
+    def is_mailinglist(self) -> bool:
+        """Return True if this chat is a mailing list, False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_MAILINGLIST
+
+    def is_broadcast(self) -> bool:
+        """Return True if this chat is a broadcast list, False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_BROADCAST
+
+    def is_multiuser(self) -> bool:
+        """Return True if this chat is a multi-user chat (group, mailing list or broadcast), False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_MAILINGLIST,
+            const.DC_CHAT_TYPE_BROADCAST,
+        )
+
+    def is_self_talk(self) -> bool:
+        """Return True if this chat is the self-chat (a.k.a. "Saved Messages"), False otherwise"""
+        return bool(lib.dc_chat_is_self_talk(self._dc_chat))
+
+    def is_device_talk(self) -> bool:
+        """Returns True if this chat is the "Device Messages" chat, False otherwise"""
+        return bool(lib.dc_chat_is_device_talk(self._dc_chat))
+
     def is_muted(self) -> bool:
         """return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
         """
-        return lib.dc_chat_is_muted(self._dc_chat)
+        return bool(lib.dc_chat_is_muted(self._dc_chat))
 
-    def is_contact_request(self):
+    def is_pinned(self) -> bool:
+        """Return True if this chat is pinned, False otherwise"""
+        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_PINNED
+
+    def is_archived(self) -> bool:
+        """Return True if this chat is archived, False otherwise.
+        :returns: True if archived, False otherwise
+        """
+        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
+
+    def is_contact_request(self) -> bool:
         """return True if this chat is a contact request chat.
 
         :returns: True if chat is a contact request chat, False otherwise.
         """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
+        return bool(lib.dc_chat_is_contact_request(self._dc_chat))
 
-    def is_promoted(self):
+    def is_promoted(self) -> bool:
         """return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
         have been sent messages.
@@ -100,14 +140,14 @@ class Chat(object):
 
         :returns: True if the chat is writable, False otherwise
         """
-        return lib.dc_chat_can_send(self._dc_chat)
+        return bool(lib.dc_chat_can_send(self._dc_chat))
 
     def is_protected(self) -> bool:
         """return True if this chat is a protected chat.
 
         :returns: True if chat is protected, False otherwise.
         """
-        return lib.dc_chat_is_protected(self._dc_chat)
+        return bool(lib.dc_chat_is_protected(self._dc_chat))
 
     def get_name(self) -> Optional[str]:
         """return name of this chat.
@@ -125,6 +165,18 @@ class Chat(object):
         name = as_dc_charpointer(name)
         return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
 
+    def get_color(self):
+        """return the color of the chat.
+        :returns: color as 0x00rrggbb
+        """
+        return lib.dc_chat_get_color(self._dc_chat)
+
+    def get_summary(self):
+        """return dictionary with summary information."""
+        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
+        s = from_dc_charpointer(dc_res)
+        return json.loads(s)
+
     def mute(self, duration: Optional[int] = None) -> None:
         """mutes the chat
 
@@ -148,6 +200,24 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
+    def pin(self) -> None:
+        """Pin the chat."""
+        lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_PINNED)
+
+    def unpin(self) -> None:
+        """Unpin the chat."""
+        if self.is_pinned():
+            lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_NORMAL)
+
+    def archive(self) -> None:
+        """Archive the chat."""
+        lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_ARCHIVED)
+
+    def unarchive(self) -> None:
+        """Unarchive the chat."""
+        if self.is_archived():
+            lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_NORMAL)
+
     def get_mute_duration(self) -> int:
         """Returns the number of seconds until the mute of this chat is lifted.
 
@@ -364,12 +434,6 @@ class Chat(object):
         """
         return lib.dc_marknoticed_chat(self.account._dc_context, self.id)
 
-    def get_summary(self):
-        """return dictionary with summary information."""
-        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
-        s = from_dc_charpointer(dc_res)
-        return json.loads(s)
-
     # ------  group management API ------------------------------
 
     def add_contact(self, obj):
@@ -460,12 +524,6 @@ class Chat(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_color(self):
-        """return the color of the chat.
-        :returns: color as 0x00rrggbb
-        """
-        return lib.dc_chat_get_color(self._dc_chat)
-
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
@@ -474,12 +532,6 @@ class Chat(object):
         """
         return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)
 
-    def is_archived(self):
-        """return True if this chat is archived.
-        :returns: True if archived.
-        """
-        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
-
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.
 

python/src/deltachat/contact.py

@@ -22,7 +22,9 @@ class Contact(object):
         self.account = account
         self.id = id
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
+        if other is None:
+            return False
         return self.account._dc_context == other.account._dc_context and self.id == other.id
 
     def __ne__(self, other):

python/src/deltachat/events.py

@@ -271,7 +271,8 @@ class EventThread(threading.Thread):
             data1 = ffi_event.data1
             if data1 == 0 or data1 == 1000:
                 success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
+                comment = ffi_event.data2
+                yield "ac_configure_completed", dict(success=success, comment=comment)
         elif name == "DC_EVENT_INCOMING_MSG":
             msg = account.get_message_by_id(ffi_event.data2)
             yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))

python/src/deltachat/hookspec.py

@@ -38,7 +38,7 @@ class PerAccount:
         """log a message related to the account."""
 
     @account_hookspec
-    def ac_configure_completed(self, success):
+    def ac_configure_completed(self, success, comment):
         """Called after a configure process completed."""
 
     @account_hookspec

python/src/deltachat/message.py

@@ -1,9 +1,10 @@
 """ The Message object. """
 
+import json
 import os
 import re
 from datetime import datetime, timezone
-from typing import Optional
+from typing import Optional, Union
 
 from . import const, props
 from .capi import ffi, lib
@@ -26,7 +27,9 @@ class Message(object):
         msg_id = self.id
         assert msg_id is not None and msg_id >= 0, repr(msg_id)
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
+        if other is None:
+            return False
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
@@ -52,8 +55,8 @@ class Message(object):
     def new_empty(cls, account, view_type):
         """create a non-persistent message.
 
-        :param: view_type is the message type code or one of the strings:
-           "text", "audio", "video", "file", "sticker"
+        :param view_type: the message type code or one of the strings:
+           "text", "audio", "video", "file", "sticker", "videochat", "webxdc"
         """
         if isinstance(view_type, int):
             view_type_code = view_type
@@ -128,6 +131,36 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
+    def get_status_updates(self, serial: int = 0) -> list:
+        """Get the status updates of this webxdc message.
+
+        The status updates may be sent by yourself or by other members.
+        If this message doesn't have a webxdc instance, an empty list is returned.
+
+        :param serial: The last known serial. Pass 0 if there are no known serials to receive all updates.
+        """
+        return json.loads(
+            from_dc_charpointer(lib.dc_get_webxdc_status_updates(self.account._dc_context, self.id, serial))
+        )
+
+    def send_status_update(self, json_data: Union[str, dict], description: str) -> bool:
+        """Send an status update for the webxdc instance of this message.
+
+        If the webxdc instance is a draft, the update is not sent immediately.
+        Instead, the updates are collected and sent out in a batch when the instance is actually sent.
+
+        :param json_data: program-readable data, the actual payload.
+        :param description: The user-visible description of JSON data
+        :returns: True on success, False otherwise
+        """
+        if isinstance(json_data, dict):
+            json_data = json.dumps(json_data, default=str)
+        return bool(
+            lib.dc_send_webxdc_status_update(
+                self.account._dc_context, self.id, as_dc_charpointer(json_data), as_dc_charpointer(description)
+            )
+        )
+
     def is_system_message(self):
         """return True if this message is a system/info message."""
         return bool(lib.dc_msg_is_info(self._dc_msg))
@@ -159,6 +192,10 @@ class Message(object):
         """
         return from_dc_charpointer(lib.dc_get_msg_info(self.account._dc_context, self.id))
 
+    def get_summarytext(self, width: int) -> str:
+        """Get a message summary as a single line of text. Typically used for notifications."""
+        return from_dc_charpointer(lib.dc_msg_get_summarytext(self._dc_msg, width))
+
     def continue_key_transfer(self, setup_code):
         """extract key and use it as primary key for this account."""
         res = lib.dc_continue_key_transfer(self.account._dc_context, self.id, as_dc_charpointer(setup_code))
@@ -396,6 +433,14 @@ class Message(object):
         """return True if it's a video message."""
         return self._view_type == const.DC_MSG_VIDEO
 
+    def is_videochat_invitation(self):
+        """return True if it's a videochat invitation message."""
+        return self._view_type == const.DC_MSG_VIDEOCHAT_INVITATION
+
+    def is_webxdc(self):
+        """return True if it's a Webxdc message."""
+        return self._view_type == const.DC_MSG_WEBXDC
+
     def is_file(self):
         """return True if it's a file message."""
         return self._view_type == const.DC_MSG_FILE
@@ -415,6 +460,8 @@ _view_type_mapping = {
     "video": const.DC_MSG_VIDEO,
     "file": const.DC_MSG_FILE,
     "sticker": const.DC_MSG_STICKER,
+    "videochat": const.DC_MSG_VIDEOCHAT_INVITATION,
+    "webxdc": const.DC_MSG_WEBXDC,
 }
 
 

python/src/deltachat/testplugin.py

@@ -11,7 +11,7 @@ import threading
 import time
 import weakref
 from queue import Queue
-from typing import Callable, List
+from typing import Callable, List, Optional
 
 import pytest
 import requests
@@ -241,6 +241,7 @@ def data(request):
                 os.path.normpath(x)
                 for x in [
                     os.path.join(os.path.dirname(request.fspath.strpath), "data"),
+                    os.path.join(os.path.dirname(request.fspath.strpath), "..", "..", "test-data"),
                     os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data"),
                 ]
             ]
@@ -293,8 +294,8 @@ class ACSetup:
 
         class PendingTracker:
             @account_hookimpl
-            def ac_configure_completed(this, success):
-                self._configured_events.put((account, success))
+            def ac_configure_completed(this, success: bool, comment: Optional[str]) -> None:
+                self._configured_events.put((account, success, comment))
 
         account.add_account_plugin(PendingTracker(), name="pending_tracker")
         self._account2state[account] = self.CONFIGURING
@@ -332,9 +333,9 @@ class ACSetup:
         print("finished, account2state", self._account2state)
 
     def _pop_config_success(self):
-        acc, success = self._configured_events.get()
+        acc, success, comment = self._configured_events.get()
         if not success:
-            pytest.fail("configuring online account failed: {}".format(acc))
+            pytest.fail("configuring online account {} failed: {}".format(acc, comment))
         self._account2state[acc] = self.CONFIGURED
         return acc
 
@@ -429,16 +430,16 @@ class ACFactory:
         if addr in self.testprocess._addr2files:
             return self._getaccount(addr)
 
-    def get_unconfigured_account(self):
-        return self._getaccount()
+    def get_unconfigured_account(self, closed=False):
+        return self._getaccount(closed=closed)
 
-    def _getaccount(self, try_cache_addr=None):
+    def _getaccount(self, try_cache_addr=None, closed=False):
         logid = "ac{}".format(len(self._accounts) + 1)
         # we need to use fixed database basename for maybe_cache_* functions to work
         path = self.tmpdir.mkdir(logid).join("dc.db")
         if try_cache_addr:
             self.testprocess.cache_maybe_retrieve_configured_db_files(try_cache_addr, path)
-        ac = Account(path.strpath, logging=self._logging)
+        ac = Account(path.strpath, logging=self._logging, closed=closed)
         ac._logid = logid  # later instantiated FFIEventLogger needs this
         ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
         if self.pytestconfig.getoption("--debug-setup"):
@@ -467,9 +468,11 @@ class ACFactory:
             else:
                 print("WARN: could not use preconfigured keys for {!r}".format(addr))
 
-    def get_pseudo_configured_account(self):
+    def get_pseudo_configured_account(self, passphrase: Optional[str] = None) -> Account:
         # do a pseudo-configured account
-        ac = self.get_unconfigured_account()
+        ac = self.get_unconfigured_account(closed=bool(passphrase))
+        if passphrase:
+            ac.open(passphrase)
         acname = ac._logid
         addr = "{}@offline.org".format(acname)
         ac.update_config(

python/tests/auditwheels.py

@@ -1,12 +1,10 @@
 import os
-import platform
 import subprocess
 import sys
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
-    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
@@ -17,7 +15,5 @@ if __name__ == "__main__":
                     p,
                     "-w",
                     workspacedir,
-                    "--plat",
-                    "manylinux2014_" + arch,
                 ]
             )

python/tests/package_wheels.py

@@ -1,13 +0,0 @@
-import os
-import subprocess
-import sys
-
-if __name__ == "__main__":
-    assert len(sys.argv) == 2
-    wheelhousedir = sys.argv[1]
-    # pip wheel will build in an isolated tmp dir that does not have git
-    # history so setuptools_scm can not automatically determine a
-    # version there. So pass in the version through an env var.
-    version = subprocess.check_output(["python", "setup.py", "--version"]).strip().split(b"\n")[-1]
-    os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version.decode("ascii")
-    subprocess.check_call(("pip wheel . -w %s" % wheelhousedir).split())

python/tests/test_1_online.py

@@ -238,6 +238,70 @@ def test_html_message(acfactory, lp):
     assert html_text in msg2.html
 
 
+def test_videochat_invitation_message(acfactory, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    chat = acfactory.get_accepted_chat(ac1, ac2)
+    text = "You are invited to a video chat, click https://meet.jit.si/WxEGad0gGzX to join."
+
+    lp.sec("ac1: prepare and send text message to ac2")
+    msg1 = chat.send_text("message0")
+    assert not msg1.is_videochat_invitation()
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == "message0"
+    assert not msg2.is_videochat_invitation()
+
+    lp.sec("ac1: prepare and send videochat invitation to ac2")
+    msg1 = Message.new_empty(ac1, "videochat")
+    msg1.set_text(text)
+    msg1 = chat.send_msg(msg1)
+    assert msg1.is_videochat_invitation()
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == text
+    assert msg2.is_videochat_invitation()
+
+
+def test_webxdc_message(acfactory, data, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    chat = acfactory.get_accepted_chat(ac1, ac2)
+
+    lp.sec("ac1: prepare and send text message to ac2")
+    msg1 = chat.send_text("message0")
+    assert not msg1.is_webxdc()
+    assert not msg1.send_status_update({"payload": "not an webxdc"}, "invalid")
+    assert not msg1.get_status_updates()
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == "message0"
+    assert not msg2.is_webxdc()
+    assert not msg1.get_status_updates()
+
+    lp.sec("ac1: prepare and send webxdc instance to ac2")
+    msg1 = Message.new_empty(ac1, "webxdc")
+    msg1.set_text("message1")
+    msg1.set_file(data.get_path("webxdc/minimal.xdc"))
+    msg1 = chat.send_msg(msg1)
+    assert msg1.is_webxdc()
+    assert msg1.filename
+
+    assert msg1.send_status_update({"payload": "test1"}, "some test data")
+    assert msg1.send_status_update({"payload": "test2"}, "more test data")
+    assert len(msg1.get_status_updates()) == 2
+    update1 = msg1.get_status_updates()[0]
+    assert update1["payload"] == "test1"
+    assert len(msg1.get_status_updates(update1["serial"])) == 1
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == "message1"
+    assert msg2.is_webxdc()
+    assert msg2.filename
+
+
 def test_mvbox_sentbox_threads(acfactory, lp):
     lp.sec("ac1: start with mvbox thread")
     ac1 = acfactory.new_online_configuring_account(mvbox_move=True, sentbox_watch=True)
@@ -1369,8 +1433,9 @@ def test_set_get_contact_avatar(acfactory, data, lp):
 
 
 def test_add_remove_member_remote_events(acfactory, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
     ac1_addr = ac1.get_config("addr")
+    ac3_addr = ac3.get_config("addr")
     # activate local plugin for ac2
     in_list = queue.Queue()
 
@@ -1412,7 +1477,7 @@ def test_add_remove_member_remote_events(acfactory, lp):
     lp.sec("ac1: add address2")
     # note that if the above create_chat() would not
     # happen we would not receive a proper member_added event
-    contact2 = chat.add_contact("devnull@testrun.org")
+    contact2 = chat.add_contact(ac3_addr)
     ev = in_list.get()
     assert ev.action == "chat-modified"
     ev = in_list.get()
@@ -1420,7 +1485,7 @@ def test_add_remove_member_remote_events(acfactory, lp):
     ev = in_list.get()
     assert ev.action == "added"
     assert ev.message.get_sender_contact().addr == ac1_addr
-    assert ev.contact.addr == "devnull@testrun.org"
+    assert ev.contact.addr == ac3_addr
 
     lp.sec("ac1: remove address2")
     chat.remove_contact(contact2)
@@ -1789,7 +1854,7 @@ def test_configure_error_msgs_invalid_server(acfactory):
     # Can't connect so it probably should say something about "internet"
     # again, should not repeat itself
     # If this fails then probably `e.msg.to_lowercase().contains("could not resolve")`
-    # in configure/mod.rs returned false because the error message was changed
+    # in configure.rs returned false because the error message was changed
     # (i.e. did not contain "could not resolve" anymore)
     assert (ev.data2.count("internet") + ev.data2.count("network")) == 1
     # Should mention that it can't connect:

python/tests/test_3_offline.py

@@ -11,6 +11,7 @@ from deltachat.capi import ffi, lib
 from deltachat.cutil import iter_array
 from deltachat.hookspec import account_hookimpl
 from deltachat.message import Message
+from deltachat.tracker import ImexFailed
 
 
 @pytest.mark.parametrize(
@@ -154,9 +155,11 @@ class TestOfflineContact:
         ac1 = acfactory.get_pseudo_configured_account()
         contact1 = ac1.create_contact("some1@example.org", name="some1")
         contact2 = ac1.create_contact("some1@example.org", name="some1")
+        contact3 = None
         str(contact1)
         repr(contact1)
         assert contact1 == contact2
+        assert contact1 != contact3
         assert contact1.id
         assert contact1.addr == "some1@example.org"
         assert contact1.display_name == "some1"
@@ -250,10 +253,12 @@ class TestOfflineChat:
     def test_chat_idempotent(self, chat1, ac1):
         contact1 = chat1.get_contacts()[0]
         chat2 = contact1.create_chat()
+        chat3 = None
         assert chat2.id == chat1.id
         assert chat2.get_name() == chat1.get_name()
         assert chat1 == chat2
         assert not (chat1 != chat2)
+        assert chat1 != chat3
 
         for ichat in ac1.get_chats():
             if ichat.id == chat1.id:
@@ -405,6 +410,8 @@ class TestOfflineChat:
 
     def test_message_eq_contains(self, chat1):
         msg = chat1.send_text("msg1")
+        msg2 = None
+        assert msg != msg2
         assert msg in chat1.get_messages()
         assert not (msg not in chat1.get_messages())
         str(msg)
@@ -496,7 +503,7 @@ class TestOfflineChat:
         with pytest.raises(ValueError):
             ac1.set_config("addr", "123@example.org")
 
-    def test_import_export_one_contact(self, acfactory, tmpdir):
+    def test_import_export_on_unencrypted_acct(self, acfactory, tmpdir):
         backupdir = tmpdir.mkdir("backup")
         ac1 = acfactory.get_pseudo_configured_account()
         chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
@@ -525,6 +532,162 @@ class TestOfflineChat:
         assert messages[0].text == "msg1"
         assert os.path.exists(messages[1].filename)
 
+    def test_import_export_on_encrypted_acct(self, acfactory, tmpdir):
+        passphrase1 = "passphrase1"
+        passphrase2 = "passphrase2"
+        backupdir = tmpdir.mkdir("backup")
+        ac1 = acfactory.get_pseudo_configured_account(passphrase=passphrase1)
+
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+        assert not backupdir.listdir()
+        ac1.stop_io()
+
+        path = ac1.export_all(backupdir.strpath)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account(closed=True)
+        ac2.open(passphrase2)
+        ac2.import_all(path)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+        ac2.shutdown()
+
+        # check that passphrase is not lost after import:
+        ac2 = Account(ac2.db_path, logging=ac2._logging, closed=True)
+        ac2.open(passphrase2)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+    def test_import_export_with_passphrase(self, acfactory, tmpdir):
+        passphrase = "test_passphrase"
+        wrong_passphrase = "wrong_passprase"
+        backupdir = tmpdir.mkdir("backup")
+        ac1 = acfactory.get_pseudo_configured_account()
+
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+        assert not backupdir.listdir()
+        ac1.stop_io()
+
+        path = ac1.export_all(backupdir.strpath, passphrase)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account()
+        with pytest.raises(ImexFailed):
+            ac2.import_all(path, wrong_passphrase)
+        ac2.import_all(path, passphrase)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+    def test_import_encrypted_bak_into_encrypted_acct(self, acfactory, tmpdir):
+        """
+        Test that account passphrase isn't lost if backup failed to be imported.
+        See https://github.com/deltachat/deltachat-core-rust/issues/3379
+        """
+        acct_passphrase = "passphrase1"
+        bak_passphrase = "passphrase2"
+        wrong_passphrase = "wrong_passprase"
+        backupdir = tmpdir.mkdir("backup")
+
+        ac1 = acfactory.get_pseudo_configured_account()
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+        assert not backupdir.listdir()
+        ac1.stop_io()
+
+        path = ac1.export_all(backupdir.strpath, bak_passphrase)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account(closed=True)
+        ac2.open(acct_passphrase)
+        with pytest.raises(ImexFailed):
+            ac2.import_all(path, wrong_passphrase)
+        ac2.import_all(path, bak_passphrase)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+        ac2.shutdown()
+
+        # check that passphrase is not lost after import
+        ac2 = Account(ac2.db_path, logging=ac2._logging, closed=True)
+        ac2.open(acct_passphrase)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
     def test_set_get_draft(self, chat1):
         msg = Message.new_empty(chat1.account, "text")
         msg1 = chat1.prepare_message(msg)

python/tests/test_4_lowlevel.py

@@ -35,7 +35,7 @@ class TestACSetup:
         monkeypatch.setattr(acc, "configure", lambda **kwargs: None)
         pc.start_configure(acc)
         assert pc._account2state[acc] == pc.CONFIGURING
-        pc._configured_events.put((acc, True))
+        pc._configured_events.put((acc, True, None))
         monkeypatch.setattr(pc, "init_imap", lambda *args, **kwargs: None)
         pc.wait_one_configured(acc)
         assert pc._account2state[acc] == pc.CONFIGURED
@@ -55,11 +55,11 @@ class TestACSetup:
         pc.start_configure(ac2)
         assert pc._account2state[ac1] == pc.CONFIGURING
         assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac1, True))
+        pc._configured_events.put((ac1, True, None))
         pc.wait_one_configured(ac1)
         assert pc._account2state[ac1] == pc.CONFIGURED
         assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac2, True))
+        pc._configured_events.put((ac2, True, None))
         pc.bring_online()
         assert pc._account2state[ac1] == pc.IDLEREADY
         assert pc._account2state[ac2] == pc.IDLEREADY

python/tox.ini

@@ -9,9 +9,8 @@ envlist =
 [testenv]
 commands = 
     pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
-    python tests/package_wheels.py {toxworkdir}/wheelhouse
+    pip wheel . -w {toxworkdir}/wheelhouse --no-deps
 passenv = 
-    TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_NEW_TMP_EMAIL 
@@ -28,6 +27,10 @@ deps =
 [testenv:auditwheels]
 skipsdist = True
 deps = auditwheel
+passenv =
+    AUDITWHEEL_ARCH
+    AUDITWHEEL_PLAT
+    AUDITWHEEL_POLICY
 commands = 
     python tests/auditwheels.py {toxworkdir}/wheelhouse
 
@@ -42,8 +45,8 @@ deps =
     pygments
     restructuredtext_lint 
 commands =
-    isort --check --profile black src/deltachat examples/ tests/
-    black --check src/deltachat examples/ tests/
+    isort --check setup.py install_python_bindings.py src/deltachat examples/ tests/
+    black --check setup.py install_python_bindings.py src/deltachat examples/ tests/
     flake8 src/deltachat
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
@@ -89,4 +92,4 @@ markers =
 
 [flake8]
 max-line-length = 120
-ignore = E203, E266, E501, W503
+ignore = E203, E266, E501, W503

rustfmt.toml

@@ -1 +0,0 @@
-edition = "2018"

scripts/README.md

@@ -14,18 +14,14 @@ and an own build machine.
 - `remote_tests_rust.sh` rsyncs to the build machine and runs
   `run-rust-test.sh` remotely on the build machine. 
 
-- `doxygen/Dockerfile` specifies an image that contains
-  the doxygen tool which is used by `run-doxygen.sh` 
-  to generate C-docs which are then uploaded 
-  via `ci_upload.sh` to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-  (and the master branch is linked to https://c.delta.chat proper). 
+- `run-doxygen.sh` generates C-docs which are then uploaded to https://c.delta.chat/
 
 
 ## Triggering runs on the build machine locally (fast!)
 
 There is experimental support for triggering a remote Python or Rust test run 
 from your local checkout/branch. You will need to be authorized to login to 
-the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
+the build machine (ask your friendly sysadmin on #deltachat Libera Chat) to type::
 
     scripts/manual_remote_tests.sh rust
     scripts/manual_remote_tests.sh python

scripts/concourse/docs_wheels.yml

@@ -29,18 +29,19 @@ jobs:
         - name: c-docs
       image_resource:
         source:
-          repository: hrektts/doxygen
+          repository: alpine
         type: registry-image
       platform: linux
       run:
-        path: bash
+        path: sh
         args:
-        - -exc
+        - -ec
         - |
+          apk add --no-cache doxygen git
           cd deltachat-core-rust
-          bash scripts/run-doxygen.sh
+          scripts/run-doxygen.sh
           cd ..
-          cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
+          cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/
 
   - task: upload-c-docs
     config:
@@ -78,14 +79,15 @@ jobs:
         - name: deltachat-core-rust
       image_resource:
         source:
-          repository: vito/oci-build-task
+          repository: concourse/oci-build-task
         type: registry-image
       outputs:
         - name: coredeps-image
           path: image
       params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps
+        CONTEXT: deltachat-core-rust/scripts/coredeps
         UNPACK_ROOTFS: "true"
+        BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_x86_64
       platform: linux
       caches:
       - path: cache
@@ -177,14 +179,15 @@ jobs:
         - name: deltachat-core-rust
       image_resource:
         source:
-          repository: vito/oci-build-task
+          repository: concourse/oci-build-task
         type: registry-image
       outputs:
         - name: coredeps-image
           path: image
       params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
+        CONTEXT: deltachat-core-rust/scripts/coredeps
         UNPACK_ROOTFS: "true"
+        BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_aarch64
       platform: linux
       caches:
       - path: cache
@@ -230,3 +233,73 @@ jobs:
           devpi use https://m.devpi.net/dc/master
           devpi login ((devpi.login)) --password ((devpi.password))
           devpi upload py-wheels/*manylinux201*
+
+- name: python-musl-x86_64
+  plan:
+  - get: deltachat-core-rust
+  - get: deltachat-core-rust-release
+    trigger: true
+
+  # Build manylinux image with additional dependencies
+  - task: build-coredeps
+    privileged: true
+    config:
+      inputs:
+        # Building the latest, not tagged coredeps
+        - name: deltachat-core-rust
+      image_resource:
+        source:
+          repository: concourse/oci-build-task
+        type: registry-image
+      outputs:
+        - name: coredeps-image
+          path: image
+      params:
+        CONTEXT: deltachat-core-rust/scripts/coredeps
+        UNPACK_ROOTFS: "true"
+        BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_x86_64
+      platform: linux
+      caches:
+      - path: cache
+      run:
+        path: build
+
+  # Use built image to build python wheels
+  - task: build-wheels
+    image: coredeps-image
+    config:
+      inputs:
+        - name: deltachat-core-rust-release
+          path: .
+      outputs:
+        - name: py-wheels
+          path: ./python/.docker-tox/wheelhouse/
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          scripts/run_all.sh
+
+  # Upload musl x86_64 wheels
+  - task: upload-wheels
+    config:
+      inputs:
+        - name: py-wheels
+      image_resource:
+        type: registry-image
+        source:
+          repository: debian
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apt-get update -y
+          apt-get install -y --no-install-recommends python3-pip python3-setuptools
+          pip3 install devpi
+          devpi use https://m.devpi.net/dc/master
+          devpi login ((devpi.login)) --password ((devpi.password))
+          devpi upload py-wheels/*musllinux_1_1_x86_64*

scripts/coredeps/Dockerfile

@@ -0,0 +1,8 @@
+ARG BASEIMAGE=quay.io/pypa/manylinux2014_x86_64
+#ARG BASEIMAGE=quay.io/pypa/musllinux_1_1_x86_64
+#ARG BASEIMAGE=quay.io/pypa/manylinux2014_aarch64
+
+FROM $BASEIMAGE
+RUN pipx install tox
+COPY install-rust.sh /scripts/
+RUN /scripts/install-rust.sh

scripts/coredeps/install-rust.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Install Rust
+#
+# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
+#
+# Avoid using rustup here as it depends on reading /proc/self/exe and
+# has problems running under QEMU.
+RUST_VERSION=1.61.0
+
+ARCH="$(uname -m)"
+test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu
+
+curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$ARCH-unknown-linux-$LIBC.tar.gz" | tar xz
+cd "rust-${RUST_VERSION}-$ARCH-unknown-linux-$LIBC"
+./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$ARCH-unknown-linux-$LIBC"
+rustc --version
+cd ..
+rm -fr "rust-${RUST_VERSION}-$ARCH-unknown-linux-$LIBC"

scripts/docker-coredeps-arm64/Dockerfile

@@ -1,21 +0,0 @@
-FROM quay.io/pypa/manylinux2014_aarch64
-
-# Configure ld.so/ldconfig and pkg-config
-RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
-    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
-ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
-
-# Install a recent Perl, needed to install the openssl crate
-ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-
-ENV PIP_DISABLE_PIP_VERSION_CHECK 1
-
-# Install python tools (auditwheels,tox, ...)
-ADD deps/build_python.sh /builder/build_python.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
-
-# Install Rust
-ADD deps/build_rust.sh /builder/build_rust.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps-arm64/deps/build_openssl.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-OPENSSL_VERSION=1.1.1a
-OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
-
-curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf openssl-${OPENSSL_VERSION}.tar.gz
-cd openssl-${OPENSSL_VERSION} 
-./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
-
-sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
-sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
-sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
-
-make depend
-make 
-make install_sw install_ssldirs
-ldconfig -v | grep ssl 

scripts/docker-coredeps-arm64/deps/build_perl.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
-curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar -xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION}
-
-./Configure -de
-make
-make install

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.7 environment as the base environment 
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel 
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
-
-popd

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.61.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
-rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/Dockerfile

@@ -1,21 +0,0 @@
-FROM quay.io/pypa/manylinux2014_x86_64
-
-# Configure ld.so/ldconfig and pkg-config
-RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
-    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
-ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
-
-# Install a recent Perl, needed to install the openssl crate
-ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-
-ENV PIP_DISABLE_PIP_VERSION_CHECK 1
-
-# Install python tools (auditwheels,tox, ...)
-ADD deps/build_python.sh /builder/build_python.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
-
-# Install Rust
-ADD deps/build_rust.sh /builder/build_rust.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps/deps/build_openssl.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-OPENSSL_VERSION=1.1.1a
-OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
-
-curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf openssl-${OPENSSL_VERSION}.tar.gz
-cd openssl-${OPENSSL_VERSION} 
-./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
-
-sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
-sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
-sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
-
-make depend
-make 
-make install_sw install_ssldirs
-ldconfig -v | grep ssl 

scripts/docker-coredeps/deps/build_perl.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
-curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar -xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION}
-
-./Configure -de
-make
-make install

scripts/docker-coredeps/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.7 environment as the base environment
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
-
-popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.61.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
-rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-doxygen/Dockerfile

@@ -1,5 +0,0 @@
-FROM debian:stable
-
-# this is tagged as deltachat/doxygen 
-
-RUN apt-get update && apt-get install -y doxygen 

scripts/run-doxygen.sh

@@ -1,6 +1,5 @@
-#!/usr/bin/env bash
-
-set -ex
+#!/bin/sh
+set -e
 
 cd deltachat-ffi 
 PROJECT_NUMBER=$(git log -1 --format="%h (%cd)") doxygen

scripts/run-python-test.sh

@@ -22,4 +22,4 @@ export PYTHONDONTWRITEBYTECODE=1
 # run python tests (tox invokes pytest to run tests in python/tests)
 #TOX_PARALLEL_NO_SPINNER=1 tox -e lint,doc
 tox -e lint
-tox -e doc,py37
+tox -e doc,py3

scripts/run_all.sh

@@ -5,7 +5,6 @@
 set -e -x
 
 # Perform clean build of core and install.
-export TOXWORKDIR=.docker-tox
 
 # compile core lib
 
@@ -21,17 +20,8 @@ export DCC_RS_TARGET=release
 # needed by tox below.
 export PATH=$PATH:/opt/python/cp37-cp37m/bin
 export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-rm -f python3.7
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-rm -f python3.8
-ln -s /opt/python/cp38-cp38/bin/python3.8
-rm -f python3.9
-ln -s /opt/python/cp39-cp39/bin/python3.9
-rm -f python3.10
-ln -s /opt/python/cp310-cp310/bin/python3.10
-popd
 
+TOXWORKDIR=.docker-tox
 pushd python
 # prepare a clean tox run
 rm -rf tests/__pycache__
@@ -43,7 +33,10 @@ mkdir -p $TOXWORKDIR
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,auditwheels
+
+# Try to build wheels for a range of interpreters, but don't fail if they are not available.
+# E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,pypy37,pypy38,pypy39,auditwheels --skip-missing-interpreters true
 popd
 
 

spec.md

@@ -1,6 +1,6 @@
 # chat-mail specification
 
-Version: 0.33.0
+Version: 0.34.0
 Status:  In-progress 
 Format:  [Semantic Line Breaks](https://sembr.org/)
 
@@ -474,4 +474,20 @@ as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
 
+# Transitioning to a new e-mail address (AEAP)
+
+When receiving a message:
+- If the key exists, but belongs to another address
+- AND there is a `Chat-Version` header
+- AND the message is signed correctly
+- AND the From address is (also) in the encrypted (and therefore signed) headers 
+- AND the message timestamp is newer than the contact's `lastseen`
+  (to prevent changing the address back when messages arrive out of order)
+  (this condition is not that important
+  since we will have eventual consistency even without it):
+
+  Replace the contact in _all_ groups,
+  possibly deduplicate the members list,
+  and add a system message to all of these chats.
+
 Copyright © 2017-2021 Delta Chat contributors.

src/accounts.rs

@@ -1,19 +1,15 @@
 //! # Account manager module.
 
 use std::collections::BTreeMap;
-
-use async_std::channel::{self, Receiver, Sender};
-use async_std::fs;
-use async_std::path::PathBuf;
-use async_std::prelude::*;
-use async_std::sync::{Arc, RwLock};
-use uuid::Uuid;
+use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
+use tokio::fs;
+use uuid::Uuid;
 
 use crate::context::Context;
-use crate::events::{Event, EventType, Events};
+use crate::events::{Event, EventEmitter, EventType, Events};
 
 /// Account manager, that can handle multiple accounts in a single place.
 #[derive(Debug)]
@@ -21,7 +17,6 @@ pub struct Accounts {
     dir: PathBuf,
     config: Config,
     accounts: BTreeMap<u32, Context>,
-    emitter: EventEmitter,
 
     /// Event channel to emit account manager errors.
     events: Events,
@@ -30,7 +25,7 @@ pub struct Accounts {
 impl Accounts {
     /// Loads or creates an accounts folder at the given `dir`.
     pub async fn new(dir: PathBuf) -> Result<Self> {
-        if !dir.exists().await {
+        if !dir.exists() {
             Accounts::create(&dir).await?;
         }
 
@@ -51,40 +46,24 @@ impl Accounts {
     /// Opens an existing accounts structure. Will error if the folder doesn't exist,
     /// no account exists and no config exists.
     pub async fn open(dir: PathBuf) -> Result<Self> {
-        ensure!(dir.exists().await, "directory does not exist");
+        ensure!(dir.exists(), "directory does not exist");
 
         let config_file = dir.join(CONFIG_NAME);
-        ensure!(
-            config_file.exists().await,
-            "{:?} does not exist",
-            config_file
-        );
+        ensure!(config_file.exists(), "{:?} does not exist", config_file);
 
         let config = Config::from_file(config_file)
             .await
             .context("failed to load accounts config")?;
+        let events = Events::new();
         let accounts = config
-            .load_accounts()
+            .load_accounts(&events)
             .await
             .context("failed to load accounts")?;
 
-        let emitter = EventEmitter::new();
-
-        let events = Events::default();
-
-        emitter.sender.send(events.get_emitter()).await?;
-
-        for account in accounts.values() {
-            emitter.add_account(account).await.with_context(|| {
-                format!("failed to add account {} to event emitter", account.id)
-            })?;
-        }
-
         Ok(Self {
             dir,
             config,
             accounts,
-            emitter,
             events,
         })
     }
@@ -121,8 +100,12 @@ impl Accounts {
     pub async fn add_account(&mut self) -> Result<u32> {
         let account_config = self.config.new_account(&self.dir).await?;
 
-        let ctx = Context::new(account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
+        let ctx = Context::new(
+            &account_config.dbfile(),
+            account_config.id,
+            self.events.clone(),
+        )
+        .await?;
         self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -132,8 +115,12 @@ impl Accounts {
     pub async fn add_closed_account(&mut self) -> Result<u32> {
         let account_config = self.config.new_account(&self.dir).await?;
 
-        let ctx = Context::new_closed(account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
+        let ctx = Context::new_closed(
+            &account_config.dbfile(),
+            account_config.id,
+            self.events.clone(),
+        )
+        .await?;
         self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -156,7 +143,7 @@ impl Accounts {
             loop {
                 counter += 1;
 
-                if let Err(err) = fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                if let Err(err) = fs::remove_dir_all(&cfg.dir)
                     .await
                     .context("failed to remove account data")
                 {
@@ -165,7 +152,7 @@ impl Accounts {
                     }
 
                     // Wait 1 second and try again.
-                    async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
+                    tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
                 } else {
                     break;
                 }
@@ -181,16 +168,8 @@ impl Accounts {
         let blobdir = Context::derive_blobdir(&dbfile);
         let walfile = Context::derive_walfile(&dbfile);
 
-        ensure!(
-            dbfile.exists().await,
-            "no database found: {}",
-            dbfile.display()
-        );
-        ensure!(
-            blobdir.exists().await,
-            "no blobdir found: {}",
-            blobdir.display()
-        );
+        ensure!(dbfile.exists(), "no database found: {}", dbfile.display());
+        ensure!(blobdir.exists(), "no blobdir found: {}", blobdir.display());
 
         let old_id = self.config.get_selected_account().await;
 
@@ -201,7 +180,7 @@ impl Accounts {
             .await
             .context("failed to create new account")?;
 
-        let new_dbfile = account_config.dbfile().into();
+        let new_dbfile = account_config.dbfile();
         let new_blobdir = Context::derive_blobdir(&new_dbfile);
         let new_walfile = Context::derive_walfile(&new_dbfile);
 
@@ -215,7 +194,7 @@ impl Accounts {
             fs::rename(&blobdir, &new_blobdir)
                 .await
                 .context("failed to rename blobdir")?;
-            if walfile.exists().await {
+            if walfile.exists() {
                 fs::rename(&walfile, &new_walfile)
                     .await
                     .context("failed to rename walfile")?;
@@ -225,14 +204,13 @@ impl Accounts {
 
         match res {
             Ok(_) => {
-                let ctx = Context::new(new_dbfile, account_config.id).await?;
-                self.emitter.add_account(&ctx).await?;
+                let ctx = Context::new(&new_dbfile, account_config.id, self.events.clone()).await?;
                 self.accounts.insert(account_config.id, ctx);
                 Ok(account_config.id)
             }
             Err(err) => {
                 // remove temp account
-                fs::remove_dir_all(async_std::path::PathBuf::from(&account_config.dir))
+                fs::remove_dir_all(std::path::PathBuf::from(&account_config.dir))
                     .await
                     .context("failed to remove account data")?;
 
@@ -277,6 +255,9 @@ impl Accounts {
     }
 
     pub async fn stop_io(&self) {
+        // Sending an event here wakes up event loop even
+        // if there are no accounts.
+        info!(self, "Stopping IO for all accounts");
         for account in self.accounts.values() {
             account.stop_io().await;
         }
@@ -299,74 +280,9 @@ impl Accounts {
         self.events.emit(Event { id: 0, typ: event })
     }
 
-    /// Returns unified event emitter.
+    /// Returns event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
-        self.emitter.clone()
-    }
-}
-
-/// Unified event emitter for multiple accounts.
-#[derive(Debug, Clone)]
-pub struct EventEmitter {
-    /// Aggregate stream of events from all accounts.
-    stream: Arc<RwLock<futures::stream::SelectAll<crate::events::EventEmitter>>>,
-
-    /// Sender for the channel where new account emitters will be pushed.
-    sender: Sender<crate::events::EventEmitter>,
-
-    /// Receiver for the channel where new account emitters will be pushed.
-    receiver: Receiver<crate::events::EventEmitter>,
-}
-
-impl EventEmitter {
-    pub fn new() -> Self {
-        let (sender, receiver) = channel::unbounded();
-        Self {
-            stream: Arc::new(RwLock::new(futures::stream::SelectAll::new())),
-            sender,
-            receiver,
-        }
-    }
-
-    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
-    pub fn recv_sync(&mut self) -> Option<Event> {
-        async_std::task::block_on(self.recv()).unwrap_or_default()
-    }
-
-    /// Async recv of an event. Return `None` if all `Sender`s have been dropped.
-    pub async fn recv(&mut self) -> Result<Option<Event>> {
-        let mut stream = self.stream.write().await;
-        loop {
-            match futures::future::select(self.receiver.recv(), stream.next()).await {
-                futures::future::Either::Left((emitter, _)) => {
-                    stream.push(emitter?);
-                }
-                futures::future::Either::Right((ev, _)) => return Ok(ev),
-            }
-        }
-    }
-
-    /// Add event emitter of a new account to the aggregate event emitter.
-    pub async fn add_account(&self, context: &Context) -> Result<()> {
-        self.sender.send(context.get_event_emitter()).await?;
-        Ok(())
-    }
-}
-
-impl Default for EventEmitter {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl async_std::stream::Stream for EventEmitter {
-    type Item = Event;
-
-    fn poll_next(
-        mut self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Option<Self::Item>> {
-        std::pin::Pin::new(&mut self).poll_next(cx)
+        self.events.get_emitter()
     }
 }
 
@@ -392,7 +308,7 @@ struct InnerConfig {
 }
 
 impl Config {
-    pub async fn new(dir: &PathBuf) -> Result<Self> {
+    pub async fn new(dir: &Path) -> Result<Self> {
         let inner = InnerConfig {
             accounts: Vec::new(),
             selected_account: 0,
@@ -423,10 +339,10 @@ impl Config {
         Ok(Config { file, inner })
     }
 
-    pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
+    pub async fn load_accounts(&self, events: &Events) -> Result<BTreeMap<u32, Context>> {
         let mut accounts = BTreeMap::new();
         for account_config in &self.inner.accounts {
-            let ctx = Context::new(account_config.dbfile().into(), account_config.id)
+            let ctx = Context::new(&account_config.dbfile(), account_config.id, events.clone())
                 .await
                 .with_context(|| {
                     format!(
@@ -442,7 +358,7 @@ impl Config {
     }
 
     /// Create a new account in the given root directory.
-    async fn new_account(&mut self, dir: &PathBuf) -> Result<AccountConfig> {
+    async fn new_account(&mut self, dir: &Path) -> Result<AccountConfig> {
         let id = {
             let id = self.inner.next_id;
             let uuid = Uuid::new_v4();
@@ -450,7 +366,7 @@ impl Config {
 
             self.inner.accounts.push(AccountConfig {
                 id,
-                dir: target_dir.into(),
+                dir: target_dir,
                 uuid,
             });
             self.inner.next_id += 1;
@@ -531,10 +447,10 @@ impl AccountConfig {
 mod tests {
     use super::*;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_account_new_open() {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts1").into();
+        let p: PathBuf = dir.path().join("accounts1");
 
         let mut accounts1 = Accounts::new(p.clone()).await.unwrap();
         accounts1.add_account().await.unwrap();
@@ -549,10 +465,10 @@ mod tests {
         assert_eq!(accounts1.accounts.len(), accounts2.accounts.len());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_account_new_add_remove() {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone()).await.unwrap();
         assert_eq!(accounts.accounts.len(), 0);
@@ -576,10 +492,10 @@ mod tests {
         assert_eq!(accounts.accounts.len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_accounts_remove_last() -> Result<()> {
         let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone()).await?;
         assert!(accounts.get_selected_account().await.is_none());
@@ -597,17 +513,19 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_migrate_account() {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone()).await.unwrap();
         assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
-        let extern_dbfile: PathBuf = dir.path().join("other").into();
-        let ctx = Context::new(extern_dbfile.clone(), 0).await.unwrap();
+        let extern_dbfile: PathBuf = dir.path().join("other");
+        let ctx = Context::new(&extern_dbfile, 0, Events::new())
+            .await
+            .unwrap();
         ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
             .await
             .unwrap();
@@ -632,10 +550,10 @@ mod tests {
     }
 
     /// Tests that accounts are sorted by ID.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_accounts_sorted() {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone()).await.unwrap();
 
@@ -650,10 +568,10 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_accounts_ids_unique_increasing_and_persisted() -> Result<()> {
         let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
         let dummy_accounts = 10;
 
         let (id0, id1, id2) = {
@@ -732,10 +650,10 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_no_accounts_event_emitter() -> Result<()> {
         let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let accounts = Accounts::new(p.clone()).await?;
 
@@ -743,25 +661,25 @@ mod tests {
         assert_eq!(accounts.accounts.len(), 0);
 
         // Create event emitter.
-        let mut event_emitter = accounts.get_event_emitter().await;
+        let event_emitter = accounts.get_event_emitter().await;
 
         // Test that event emitter does not return `None` immediately.
         let duration = std::time::Duration::from_millis(1);
-        assert!(async_std::future::timeout(duration, event_emitter.recv())
+        assert!(tokio::time::timeout(duration, event_emitter.recv())
             .await
             .is_err());
 
         // When account manager is dropped, event emitter is exhausted.
         drop(accounts);
-        assert_eq!(event_emitter.recv().await?, None);
+        assert_eq!(event_emitter.recv().await, None);
 
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_encrypted_account() -> Result<()> {
         let dir = tempfile::tempdir().context("failed to create tempdir")?;
-        let p: PathBuf = dir.path().join("accounts").into();
+        let p: PathBuf = dir.path().join("accounts");
 
         let mut accounts = Accounts::new(p.clone())
             .await

src/blob.rs

@@ -4,15 +4,13 @@ use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
 use std::io::Cursor;
+use std::path::{Path, PathBuf};
 
-use async_std::path::{Path, PathBuf};
-use async_std::prelude::*;
-use async_std::{fs, io};
-
-use anyhow::{format_err, Context as _, Error};
+use anyhow::{format_err, Context as _, Error, Result};
 use image::{DynamicImage, ImageFormat};
 use num_traits::FromPrimitive;
-use thiserror::Error;
+use tokio::io::AsyncWriteExt;
+use tokio::{fs, io};
 
 use crate::config::Config;
 use crate::constants::{
@@ -44,31 +42,15 @@ impl<'a> BlobObject<'a> {
     /// name, followed by a random number and followed by a possible
     /// extension.  The `data` will be written into the file without
     /// race-conditions.
-    ///
-    /// # Errors
-    ///
-    /// [BlobError::CreateFailure] is used when the file could not
-    /// be created.  You can expect [BlobError.cause] to contain an
-    /// underlying error.
-    ///
-    /// [BlobError::WriteFailure] is used when the file could not
-    /// be written to.  You can expect [BlobError.cause] to contain an
-    /// underlying error.
     pub async fn create(
         context: &'a Context,
         suggested_name: &str,
         data: &[u8],
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+    ) -> Result<BlobObject<'a>> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name);
         let (name, mut file) = BlobObject::create_new_file(context, blobdir, &stem, &ext).await?;
-        file.write_all(data)
-            .await
-            .map_err(|err| BlobError::WriteFailure {
-                blobdir: blobdir.to_path_buf(),
-                blobname: name.clone(),
-                cause: err.into(),
-            })?;
+        file.write_all(data).await.context("file write failure")?;
 
         // workaround a bug in async-std
         // (the executor does not handle blocking operation in Drop correctly,
@@ -89,7 +71,7 @@ impl<'a> BlobObject<'a> {
         dir: &Path,
         stem: &str,
         ext: &str,
-    ) -> Result<(String, fs::File), BlobError> {
+    ) -> Result<(String, fs::File)> {
         const MAX_ATTEMPT: u32 = 16;
         let mut attempt = 0;
         let mut name = format!("{}{}", stem, ext);
@@ -105,12 +87,8 @@ impl<'a> BlobObject<'a> {
                 Ok(file) => return Ok((name, file)),
                 Err(err) => {
                     if attempt >= MAX_ATTEMPT {
-                        return Err(BlobError::CreateFailure {
-                            blobdir: dir.to_path_buf(),
-                            blobname: name,
-                            cause: err,
-                        });
-                    } else if attempt == 1 && !dir.exists().await {
+                        return Err(err).context("failed to create file");
+                    } else if attempt == 1 && !dir.exists() {
                         fs::create_dir_all(dir).await.ok_or_log(context);
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
@@ -126,40 +104,19 @@ impl<'a> BlobObject<'a> {
     /// but also copies an existing file into it.  This is done in a
     /// in way which avoids race-conditions when multiple files are
     /// concurrently created.
-    ///
-    /// # Errors
-    ///
-    /// In addition to the errors in [BlobObject::create] the
-    /// [BlobError::CopyFailure] is used when the data can not be
-    /// copied.
-    pub async fn create_and_copy(
-        context: &'a Context,
-        src: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+    pub async fn create_and_copy(context: &'a Context, src: &Path) -> Result<BlobObject<'a>> {
         let mut src_file = fs::File::open(src)
             .await
-            .map_err(|err| BlobError::CopyFailure {
-                blobdir: context.get_blobdir().to_path_buf(),
-                blobname: String::from(""),
-                src: src.to_path_buf(),
-                cause: err,
-            })?;
+            .with_context(|| format!("failed to open file {}", src.display()))?;
         let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
         let (name, mut dst_file) =
             BlobObject::create_new_file(context, context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
         if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
-            {
-                // Attempt to remove the failed file, swallow errors resulting from that.
-                let path = context.get_blobdir().join(&name_for_err);
-                fs::remove_file(path).await.ok();
-            }
-            return Err(BlobError::CopyFailure {
-                blobdir: context.get_blobdir().to_path_buf(),
-                blobname: name_for_err,
-                src: src.to_path_buf(),
-                cause: err,
-            });
+            // Attempt to remove the failed file, swallow errors resulting from that.
+            let path = context.get_blobdir().join(&name_for_err);
+            fs::remove_file(path).await.ok();
+            return Err(err).context("failed to copy file");
         }
 
         // workaround, see create() for details
@@ -184,16 +141,7 @@ impl<'a> BlobObject<'a> {
     ///
     /// Paths into the blob directory may be either defined by an absolute path
     /// or by the relative prefix `$BLOBDIR`.
-    ///
-    /// # Errors
-    ///
-    /// This merely delegates to the [BlobObject::create_and_copy] and
-    /// the [BlobObject::from_path] methods.  See those for possible
-    /// errors.
-    pub async fn new_from_path(
-        context: &'a Context,
-        src: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+    pub async fn new_from_path(context: &'a Context, src: &Path) -> Result<BlobObject<'a>> {
         if src.starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
         } else if src.starts_with("$BLOBDIR/") {
@@ -209,32 +157,14 @@ impl<'a> BlobObject<'a> {
     /// must use a valid blob name.  That is after sanitisation the
     /// name must still be the same, that means it must be valid UTF-8
     /// and not have any special characters in it.
-    ///
-    /// # Errors
-    ///
-    /// [BlobError::WrongBlobdir] is used if the path is not in
-    /// the blob directory.
-    ///
-    /// [BlobError::WrongName] is used if the file name does not
-    /// remain identical after sanitisation.
-    pub fn from_path(
-        context: &'a Context,
-        path: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let rel_path =
-            path.strip_prefix(context.get_blobdir())
-                .map_err(|_| BlobError::WrongBlobdir {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    src: path.to_path_buf(),
-                })?;
+    pub fn from_path(context: &'a Context, path: &Path) -> Result<BlobObject<'a>> {
+        let rel_path = path
+            .strip_prefix(context.get_blobdir())
+            .context("wrong blobdir")?;
         if !BlobObject::is_acceptible_blob_name(rel_path) {
-            return Err(BlobError::WrongName {
-                blobname: path.to_path_buf(),
-            });
+            return Err(format_err!("wrong name"));
         }
-        let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
-            blobname: path.to_path_buf(),
-        })?;
+        let name = rel_path.to_str().context("wrong name")?;
         BlobObject::from_name(context, name.to_string())
     }
 
@@ -244,24 +174,13 @@ impl<'a> BlobObject<'a> {
     /// prefixed, as returned by [BlobObject::as_name].  This is how
     /// you want to create a [BlobObject] for a filename read from the
     /// database.
-    ///
-    /// # Errors
-    ///
-    /// [BlobError::WrongName] is used if the name is not a valid
-    /// blobname, i.e. if [BlobObject::sanitise_name] does modify the
-    /// provided name.
-    pub fn from_name(
-        context: &'a Context,
-        name: String,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+    pub fn from_name(context: &'a Context, name: String) -> Result<BlobObject<'a>> {
         let name: String = match name.starts_with("$BLOBDIR/") {
             true => name.splitn(2, '/').last().unwrap().to_string(),
             false => name,
         };
         if !BlobObject::is_acceptible_blob_name(&name) {
-            return Err(BlobError::WrongName {
-                blobname: PathBuf::from(name),
-            });
+            return Err(format_err!("not an acceptable blob name: {}", &name));
         }
         Ok(BlobObject {
             blobdir: context.get_blobdir(),
@@ -394,7 +313,7 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<()> {
         let blob_abs = self.to_abs_path();
 
         let img_wh =
@@ -416,7 +335,7 @@ impl<'a> BlobObject<'a> {
         Ok(())
     }
 
-    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_image_size(&self, context: &Context) -> Result<()> {
         let blob_abs = self.to_abs_path();
         if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
             != Some((Viewtype::Image, "image/jpeg"))
@@ -439,8 +358,7 @@ impl<'a> BlobObject<'a> {
         {
             return Err(format_err!(
                 "Internal error: recode_to_size(..., None) shouldn't change the name of the image"
-            )
-            .into());
+            ));
         }
         Ok(())
     }
@@ -451,118 +369,82 @@ impl<'a> BlobObject<'a> {
         mut blob_abs: PathBuf,
         mut img_wh: u32,
         max_bytes: Option<usize>,
-    ) -> Result<Option<String>, BlobError> {
-        let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
-        })?;
-        let orientation = self.get_exif_orientation(context);
-        let mut encoded = Vec::new();
-        let mut changed_name = None;
+    ) -> Result<Option<String>> {
+        tokio::task::block_in_place(move || {
+            let mut img = image::open(&blob_abs).context("image recode failure")?;
+            let orientation = self.get_exif_orientation(context);
+            let mut encoded = Vec::new();
+            let mut changed_name = None;
 
-        fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
-            encoded.clear();
-            let mut buf = Cursor::new(encoded);
-            img.write_to(&mut buf, image::ImageFormat::Jpeg)?;
-            Ok(())
-        }
-        fn encoded_img_exceeds_bytes(
-            context: &Context,
-            img: &DynamicImage,
-            max_bytes: Option<usize>,
-            encoded: &mut Vec<u8>,
-        ) -> anyhow::Result<bool> {
-            if let Some(max_bytes) = max_bytes {
-                encode_img(img, encoded)?;
-                if encoded.len() > max_bytes {
-                    info!(
-                        context,
-                        "image size {}B ({}x{}px) exceeds {}B, need to scale down",
-                        encoded.len(),
-                        img.width(),
-                        img.height(),
-                        max_bytes,
-                    );
-                    return Ok(true);
-                }
-            }
-            Ok(false)
-        }
-        let exceeds_width = img.width() > img_wh || img.height() > img_wh;
+            let exceeds_width = img.width() > img_wh || img.height() > img_wh;
 
-        let do_scale =
-            exceeds_width || encoded_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
-        let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
+            let do_scale =
+                exceeds_width || encoded_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
+            let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
 
-        if do_scale || do_rotate {
-            if do_rotate {
-                img = match orientation {
-                    Ok(90) => img.rotate90(),
-                    Ok(180) => img.rotate180(),
-                    Ok(270) => img.rotate270(),
-                    _ => img,
-                }
-            }
-
-            if do_scale {
-                if !exceeds_width {
-                    // The image is already smaller than img_wh, but exceeds max_bytes
-                    // We can directly start with trying to scale down to 2/3 of its current width
-                    img_wh = max(img.width(), img.height()) * 2 / 3
-                }
-
-                loop {
-                    let new_img = img.thumbnail(img_wh, img_wh);
-
-                    if encoded_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
-                        if img_wh < 20 {
-                            return Err(format_err!(
-                                "Failed to scale image to below {}B",
-                                max_bytes.unwrap_or_default()
-                            )
-                            .into());
-                        }
-
-                        img_wh = img_wh * 2 / 3;
-                    } else {
-                        if encoded.is_empty() {
-                            encode_img(&new_img, &mut encoded)?;
-                        }
-
-                        info!(
-                            context,
-                            "Final scaled-down image size: {}B ({}px)",
-                            encoded.len(),
-                            img_wh
-                        );
-                        break;
+            if do_scale || do_rotate {
+                if do_rotate {
+                    img = match orientation {
+                        Ok(90) => img.rotate90(),
+                        Ok(180) => img.rotate180(),
+                        Ok(270) => img.rotate270(),
+                        _ => img,
                     }
                 }
+
+                if do_scale {
+                    if !exceeds_width {
+                        // The image is already smaller than img_wh, but exceeds max_bytes
+                        // We can directly start with trying to scale down to 2/3 of its current width
+                        img_wh = max(img.width(), img.height()) * 2 / 3
+                    }
+
+                    loop {
+                        let new_img = img.thumbnail(img_wh, img_wh);
+
+                        if encoded_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
+                            if img_wh < 20 {
+                                return Err(format_err!(
+                                    "Failed to scale image to below {}B",
+                                    max_bytes.unwrap_or_default()
+                                ));
+                            }
+
+                            img_wh = img_wh * 2 / 3;
+                        } else {
+                            if encoded.is_empty() {
+                                encode_img(&new_img, &mut encoded)?;
+                            }
+
+                            info!(
+                                context,
+                                "Final scaled-down image size: {}B ({}px)",
+                                encoded.len(),
+                                img_wh
+                            );
+                            break;
+                        }
+                    }
+                }
+
+                // The file format is JPEG now, we may have to change the file extension
+                if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
+                    blob_abs = blob_abs.with_extension("jpg");
+                    let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
+                    let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
+                    changed_name = Some(format!("$BLOBDIR/{}", file_name));
+                }
+
+                if encoded.is_empty() {
+                    encode_img(&img, &mut encoded)?;
+                }
+
+                std::fs::write(&blob_abs, &encoded)
+                    .context("failed to write recoded blob to file")?;
             }
 
-            // The file format is JPEG now, we may have to change the file extension
-            if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
-                blob_abs = blob_abs.with_extension("jpg");
-                let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
-                let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
-                changed_name = Some(format!("$BLOBDIR/{}", file_name));
-            }
-
-            if encoded.is_empty() {
-                encode_img(&img, &mut encoded)?;
-            }
-
-            fs::write(&blob_abs, &encoded)
-                .await
-                .map_err(|err| BlobError::WriteFailure {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-                    cause: err.into(),
-                })?;
-        }
-
-        Ok(changed_name)
+            Ok(changed_name)
+        })
     }
 
     pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
@@ -590,44 +472,33 @@ impl<'a> fmt::Display for BlobObject<'a> {
     }
 }
 
-/// Errors for the [BlobObject].
-#[derive(Debug, Error)]
-pub enum BlobError {
-    #[error("Failed to create blob {blobname} in {}", .blobdir.display())]
-    CreateFailure {
-        blobdir: PathBuf,
-        blobname: String,
-        #[source]
-        cause: std::io::Error,
-    },
-    #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())]
-    WriteFailure {
-        blobdir: PathBuf,
-        blobname: String,
-        #[source]
-        cause: anyhow::Error,
-    },
-    #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
-    CopyFailure {
-        blobdir: PathBuf,
-        blobname: String,
-        src: PathBuf,
-        #[source]
-        cause: std::io::Error,
-    },
-    #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())]
-    RecodeFailure {
-        blobdir: PathBuf,
-        blobname: String,
-        #[source]
-        cause: image::ImageError,
-    },
-    #[error("File path {} is not in {}", .src.display(), .blobdir.display())]
-    WrongBlobdir { blobdir: PathBuf, src: PathBuf },
-    #[error("Blob has a badname {}", .blobname.display())]
-    WrongName { blobname: PathBuf },
-    #[error("{0}")]
-    Other(#[from] anyhow::Error),
+fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
+    encoded.clear();
+    let mut buf = Cursor::new(encoded);
+    img.write_to(&mut buf, image::ImageFormat::Jpeg)?;
+    Ok(())
+}
+fn encoded_img_exceeds_bytes(
+    context: &Context,
+    img: &DynamicImage,
+    max_bytes: Option<usize>,
+    encoded: &mut Vec<u8>,
+) -> anyhow::Result<bool> {
+    if let Some(max_bytes) = max_bytes {
+        encode_img(img, encoded)?;
+        if encoded.len() > max_bytes {
+            info!(
+                context,
+                "image size {}B ({}x{}px) exceeds {}B, need to scale down",
+                encoded.len(),
+                img.width(),
+                img.height(),
+                max_bytes,
+            );
+            return Ok(true);
+        }
+    }
+    Ok(false)
 }
 
 #[cfg(test)]
@@ -643,7 +514,16 @@ mod tests {
 
     use super::*;
 
-    #[async_std::test]
+    fn check_image_size(path: impl AsRef<Path>, width: u32, height: u32) -> image::DynamicImage {
+        tokio::task::block_in_place(move || {
+            let img = image::open(path).expect("failed to open image");
+            assert_eq!(img.width(), width, "invalid width");
+            assert_eq!(img.height(), height, "invalid height");
+            img
+        })
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo", b"hello").await.unwrap();
@@ -654,28 +534,28 @@ mod tests {
         assert_eq!(blob.to_abs_path(), t.get_blobdir().join("foo"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_lowercase_ext() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo.TXT", b"hello").await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_as_file_name() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_as_rel_path() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_suffix() {
         let t = TestContext::new().await;
         let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
@@ -684,16 +564,16 @@ mod tests {
         assert_eq!(blob.suffix(), None);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_dup() {
         let t = TestContext::new().await;
         BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         let foo_path = t.get_blobdir().join("foo.txt");
-        assert!(foo_path.exists().await);
+        assert!(foo_path.exists());
         BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
         let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
-        while let Some(dirent) = dir.next().await {
-            let fname = dirent.unwrap().file_name();
+        while let Ok(Some(dirent)) = dir.next_entry().await {
+            let fname = dirent.file_name();
             if fname == foo_path.file_name().unwrap() {
                 assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
             } else {
@@ -704,20 +584,20 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_double_ext_preserved() {
         let t = TestContext::new().await;
         BlobObject::create(&t, "foo.tar.gz", b"hello")
             .await
             .unwrap();
         let foo_path = t.get_blobdir().join("foo.tar.gz");
-        assert!(foo_path.exists().await);
+        assert!(foo_path.exists());
         BlobObject::create(&t, "foo.tar.gz", b"world")
             .await
             .unwrap();
         let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
-        while let Some(dirent) = dir.next().await {
-            let fname = dirent.unwrap().file_name();
+        while let Ok(Some(dirent)) = dir.next_entry().await {
+            let fname = dirent.file_name();
             if fname == foo_path.file_name().unwrap() {
                 assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
             } else {
@@ -729,7 +609,7 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_long_names() {
         let t = TestContext::new().await;
         let s = "1".repeat(150);
@@ -738,7 +618,7 @@ mod tests {
         assert!(blobname.len() < 128);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_and_copy() {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
@@ -753,10 +633,10 @@ mod tests {
             .await
             .is_err());
         let whoops = t.get_blobdir().join("whoops");
-        assert!(!whoops.exists().await);
+        assert!(!whoops.exists());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_from_path() {
         let t = TestContext::new().await;
 
@@ -776,7 +656,7 @@ mod tests {
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
     }
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_from_name_long() {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
@@ -839,34 +719,24 @@ mod tests {
         assert!(!stem.contains('?'));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_selfavatar_outside_blobdir() {
         let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.jpg");
         let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
+        fs::write(&avatar_src, avatar_bytes).await.unwrap();
         let avatar_blob = t.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists().await);
+        assert!(!avatar_blob.exists());
         t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
+        assert!(avatar_blob.exists());
+        assert!(fs::metadata(&avatar_blob).await.unwrap().len() < avatar_bytes.len() as u64);
         let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
 
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(&avatar_blob).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+        check_image_size(avatar_src, 1000, 1000);
+        check_image_size(&avatar_blob, BALANCED_AVATAR_SIZE, BALANCED_AVATAR_SIZE);
 
         async fn file_size(path_buf: &PathBuf) -> u64 {
             let file = File::open(path_buf).await.unwrap();
@@ -880,25 +750,22 @@ mod tests {
             .unwrap();
         assert!(file_size(&avatar_blob).await <= 3000);
         assert!(file_size(&avatar_blob).await > 2000);
-        let img = image::open(&avatar_blob).unwrap();
-        assert!(img.width() > 130);
-        assert_eq!(img.width(), img.height());
+        tokio::task::block_in_place(move || {
+            let img = image::open(&avatar_blob).unwrap();
+            assert!(img.width() > 130);
+            assert_eq!(img.width(), img.height());
+        });
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_selfavatar_in_blobdir() {
         let t = TestContext::new().await;
         let avatar_src = t.get_blobdir().join("avatar.png");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(test_utils::AVATAR_900x900_BYTES)
+        fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES)
             .await
             .unwrap();
 
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
+        check_image_size(&avatar_src, 900, 900);
 
         t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
@@ -909,37 +776,30 @@ mod tests {
             avatar_src.with_extension("jpg").to_str().unwrap()
         );
 
-        let img = image::open(avatar_cfg).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+        check_image_size(avatar_cfg, BALANCED_AVATAR_SIZE, BALANCED_AVATAR_SIZE);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_selfavatar_copy_without_recode() {
         let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
+        fs::write(&avatar_src, avatar_bytes).await.unwrap();
         let avatar_blob = t.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists().await);
+        assert!(!avatar_blob.exists());
         t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
-        assert!(avatar_blob.exists().await);
+        assert!(avatar_blob.exists());
         assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
+            fs::metadata(&avatar_blob).await.unwrap().len(),
             avatar_bytes.len() as u64
         );
         let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_1() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
@@ -959,7 +819,7 @@ mod tests {
         .unwrap();
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
@@ -985,7 +845,7 @@ mod tests {
         // Do this in parallel to speed up the test a bit
         // (it still takes very long though)
         let bytes2 = bytes.clone();
-        let join_handle = async_std::task::spawn(async move {
+        let join_handle = tokio::task::spawn(async move {
             let img_rotated = send_image_check_mediaquality(
                 Some("0"),
                 &bytes2,
@@ -1013,10 +873,10 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
-        join_handle.await;
+        join_handle.await.unwrap();
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_3() {
         let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
@@ -1064,10 +924,10 @@ mod tests {
             .await?;
         let file = alice.get_blobdir().join("file.jpg");
 
-        File::create(&file).await?.write_all(bytes).await?;
-        let img = image::open(&file)?;
-        assert_eq!(img.width(), original_width);
-        assert_eq!(img.height(), original_height);
+        fs::write(&file, &bytes)
+            .await
+            .context("failed to write file")?;
+        check_image_size(&file, original_width, original_height);
 
         let blob = BlobObject::new_from_path(&alice, &file).await?;
         assert_eq!(blob.get_exif_orientation(&alice).unwrap_or(0), orientation);
@@ -1079,9 +939,11 @@ mod tests {
         let alice_msg = alice.get_last_msg().await;
         assert_eq!(alice_msg.get_width() as u32, compressed_width);
         assert_eq!(alice_msg.get_height() as u32, compressed_height);
-        let img = image::open(alice_msg.get_file(&alice).unwrap())?;
-        assert_eq!(img.width() as u32, compressed_width);
-        assert_eq!(img.height() as u32, compressed_height);
+        check_image_size(
+            alice_msg.get_file(&alice).unwrap(),
+            compressed_width,
+            compressed_height,
+        );
 
         let bob_msg = bob.recv_msg(&sent).await;
         assert_eq!(bob_msg.get_width() as u32, compressed_width);
@@ -1091,19 +953,17 @@ mod tests {
         let blob = BlobObject::new_from_path(&bob, &file).await?;
         assert_eq!(blob.get_exif_orientation(&bob).unwrap_or(0), 0);
 
-        let img = image::open(file)?;
-        assert_eq!(img.width() as u32, compressed_width);
-        assert_eq!(img.height() as u32, compressed_height);
+        let img = check_image_size(file, compressed_width, compressed_height);
         Ok(img)
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_increation_in_blobdir() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
 
         let file = t.get_blobdir().join("anyfile.dat");
-        File::create(&file).await?.write_all("bla".as_ref()).await?;
+        fs::write(&file, b"bla").await?;
         let mut msg = Message::new(Viewtype::File);
         msg.set_file(file.to_str().unwrap(), None);
         let prepared_id = chat::prepare_msg(&t, chat_id, &mut msg).await?;
@@ -1116,14 +976,14 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_increation_not_blobdir() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
         assert_ne!(t.get_blobdir().to_str(), t.dir.path().to_str());
 
         let file = t.dir.path().join("anyfile.dat");
-        File::create(&file).await?.write_all("bla".as_ref()).await?;
+        fs::write(&file, b"bla").await?;
         let mut msg = Message::new(Viewtype::File);
         msg.set_file(file.to_str().unwrap(), None);
         assert!(chat::prepare_msg(&t, chat_id, &mut msg).await.is_err());

src/chatlist.rs

@@ -240,7 +240,7 @@ impl Chatlist {
             ids
         };
 
-        if add_archived_link_item && dc_get_archived_cnt(context).await? > 0 {
+        if add_archived_link_item && get_archived_cnt(context).await? > 0 {
             if ids.is_empty() && flag_add_alldone_hint {
                 ids.push((DC_CHAT_ID_ALLDONE_HINT, None));
             }
@@ -355,7 +355,7 @@ impl Chatlist {
 }
 
 /// Returns the number of archived chats
-pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
+pub async fn get_archived_cnt(context: &Context) -> Result<usize> {
     let count = context
         .sql
         .count(
@@ -371,12 +371,12 @@ mod tests {
     use super::*;
 
     use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus};
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::message::Viewtype;
+    use crate::receive_imf::receive_imf;
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_try_load() {
         let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
@@ -432,7 +432,7 @@ mod tests {
         assert_eq!(chats.len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sort_self_talk_up_on_forward() {
         let t = TestContext::new().await;
         t.update_device_chats().await.unwrap();
@@ -457,7 +457,7 @@ mod tests {
             .is_self_talk());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_search_special_chat_names() {
         let t = TestContext::new().await;
         t.update_device_chats().await.unwrap();
@@ -488,12 +488,12 @@ mod tests {
         assert_eq!(chats.len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_search_single_chat() -> anyhow::Result<()> {
         let t = TestContext::new_alice().await;
 
         // receive a one-to-one-message
-        dc_receive_imf(
+        receive_imf(
             &t,
             b"From: Bob Authname <bob@example.org>\n\
                  To: alice@example.org\n\
@@ -548,12 +548,12 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_search_single_chat_without_authname() -> anyhow::Result<()> {
         let t = TestContext::new_alice().await;
 
         // receive a one-to-one-message without authname set
-        dc_receive_imf(
+        receive_imf(
             &t,
             b"From: bob@example.org\n\
                  To: alice@example.org\n\
@@ -610,7 +610,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_summary_unwrap() {
         let t = TestContext::new().await;
         let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")

src/config.rs

@@ -8,10 +8,10 @@ use crate::blob::BlobObject;
 use crate::constants::DC_VERSION_STR;
 use crate::contact::addr_cmp;
 use crate::context::Context;
-use crate::dc_tools::{dc_get_abs_path, improve_single_line_input, EmailAddress};
 use crate::events::EventType;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
+use crate::tools::{get_abs_path, improve_single_line_input, EmailAddress};
 
 /// The available configuration keys.
 #[derive(
@@ -89,6 +89,11 @@ pub enum Config {
     #[strum(props(default = "1"))]
     FetchExistingMsgs,
 
+    /// If set to "1", then existing messages are considered to be already fetched.
+    /// This flag is reset after successful configuration.
+    #[strum(props(default = "1"))]
+    FetchedExistingMsgs,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -191,7 +196,7 @@ impl Context {
         let value = match key {
             Config::Selfavatar => {
                 let rel_path = self.sql.get_raw_config(key).await?;
-                rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
+                rel_path.map(|p| get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
@@ -432,9 +437,8 @@ mod tests {
     use std::string::ToString;
 
     use crate::constants;
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils::TestContext;
-    use crate::test_utils::TestContextManager;
+
     use num_traits::FromPrimitive;
 
     #[test]
@@ -449,7 +453,7 @@ mod tests {
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;
         let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
@@ -466,7 +470,7 @@ mod tests {
         assert_eq!(media_quality, constants::MediaQuality::Worse);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ui_config() -> Result<()> {
         let t = TestContext::new().await;
 
@@ -488,7 +492,7 @@ mod tests {
     }
 
     /// Regression test for https://github.com/deltachat/deltachat-core-rust/issues/3012
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_config_bool() -> Result<()> {
         let t = TestContext::new().await;
 
@@ -500,7 +504,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_self_addrs() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -552,68 +556,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_change_primary_self_addr() -> Result<()> {
-        let mut tcm = TestContextManager::new().await;
-        let alice = tcm.alice().await;
-        let bob = tcm.bob().await;
-
-        // Alice sends a message to Bob
-        let alice_bob_chat = alice.create_chat(&bob).await;
-        let sent = alice.send_text(alice_bob_chat.id, "Hi").await;
-        let bob_msg = bob.recv_msg(&sent).await;
-        bob_msg.chat_id.accept(&bob).await?;
-        assert_eq!(bob_msg.text.unwrap(), "Hi");
-
-        // Alice changes her self address and reconfigures
-        // (ensure_secret_key_exists() is called during configure)
-        alice
-            .set_primary_self_addr("alice@someotherdomain.xyz")
-            .await?;
-        crate::e2ee::ensure_secret_key_exists(&alice).await?;
-
-        assert_eq!(
-            alice.get_primary_self_addr().await?,
-            "alice@someotherdomain.xyz"
-        );
-
-        // Bob sends a message to Alice, encrypting to her previous key
-        let sent = bob.send_text(bob_msg.chat_id, "hi back").await;
-
-        // Alice set up message forwarding so that she still receives
-        // the message with her new address
-        let alice_msg = alice.recv_msg(&sent).await;
-        assert_eq!(alice_msg.text, Some("hi back".to_string()));
-        assert_eq!(alice_msg.get_showpadlock(), true);
-        assert_eq!(alice_msg.chat_id, alice_bob_chat.id);
-
-        // Even if Bob sends a message to Alice without In-Reply-To,
-        // it's still assigned to the 1:1 chat with Bob and not to
-        // a group (without secondary addresses, an ad-hoc group
-        // would be created)
-        dc_receive_imf(
-            &alice,
-            b"From: bob@example.net
-To: alice@example.org
-Chat-Version: 1.0
-Message-ID: <456@example.com>
-
-Message w/out In-Reply-To
-",
-            false,
-        )
-        .await?;
-
-        let alice_msg = alice.get_last_msg().await;
-
-        assert_eq!(
-            alice_msg.text,
-            Some("Message w/out In-Reply-To".to_string())
-        );
-        assert_eq!(alice_msg.get_showpadlock(), false);
-        assert_eq!(alice_msg.chat_id, alice_bob_chat.id);
-
-        Ok(())
-    }
 }

src/configure.rs

@@ -6,24 +6,23 @@ mod read_url;
 mod server_params;
 
 use anyhow::{bail, ensure, Context as _, Result};
-use async_std::prelude::*;
-use async_std::task;
-use job::Action;
+use futures::FutureExt;
+use futures_lite::FutureExt as _;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
+use tokio::task;
 
 use crate::config::Config;
-use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2};
 use crate::context::Context;
-use crate::dc_tools::{time, EmailAddress};
 use crate::imap::Imap;
 use crate::job;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
 use crate::message::{Message, Viewtype};
-use crate::oauth2::dc_get_oauth2_addr;
-use crate::param::Params;
+use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
+use crate::scheduler::InterruptInfo;
 use crate::smtp::Smtp;
 use crate::stock_str;
+use crate::tools::{time, EmailAddress};
 use crate::{chat, e2ee, provider};
 
 use auto_mozilla::moz_autoconfigure;
@@ -57,8 +56,6 @@ impl Context {
 
     /// Configures this account with the currently set parameters.
     pub async fn configure(&self) -> Result<()> {
-        use futures::future::FutureExt;
-
         ensure!(
             self.scheduler.read().await.is_none(),
             "cannot configure, already running"
@@ -79,6 +76,24 @@ impl Context {
 
         self.free_ongoing().await;
 
+        if let Err(err) = res.as_ref() {
+            progress!(
+                self,
+                0,
+                Some(
+                    stock_str::configuration_failed(
+                        self,
+                        // We are using Anyhow's .context() and to show the
+                        // inner error, too, we need the {:#}:
+                        format!("{:#}", err),
+                    )
+                    .await
+                )
+            );
+        } else {
+            progress!(self, 1000);
+        }
+
         res
     }
 
@@ -116,58 +131,16 @@ impl Context {
             }
         }
 
-        match success {
-            Ok(_) => {
-                self.set_config(Config::NotifyAboutWrongPw, Some("1"))
-                    .await?;
-                progress!(self, 1000);
-                Ok(())
-            }
-            Err(err) => {
-                progress!(
-                    self,
-                    0,
-                    Some(
-                        stock_str::configuration_failed(
-                            self,
-                            // We are using Anyhow's .context() and to show the
-                            // inner error, too, we need the {:#}:
-                            format!("{:#}", err),
-                        )
-                        .await
-                    )
-                );
-                Err(err)
-            }
-        }
+        success?;
+        self.set_config(Config::NotifyAboutWrongPw, Some("1"))
+            .await?;
+        Ok(())
     }
 }
 
 async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 1);
 
-    // Check basic settings.
-    ensure!(!param.addr.is_empty(), "Please enter an email address.");
-
-    // Only check for IMAP password, SMTP password is an "advanced" setting.
-    ensure!(!param.imap.password.is_empty(), "Please enter a password.");
-    if param.smtp.password.is_empty() {
-        param.smtp.password = param.imap.password.clone()
-    }
-
-    // Normalize authentication flags.
-    let oauth2 = match param.server_flags & DC_LP_AUTH_FLAGS as i32 {
-        DC_LP_AUTH_OAUTH2 => true,
-        DC_LP_AUTH_NORMAL => false,
-        _ => false,
-    };
-    param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
-    param.server_flags |= if oauth2 {
-        DC_LP_AUTH_OAUTH2 as i32
-    } else {
-        DC_LP_AUTH_NORMAL as i32
-    };
-
     let socks5_config = param.socks5_config.clone();
     let socks5_enabled = socks5_config.is_some();
 
@@ -177,12 +150,13 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     // Step 1: Load the parameters and check email-address and password
 
     // Do oauth2 only if socks5 is disabled. As soon as we have a http library that can do
-    // socks5 requests, this can work with socks5 too
-    if oauth2 && !socks5_enabled {
+    // socks5 requests, this can work with socks5 too.  OAuth is always set either for both
+    // IMAP and SMTP or not at all.
+    if param.imap.oauth2 && !socks5_enabled {
         // the used oauth2 addr may differ, check this.
-        // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
+        // if get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
-        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.imap.password)
+        if let Some(oauth2_addr) = get_oauth2_addr(ctx, &param.addr, &param.imap.password)
             .await?
             .and_then(|e| e.parse().ok())
         {
@@ -359,7 +333,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                 &smtp_param,
                 &socks5_config,
                 &smtp_addr,
-                oauth2,
                 provider_strict_tls,
                 &mut smtp,
             )
@@ -407,7 +380,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             &param.imap,
             &param.socks5_config,
             &param.addr,
-            oauth2,
             provider_strict_tls,
         )
         .await
@@ -431,7 +403,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 850);
 
     // Wait for SMTP configuration
-    match smtp_config_task.await {
+    match smtp_config_task.await.unwrap() {
         Ok(smtp_param) => {
             param.smtp = smtp_param;
         }
@@ -444,6 +416,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     let create_mvbox = ctx.should_watch_mvbox().await?;
 
+    // Send client ID as soon as possible before doing anything else.
+    imap.determine_capabilities(ctx).await?;
+
     imap.configure_folders(ctx, create_mvbox).await?;
 
     imap.select_with_uidvalidity(ctx, "INBOX")
@@ -469,14 +444,12 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     e2ee::ensure_secret_key_exists(ctx).await?;
     info!(ctx, "key generation completed");
 
-    job::add(
-        ctx,
-        job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
-    )
-    .await?;
+    ctx.set_config_bool(Config::FetchedExistingMsgs, false)
+        .await?;
+    ctx.interrupt_inbox(InterruptInfo::new(false)).await;
 
     progress!(ctx, 940);
-    update_device_chats_handle.await?;
+    update_device_chats_handle.await??;
 
     ctx.sql.set_raw_config_bool("configured", true).await?;
 
@@ -565,26 +538,22 @@ async fn try_imap_one_param(
     param: &ServerLoginParam,
     socks5_config: &Option<Socks5Config>,
     addr: &str,
-    oauth2: bool,
     provider_strict_tls: bool,
 ) -> Result<Imap, ConfigurationError> {
     let inf = format!(
         "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
-        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
+        param.user,
+        param.server,
+        param.port,
+        param.security,
+        param.certificate_checks,
+        param.oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    let (_s, r) = async_std::channel::bounded(1);
+    let (_s, r) = async_channel::bounded(1);
 
-    let mut imap = match Imap::new(
-        param,
-        socks5_config.clone(),
-        addr,
-        oauth2,
-        provider_strict_tls,
-        r,
-    )
-    .await
+    let mut imap = match Imap::new(param, socks5_config.clone(), addr, provider_strict_tls, r).await
     {
         Err(err) => {
             info!(context, "failure: {}", err);
@@ -616,7 +585,6 @@ async fn try_smtp_one_param(
     param: &ServerLoginParam,
     socks5_config: &Option<Socks5Config>,
     addr: &str,
-    oauth2: bool,
     provider_strict_tls: bool,
     smtp: &mut Smtp,
 ) -> Result<(), ConfigurationError> {
@@ -627,7 +595,7 @@ async fn try_smtp_one_param(
         param.port,
         param.security,
         param.certificate_checks,
-        oauth2,
+        param.oauth2,
         if let Some(socks5_config) = socks5_config {
             socks5_config.to_string()
         } else {
@@ -637,14 +605,7 @@ async fn try_smtp_one_param(
     info!(context, "Trying: {}", inf);
 
     if let Err(err) = smtp
-        .connect(
-            context,
-            param,
-            socks5_config,
-            addr,
-            oauth2,
-            provider_strict_tls,
-        )
+        .connect(context, param, socks5_config, addr, provider_strict_tls)
         .await
     {
         info!(context, "failure: {}", err);
@@ -675,10 +636,16 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
         return "no error".to_string();
     };
 
-    if errors
-        .iter()
-        .all(|e| e.msg.to_lowercase().contains("could not resolve"))
-    {
+    if errors.iter().all(|e| {
+        e.msg.to_lowercase().contains("could not resolve")
+            || e.msg
+                .to_lowercase()
+                .contains("temporary failure in name resolution")
+            || e.msg.to_lowercase().contains("name or service not known")
+            || e.msg
+                .to_lowercase()
+                .contains("failed to lookup address information")
+    }) {
         return stock_str::error_no_network(context).await;
     }
 
@@ -719,7 +686,7 @@ mod tests {
     use crate::config::Config;
     use crate::test_utils::TestContext;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_no_panic_on_bad_credentials() {
         let t = TestContext::new().await;
         t.set_config(Config::Addr, Some("probably@unexistant.addr"))

src/configure/read_url.rs

@@ -1,7 +1,6 @@
-use crate::context::Context;
+use anyhow::{anyhow, format_err};
 
-use anyhow::format_err;
-use anyhow::Context as _;
+use crate::context::Context;
 
 pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
     match read_url_inner(context, url).await {
@@ -16,24 +15,27 @@ pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
     }
 }
 
-pub async fn read_url_inner(context: &Context, mut url: &str) -> anyhow::Result<String> {
-    let mut _temp; // For the borrow checker
+pub async fn read_url_inner(context: &Context, url: &str) -> anyhow::Result<String> {
+    let client = reqwest::Client::new();
+    let mut url = url.to_string();
 
     // Follow up to 10 http-redirects
     for _i in 0..10 {
-        let mut response = surf::get(url).send().await.map_err(|e| e.into_inner())?;
+        let response = client.get(&url).send().await?;
         if response.status().is_redirection() {
-            _temp = response
-                .header("location")
-                .context("Redirection doesn't have a target location")?
+            let headers = response.headers();
+            let header = headers
+                .get_all("location")
+                .iter()
                 .last()
-                .to_string();
-            info!(context, "Following redirect to {}", _temp);
-            url = &_temp;
+                .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))?
+                .to_str()?;
+            info!(context, "Following redirect to {}", header);
+            url = header.to_string();
             continue;
         }
 
-        return response.body_string().await.map_err(|e| e.into_inner());
+        return response.text().await.map_err(Into::into);
     }
 
     Err(format_err!("Followed 10 redirections"))

src/contact.rs

@@ -2,9 +2,9 @@
 
 use std::convert::{TryFrom, TryInto};
 use std::fmt;
+use std::path::PathBuf;
 
 use anyhow::{bail, ensure, Context as _, Result};
-use async_std::path::PathBuf;
 use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use regex::Regex;
@@ -16,7 +16,6 @@ use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
 use crate::context::Context;
-use crate::dc_tools::{dc_get_abs_path, improve_single_line_input, EmailAddress};
 use crate::events::EventType;
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
@@ -25,6 +24,7 @@ use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::sql::{self, params_iter};
+use crate::tools::{get_abs_path, improve_single_line_input, EmailAddress};
 use crate::{chat, stock_str};
 
 /// Contact ID, including reserved IDs.
@@ -38,7 +38,7 @@ impl ContactId {
     pub const UNDEFINED: ContactId = ContactId::new(0);
     /// The owner of the account.
     ///
-    /// The email-address is set by `dc_set_config` using "addr".
+    /// The email-address is set by `set_config` using "addr".
     pub const SELF: ContactId = ContactId::new(1);
     pub const INFO: ContactId = ContactId::new(2);
     pub const DEVICE: ContactId = ContactId::new(5);
@@ -142,7 +142,7 @@ pub struct Contact {
     /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr` to access this field.
     addr: String,
 
-    /// Blocked state. Use dc_contact_is_blocked to access this field.
+    /// Blocked state. Use contact_is_blocked to access this field.
     pub blocked: bool,
 
     /// Time when the contact was seen last time, Unix time in seconds.
@@ -212,13 +212,13 @@ pub enum Origin {
     /// address is in our address book
     AddressBook = 0x80000,
 
-    /// set on Alice's side for contacts like Bob that have scanned the QR code offered by her. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
+    /// set on Alice's side for contacts like Bob that have scanned the QR code offered by her. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling contact_is_verified() !
     SecurejoinInvited = 0x0100_0000,
 
-    /// set on Bob's side for contacts scanned and verified from a QR code. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
+    /// set on Bob's side for contacts scanned and verified from a QR code. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling contact_is_verified() !
     SecurejoinJoined = 0x0200_0000,
 
-    /// contact added mannually by dc_create_contact(), this should be the largest origin as otherwise the user cannot modify the names
+    /// contact added mannually by create_contact(), this should be the largest origin as otherwise the user cannot modify the names
     ManuallyCreated = 0x0400_0000,
 }
 
@@ -344,7 +344,7 @@ impl Contact {
     /// We assume, the contact name, if any, is entered by the user and is used "as is" therefore,
     /// normalize() is *not* called for the name. If the contact is blocked, it is unblocked.
     ///
-    /// To add a number of contacts, see `dc_add_address_book()` which is much faster for adding
+    /// To add a number of contacts, see `add_address_book()` which is much faster for adding
     /// a bunch of addresses.
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
@@ -384,10 +384,10 @@ impl Contact {
 
     /// Check if an e-mail address belongs to a known and unblocked contact.
     ///
-    /// Known and unblocked contacts will be returned by `dc_get_contacts()`.
+    /// Known and unblocked contacts will be returned by `get_contacts()`.
     ///
     /// To validate an e-mail address independently of the contact database
-    /// use `dc_may_be_valid_addr()`.
+    /// use `may_be_valid_addr()`.
     pub async fn lookup_id_by_addr(
         context: &Context,
         addr: &str,
@@ -676,7 +676,7 @@ impl Contact {
 
     /// Returns known and unblocked contacts.
     ///
-    /// To get information about a single contact, see dc_get_contact().
+    /// To get information about a single contact, see get_contact().
     ///
     /// `listflags` is a combination of flags:
     /// - if the flag DC_GCL_ADD_SELF is set, SELF is added to the list unless filtered by other parameters
@@ -970,11 +970,11 @@ impl Contact {
         bail!("Could not delete contact with ongoing chats");
     }
 
-    /// Get a single contact object.  For a list, see eg. dc_get_contacts().
+    /// Get a single contact object.  For a list, see eg. get_contacts().
     ///
     /// For contact ContactId::SELF (1), the function returns sth.
     /// like "Me" in the selected language and the email address
-    /// defined by dc_set_config().
+    /// defined by set_config().
     pub async fn get_by_id(context: &Context, contact_id: ContactId) -> Result<Contact> {
         let contact = Contact::load_from_db(context, contact_id).await?;
 
@@ -1063,7 +1063,7 @@ impl Contact {
 
     /// Get the contact's profile image.
     /// This is the image set by each remote user on their own
-    /// using dc_set_config(context, "selfavatar", image).
+    /// using set_config(context, "selfavatar", image).
     pub async fn get_profile_image(&self, context: &Context) -> Result<Option<PathBuf>> {
         if self.id == ContactId::SELF {
             if let Some(p) = context.get_config(Config::Selfavatar).await? {
@@ -1071,7 +1071,7 @@ impl Contact {
             }
         } else if let Some(image_rel) = self.param.get(Param::ProfileImage) {
             if !image_rel.is_empty() {
-                return Ok(Some(dc_get_abs_path(context, image_rel)));
+                return Ok(Some(get_abs_path(context, image_rel)));
             }
         }
         Ok(None)
@@ -1438,15 +1438,12 @@ fn split_address_book(book: &str) -> Vec<(&str, &str)> {
 
 #[cfg(test)]
 mod tests {
-    use async_std::fs::File;
-    use async_std::io::WriteExt;
-
     use super::*;
 
     use crate::chat::{get_chat_contacts, send_text_msg, Chat};
     use crate::chatlist::Chatlist;
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::message::Message;
+    use crate::receive_imf::receive_imf;
     use crate::test_utils::{self, TestContext};
 
     #[test]
@@ -1508,7 +1505,7 @@ mod tests {
         )
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_contacts() -> Result<()> {
         let context = TestContext::new().await;
 
@@ -1572,7 +1569,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_is_self_addr() -> Result<()> {
         let t = TestContext::new().await;
         assert_eq!(t.is_self_addr("me@me.org").await?, false);
@@ -1584,7 +1581,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_or_lookup() {
         // add some contacts, this also tests add_address_book()
         let t = TestContext::new().await;
@@ -1685,12 +1682,12 @@ mod tests {
         assert!(!contact.is_blocked());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_contact_name_changes() -> Result<()> {
         let t = TestContext::new_alice().await;
 
         // first message creates contact and one-to-one-chat without name set
-        dc_receive_imf(
+        receive_imf(
             &t,
             b"From: f@example.org\n\
                  To: alice@example.org\n\
@@ -1719,7 +1716,7 @@ mod tests {
         assert_eq!(contacts.len(), 1);
 
         // second message inits the name
-        dc_receive_imf(
+        receive_imf(
             &t,
             b"From: Flobbyfoo <f@example.org>\n\
                  To: alice@example.org\n\
@@ -1747,7 +1744,7 @@ mod tests {
         assert_eq!(contacts.len(), 1);
 
         // third message changes the name
-        dc_receive_imf(
+        receive_imf(
             &t,
             b"From: Foo Flobby <f@example.org>\n\
                  To: alice@example.org\n\
@@ -1797,7 +1794,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_delete() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -1825,7 +1822,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_remote_authnames() {
         let t = TestContext::new().await;
 
@@ -1876,7 +1873,7 @@ mod tests {
         assert_eq!(contact.get_display_name(), "bob3");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_remote_authnames_create_empty() {
         let t = TestContext::new().await;
 
@@ -1925,7 +1922,7 @@ mod tests {
     ///
     /// In the past, "Not Bob" name was stuck until "Bob" changed the name to "Not Bob" and back in
     /// the "From:" field or user set the name to empty string manually.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_remote_authnames_update_to() -> Result<()> {
         let t = TestContext::new().await;
 
@@ -1958,7 +1955,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_remote_authnames_edit_empty() {
         let t = TestContext::new().await;
 
@@ -1995,7 +1992,7 @@ mod tests {
         assert!(addr_cmp(" mailto:AA@AA.ORG", "Aa@Aa.orG"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_name_in_address() {
         let t = TestContext::new().await;
 
@@ -2034,7 +2031,7 @@ mod tests {
             .is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_lookup_id_by_addr() {
         let t = TestContext::new().await;
 
@@ -2059,7 +2056,7 @@ mod tests {
         assert_eq!(id, Some(ContactId::SELF));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_contact_get_color() -> Result<()> {
         let t = TestContext::new().await;
         let contact_id = Contact::create(&t, "name", "name@example.net").await?;
@@ -2078,7 +2075,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_contact_get_encrinfo() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -2123,7 +2120,7 @@ CCCB 5AA9 F6E1 141C 9431
 
     /// Tests that status is synchronized when sending encrypted BCC-self messages and not
     /// synchronized when the message is not encrypted.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_synchronize_status() -> Result<()> {
         // Alice has two devices.
         let alice1 = TestContext::new_alice().await;
@@ -2188,7 +2185,7 @@ CCCB 5AA9 F6E1 141C 9431
     }
 
     /// Tests that DC_EVENT_SELFAVATAR_CHANGED is emitted on avatar changes.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_selfavatar_changed_event() -> Result<()> {
         // Alice has two devices.
         let alice1 = TestContext::new_alice().await;
@@ -2200,10 +2197,7 @@ CCCB 5AA9 F6E1 141C 9431
         assert_eq!(alice1.get_config(Config::Selfavatar).await?, None);
 
         let avatar_src = alice1.get_blobdir().join("avatar.png");
-        File::create(&avatar_src)
-            .await?
-            .write_all(test_utils::AVATAR_900x900_BYTES)
-            .await?;
+        tokio::fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?;
 
         alice1
             .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
@@ -2247,7 +2241,7 @@ CCCB 5AA9 F6E1 141C 9431
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_last_seen() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
@@ -2266,7 +2260,7 @@ Chat-Version: 1.0
 Date: Sun, 22 Mar 2020 22:37:55 +0000
 
 Hi."#;
-        dc_receive_imf(&alice, mime, false).await?;
+        receive_imf(&alice, mime, false).await?;
         let msg = alice.get_last_msg().await;
 
         let timestamp = msg.get_timestamp();

src/context.rs

@@ -3,27 +3,27 @@
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
-use std::time::{Instant, SystemTime};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::{Duration, Instant, SystemTime};
 
 use anyhow::{ensure, Result};
-use async_std::{
-    channel::{self, Receiver, Sender},
-    path::{Path, PathBuf},
-    sync::{Arc, Mutex, RwLock},
-};
+use async_channel::{self as channel, Receiver, Sender};
+use tokio::sync::{Mutex, RwLock};
 
 use crate::chat::{get_chat_cnt, ChatId};
 use crate::config::Config;
 use crate::constants::DC_VERSION_STR;
 use crate::contact::Contact;
-use crate::dc_tools::{duration_to_str, time};
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
 use crate::quota::QuotaInfo;
+use crate::ratelimit::Ratelimit;
 use crate::scheduler::Scheduler;
 use crate::sql::Sql;
+use crate::tools::{duration_to_str, time};
 
 #[derive(Clone, Debug)]
 pub struct Context {
@@ -55,11 +55,17 @@ pub struct InnerContext {
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Option<Scheduler>>,
+    pub(crate) ratelimit: RwLock<Ratelimit>,
 
     /// Recently loaded quota information, if any.
     /// Set to `None` if quota was never tried to load.
     pub(crate) quota: RwLock<Option<QuotaInfo>>,
 
+    /// Server ID response if ID capability is supported
+    /// and the server returned non-NIL on the inbox connection.
+    /// <https://datatracker.ietf.org/doc/html/rfc2971>
+    pub(crate) server_id: RwLock<Option<HashMap<String, String>>>,
+
     pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
 
     /// ID for this `Context` in the current process.
@@ -73,7 +79,7 @@ pub struct InnerContext {
     /// The text of the last error logged and emitted as an event.
     /// If the ui wants to display an error after a failure,
     /// `last_error` should be used to avoid races with the event thread.
-    pub(crate) last_error: RwLock<String>,
+    pub(crate) last_error: std::sync::RwLock<String>,
 }
 
 /// The state of ongoing process.
@@ -113,8 +119,8 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 
 impl Context {
     /// Creates new context and opens the database.
-    pub async fn new(dbfile: PathBuf, id: u32) -> Result<Context> {
-        let context = Self::new_closed(dbfile, id).await?;
+    pub async fn new(dbfile: &Path, id: u32, events: Events) -> Result<Context> {
+        let context = Self::new_closed(dbfile, id, events).await?;
 
         // Open the database if is not encrypted.
         if context.check_passphrase("".to_string()).await? {
@@ -124,15 +130,15 @@ impl Context {
     }
 
     /// Creates new context without opening the database.
-    pub async fn new_closed(dbfile: PathBuf, id: u32) -> Result<Context> {
+    pub async fn new_closed(dbfile: &Path, id: u32, events: Events) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         let blobdir = dbfile.with_file_name(blob_fname);
-        if !blobdir.exists().await {
-            async_std::fs::create_dir_all(&blobdir).await?;
+        if !blobdir.exists() {
+            tokio::fs::create_dir_all(&blobdir).await?;
         }
-        let context = Context::with_blobdir(dbfile, blobdir, id).await?;
+        let context = Context::with_blobdir(dbfile.into(), blobdir, id, events).await?;
         Ok(context)
     }
 
@@ -167,9 +173,10 @@ impl Context {
         dbfile: PathBuf,
         blobdir: PathBuf,
         id: u32,
+        events: Events,
     ) -> Result<Context> {
         ensure!(
-            blobdir.is_dir().await,
+            blobdir.is_dir(),
             "Blobdir does not exist: {}",
             blobdir.display()
         );
@@ -184,12 +191,14 @@ impl Context {
             oauth2_mutex: Mutex::new(()),
             wrong_pw_warning_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
-            events: Events::default(),
+            events,
             scheduler: RwLock::new(None),
+            ratelimit: RwLock::new(Ratelimit::new(Duration::new(60, 0), 6.0)), // Allow to send 6 messages immediately, no more than once every 10 seconds.
             quota: RwLock::new(None),
+            server_id: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
-            last_error: RwLock::new("".to_string()),
+            last_error: std::sync::RwLock::new("".to_string()),
         };
 
         let ctx = Context {
@@ -339,7 +348,7 @@ impl Context {
 
     pub async fn get_info(&self) -> Result<BTreeMap<&'static str, String>> {
         let unset = "0";
-        let l = LoginParam::load_candidate_params(self).await?;
+        let l = LoginParam::load_candidate_params_unchecked(self).await?;
         let l2 = LoginParam::load_configured_params(self).await?;
         let secondary_addrs = self.get_secondary_self_addrs().await?.join(", ");
         let displayname = self.get_config(Config::Displayname).await?;
@@ -426,6 +435,11 @@ impl Context {
         res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
+
+        let server_id = self.server_id.read().await;
+        res.insert("imap_server_id", format!("{:?}", server_id));
+        drop(server_id);
+
         res.insert("secondary_addrs", secondary_addrs);
         res.insert(
             "fetch_existing_msgs",
@@ -433,6 +447,12 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "fetched_existing_msgs",
+            self.get_config_bool(Config::FetchedExistingMsgs)
+                .await?
+                .to_string(),
+        );
         res.insert(
             "show_emails",
             self.get_config_int(Config::ShowEmails).await?.to_string(),
@@ -574,7 +594,7 @@ impl Context {
 
         let list = if let Some(chat_id) = chat_id {
             do_query(
-                "SELECT m.id AS id, m.timestamp AS timestamp
+                "SELECT m.id AS id
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id
@@ -598,7 +618,7 @@ impl Context {
             // According to some tests, this limit speeds up eg. 2 character searches by factor 10.
             // The limit is documented and UI may add a hint when getting 1000 results.
             do_query(
-                "SELECT m.id AS id, m.timestamp AS timestamp
+                "SELECT m.id AS id
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id
@@ -633,14 +653,14 @@ impl Context {
         Ok(mvbox.as_deref() == Some(folder_name))
     }
 
-    pub(crate) fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
+    pub(crate) fn derive_blobdir(dbfile: &Path) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         dbfile.with_file_name(blob_fname)
     }
 
-    pub(crate) fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
+    pub(crate) fn derive_walfile(dbfile: &Path) -> PathBuf {
         let mut wal_fname = OsString::new();
         wal_fname.push(dbfile.file_name().unwrap_or_default());
         wal_fname.push("-wal");
@@ -660,28 +680,28 @@ mod tests {
         get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
     };
     use crate::contact::ContactId;
-    use crate::dc_receive_imf::dc_receive_imf;
-    use crate::dc_tools::dc_create_outgoing_rfc724_mid;
     use crate::message::{Message, Viewtype};
+    use crate::receive_imf::receive_imf;
     use crate::test_utils::TestContext;
+    use crate::tools::create_outgoing_rfc724_mid;
     use anyhow::Context as _;
     use std::time::Duration;
     use strum::IntoEnumIterator;
     use tempfile::tempdir;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_wrong_db() -> Result<()> {
         let tmp = tempfile::tempdir()?;
         let dbfile = tmp.path().join("db.sqlite");
-        std::fs::write(&dbfile, b"123")?;
-        let res = Context::new(dbfile.into(), 1).await?;
+        tokio::fs::write(&dbfile, b"123").await?;
+        let res = Context::new(&dbfile, 1, Events::new()).await?;
 
         // Broken database is indistinguishable from encrypted one.
         assert_eq!(res.is_open().await, false);
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_fresh_msgs() {
         let t = TestContext::new().await;
         let fresh = t.get_fresh_msgs().await.unwrap();
@@ -702,13 +722,13 @@ mod tests {
              \n\
              hello\n",
             contact.get_addr(),
-            dc_create_outgoing_rfc724_mid(None, contact.get_addr())
+            create_outgoing_rfc724_mid(None, contact.get_addr())
         );
         println!("{}", msg);
-        dc_receive_imf(t, msg.as_bytes(), false).await.unwrap();
+        receive_imf(t, msg.as_bytes(), false).await.unwrap();
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_fresh_msgs_and_muted_chats() {
         // receive various mails in 3 chats
         let t = TestContext::new_alice().await;
@@ -758,7 +778,7 @@ mod tests {
         assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 9); // claire is counted again
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_fresh_msgs_and_muted_until() {
         let t = TestContext::new_alice().await;
         let bob = t.create_chat_with_contact("", "bob@g.it").await;
@@ -816,61 +836,61 @@ mod tests {
         assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new(dbfile.into(), 1).await.unwrap();
+        Context::new(&dbfile, 1, Events::new()).await.unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_wrong_blogdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
-        std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new(dbfile.into(), 1).await;
+        tokio::fs::write(&blobdir, b"123").await.unwrap();
+        let res = Context::new(&dbfile, 1, Events::new()).await;
         assert!(res.is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sqlite_parent_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new(dbfile.into(), 1).await.unwrap();
+        Context::new(&dbfile, 1, Events::new()).await.unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_with_empty_blobdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(dbfile.into(), blobdir, 1).await;
+        let res = Context::with_blobdir(dbfile, blobdir, 1, Events::new()).await;
         assert!(res.is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_with_blobdir_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(dbfile.into(), blobdir.into(), 1).await;
+        let res = Context::with_blobdir(dbfile, blobdir, 1, Events::new()).await;
         assert!(res.is_err());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn no_crashes_on_context_deref() {
         let t = TestContext::new().await;
         std::mem::drop(t);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_info() {
         let t = TestContext::new().await;
 
@@ -886,7 +906,7 @@ mod tests {
         assert_eq!(info.get("level").unwrap(), "awesome");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_info_completeness() {
         // For easier debugging,
         // get_info() shall return all important information configurable by the Config-values.
@@ -934,7 +954,7 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let self_talk = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
@@ -990,7 +1010,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_limit_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let chat = alice
@@ -1023,13 +1043,13 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_check_passphrase() -> Result<()> {
         let dir = tempdir()?;
         let dbfile = dir.path().join("db.sqlite");
 
         let id = 1;
-        let context = Context::new_closed(dbfile.clone().into(), id)
+        let context = Context::new_closed(&dbfile, id, Events::new())
             .await
             .context("failed to create context")?;
         assert_eq!(context.open("foo".to_string()).await?, true);
@@ -1037,7 +1057,7 @@ mod tests {
         drop(context);
 
         let id = 2;
-        let context = Context::new(dbfile.into(), id)
+        let context = Context::new(&dbfile, id, Events::new())
             .await
             .context("failed to create context")?;
         assert_eq!(context.is_open().await, false);
@@ -1048,7 +1068,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ongoing() -> Result<()> {
         let context = TestContext::new().await;
 

src/dehtml.rs

@@ -287,7 +287,7 @@ pub fn dehtml_manually(buf: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::simplify::simplify;
+    use crate::simplify::{simplify, SimplifiedText};
 
     #[test]
     fn test_dehtml() {
@@ -311,7 +311,7 @@ mod tests {
             ("<!some invalid html code>\n<b>some text</b>", "some text"),
         ];
         for (input, output) in cases {
-            assert_eq!(simplify(dehtml(input).unwrap(), true).0, output);
+            assert_eq!(simplify(dehtml(input).unwrap(), true).text, output);
         }
         let none_cases = vec!["<html> </html>", ""];
         for input in none_cases {
@@ -382,15 +382,21 @@ mod tests {
         assert_eq!(txt.trim(), "two\nlines");
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_quote_div() {
         let input = include_str!("../test-data/message/gmx-quote-body.eml");
         let dehtml = dehtml(input).unwrap();
         println!("{}", dehtml);
-        let (msg, forwarded, cut, top_quote, footer) = simplify(dehtml, false);
-        assert_eq!(msg, "Test");
-        assert_eq!(forwarded, false);
-        assert_eq!(cut, false);
+        let SimplifiedText {
+            text,
+            is_forwarded,
+            is_cut,
+            top_quote,
+            footer,
+        } = simplify(dehtml, false);
+        assert_eq!(text, "Test");
+        assert_eq!(is_forwarded, false);
+        assert_eq!(is_cut, false);
         assert_eq!(top_quote.as_deref(), Some("test"));
         assert_eq!(footer, None);
     }

src/download.rs

@@ -7,12 +7,12 @@ use std::collections::BTreeMap;
 
 use crate::config::Config;
 use crate::context::Context;
-use crate::dc_tools::time;
 use crate::imap::{Imap, ImapActionResult};
 use crate::job::{self, Action, Job, Status};
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
-use crate::param::Params;
+use crate::param::{Param, Params};
+use crate::tools::time;
 use crate::{job_try, stock_str, EventType};
 use std::cmp::max;
 
@@ -69,6 +69,42 @@ impl Context {
             Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32)))
         }
     }
+
+    // Merges the two messages to `placeholder_msg_id`;
+    // `full_msg_id` is no longer used afterwards.
+    pub(crate) async fn merge_messages(
+        &self,
+        full_msg_id: MsgId,
+        placeholder_msg_id: MsgId,
+    ) -> Result<()> {
+        let placeholder = Message::load_from_db(self, placeholder_msg_id).await?;
+        self.sql
+            .transaction(move |transaction| {
+                transaction
+                    .execute("DELETE FROM msgs WHERE id=?;", paramsv![placeholder_msg_id])?;
+                transaction.execute(
+                    "UPDATE msgs SET id=? WHERE id=?",
+                    paramsv![placeholder_msg_id, full_msg_id],
+                )?;
+                Ok(())
+            })
+            .await?;
+        let mut full = Message::load_from_db(self, placeholder_msg_id).await?;
+
+        for key in [
+            Param::WebxdcSummary,
+            Param::WebxdcSummaryTimestamp,
+            Param::WebxdcDocument,
+            Param::WebxdcDocumentTimestamp,
+        ] {
+            if let Some(value) = placeholder.param.get(key) {
+                full.param.set(key, value);
+            }
+        }
+        full.update_param(self).await?;
+
+        Ok(())
+    }
 }
 
 impl MsgId {
@@ -256,10 +292,10 @@ impl MimeMessage {
 mod tests {
     use num_traits::FromPrimitive;
 
-    use crate::chat::send_msg;
-    use crate::dc_receive_imf::dc_receive_imf_inner;
+    use crate::chat::{get_chat_msgs, send_msg};
     use crate::ephemeral::Timer;
     use crate::message::Viewtype;
+    use crate::receive_imf::receive_imf_inner;
     use crate::test_utils::TestContext;
 
     use super::*;
@@ -280,7 +316,7 @@ mod tests {
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_download_limit() -> Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -303,7 +339,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_update_download_state() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat = t.create_chat_with_contact("Bob", "bob@example.org").await;
@@ -328,7 +364,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_partial_receive_imf() -> Result<()> {
         let t = TestContext::new_alice().await;
 
@@ -342,7 +378,7 @@ mod tests {
              Date: Sun, 22 Mar 2020 22:37:57 +0000\
              Content-Type: text/plain";
 
-        dc_receive_imf_inner(
+        receive_imf_inner(
             &t,
             "Mr.12345678901@example.com",
             header.as_bytes(),
@@ -359,7 +395,7 @@ mod tests {
             .unwrap()
             .contains(&stock_str::partial_download_msg_body(&t, 100000).await));
 
-        dc_receive_imf_inner(
+        receive_imf_inner(
             &t,
             "Mr.12345678901@example.com",
             format!("{}\n\n100k text...", header).as_bytes(),
@@ -376,7 +412,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_partial_download_and_ephemeral() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat_id = t
@@ -388,7 +424,7 @@ mod tests {
             .await?;
 
         // download message from bob partially, this must not change the ephemeral timer
-        dc_receive_imf_inner(
+        receive_imf_inner(
             &t,
             "first@example.org",
             b"From: Bob <bob@example.org>\n\
@@ -410,4 +446,119 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_status_update_expands_to_nothing() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat_id = alice.create_chat(&bob).await.id;
+
+        let file = alice.get_blobdir().join("minimal.xdc");
+        tokio::fs::write(&file, include_bytes!("../test-data/webxdc/minimal.xdc")).await?;
+        let mut instance = Message::new(Viewtype::File);
+        instance.set_file(file.to_str().unwrap(), None);
+        let _sent1 = alice.send_msg(chat_id, &mut instance).await;
+
+        alice
+            .send_webxdc_status_update(instance.id, r#"{"payload":7}"#, "d")
+            .await?;
+        alice.flush_status_updates().await?;
+        let sent2 = alice.pop_sent_msg().await;
+        let sent2_rfc742_mid = Message::load_from_db(&alice, sent2.sender_msg_id)
+            .await?
+            .rfc724_mid;
+
+        // not downloading the status update results in an placeholder
+        receive_imf_inner(
+            &bob,
+            &sent2_rfc742_mid,
+            sent2.payload().as_bytes(),
+            false,
+            Some(sent2.payload().len() as u32),
+            false,
+        )
+        .await?;
+        let msg = bob.get_last_msg().await;
+        let chat_id = msg.chat_id;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 1);
+        assert_eq!(msg.download_state(), DownloadState::Available);
+
+        // downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat
+        // (usually status updates are too small for not being downloaded directly)
+        receive_imf_inner(
+            &bob,
+            &sent2_rfc742_mid,
+            sent2.payload().as_bytes(),
+            false,
+            None,
+            false,
+        )
+        .await?;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 0);
+        assert!(Message::load_from_db(&bob, msg.id)
+            .await?
+            .chat_id
+            .is_trash());
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_mdn_expands_to_nothing() -> Result<()> {
+        let bob = TestContext::new_bob().await;
+        let raw = b"Subject: Message opened\n\
+            Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <bar@example.org>\n\
+            To: Alice <alice@example.org>\n\
+            From: Bob <bob@example.org>\n\
+            Content-Type: multipart/report; report-type=disposition-notification;\n\t\
+            boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\
+            \n\
+            \n\
+            --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+            Content-Type: text/plain; charset=utf-8\n\
+            \n\
+            bla\n\
+            \n\
+            \n\
+            --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+            Content-Type: message/disposition-notification\n\
+            \n\
+            Reporting-UA: Delta Chat 1.88.0\n\
+            Original-Recipient: rfc822;bob@example.org\n\
+            Final-Recipient: rfc822;bob@example.org\n\
+            Original-Message-ID: <foo@example.org>\n\
+            Disposition: manual-action/MDN-sent-automatically; displayed\n\
+            \n\
+            \n\
+            --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\
+            ";
+
+        // not downloading the mdn results in an placeholder
+        receive_imf_inner(
+            &bob,
+            "bar@example.org",
+            raw,
+            false,
+            Some(raw.len() as u32),
+            false,
+        )
+        .await?;
+        let msg = bob.get_last_msg().await;
+        let chat_id = msg.chat_id;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 1);
+        assert_eq!(msg.download_state(), DownloadState::Available);
+
+        // downloading the mdn afterwards expands to nothing and deletes the placeholder directly
+        // (usually mdn are too small for not being downloaded directly)
+        receive_imf_inner(&bob, "bar@example.org", raw, false, None, false).await?;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 0);
+        assert!(Message::load_from_db(&bob, msg.id)
+            .await?
+            .chat_id
+            .is_trash());
+
+        Ok(())
+    }
 }

src/e2ee.rs

@@ -8,11 +8,13 @@ use num_traits::FromPrimitive;
 
 use crate::aheader::{Aheader, EncryptPreference};
 use crate::config::Config;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::Keyring;
+use crate::log::LogExt;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::pgp;
 
@@ -131,6 +133,56 @@ impl EncryptHelper {
     }
 }
 
+/// Applies Autocrypt header to Autocrypt peer state and saves it into the database.
+///
+/// If we already know this fingerprint from another contact's peerstate, return that
+/// peerstate in order to make AEAP work, but don't save it into the db yet.
+///
+/// Returns updated peerstate.
+pub(crate) async fn get_autocrypt_peerstate(
+    context: &Context,
+    from: &str,
+    autocrypt_header: Option<&Aheader>,
+    message_time: i64,
+) -> Result<Option<Peerstate>> {
+    let mut peerstate;
+
+    // Apply Autocrypt header
+    if let Some(header) = autocrypt_header {
+        // The "from_nongossiped_fingerprint" part is for AEAP:
+        // If we know this fingerprint from another addr,
+        // we may want to do a transition from this other addr
+        // (and keep its peerstate)
+        peerstate = Peerstate::from_nongossiped_fingerprint_or_addr(
+            context,
+            &header.public_key.fingerprint(),
+            from,
+        )
+        .await?;
+
+        if let Some(ref mut peerstate) = peerstate {
+            if addr_cmp(&peerstate.addr, from) {
+                peerstate.apply_header(header, message_time);
+                peerstate.save_to_db(&context.sql, false).await?;
+            }
+            // If `peerstate.addr` and `from` differ, this means that
+            // someone is using the same key but a different addr, probably
+            // because they made an AEAP transition.
+            // But we don't know if that's legit until we checked the
+            // signatures, so wait until then with writing anything
+            // to the database.
+        } else {
+            let p = Peerstate::from_header(header, message_time);
+            p.save_to_db(&context.sql, true).await?;
+            peerstate = Some(p);
+        }
+    } else {
+        peerstate = Peerstate::from_addr(context, from).await?;
+    }
+
+    Ok(peerstate)
+}
+
 /// Tries to decrypt a message, but only if it is structured as an
 /// Autocrypt message.
 ///
@@ -140,10 +192,42 @@ impl EncryptHelper {
 /// If the message is wrongly signed, this will still return the decrypted
 /// message but the HashSet will be empty.
 pub async fn try_decrypt(
+    context: &Context,
+    mail: &ParsedMail<'_>,
+    decryption_info: &DecryptionInfo,
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
+    // Possibly perform decryption
+    let public_keyring_for_validate = keyring_from_peerstate(&decryption_info.peerstate);
+
+    let context = context;
+    let encrypted_data_part = match get_autocrypt_mime(mail)
+        .or_else(|| get_mixed_up_mime(mail))
+        .or_else(|| get_attachment_mime(mail))
+    {
+        None => {
+            // not an autocrypt mime message, abort and ignore
+            return Ok(None);
+        }
+        Some(res) => res,
+    };
+    info!(context, "Detected Autocrypt-mime message");
+    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context)
+        .await
+        .context("failed to get own keyring")?;
+
+    decrypt_part(
+        encrypted_data_part,
+        private_keyring,
+        public_keyring_for_validate,
+    )
+    .await
+}
+
+pub async fn create_decryption_info(
     context: &Context,
     mail: &ParsedMail<'_>,
     message_time: i64,
-) -> Result<(Option<Vec<u8>>, HashSet<Fingerprint>)> {
+) -> Result<DecryptionInfo> {
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
@@ -152,56 +236,34 @@ pub async fn try_decrypt(
         .map(|from| from.addr)
         .unwrap_or_default();
 
-    let mut peerstate = Peerstate::from_addr(context, &from).await?;
+    let autocrypt_header = Aheader::from_headers(&from, &mail.headers)
+        .ok_or_log_msg(context, "Failed to parse Autocrypt header")
+        .flatten();
 
-    // Apply Autocrypt header
-    match Aheader::from_headers(&from, &mail.headers) {
-        Ok(Some(ref header)) => {
-            if let Some(ref mut peerstate) = peerstate {
-                peerstate.apply_header(header, message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            } else {
-                let p = Peerstate::from_header(header, message_time);
-                p.save_to_db(&context.sql, true).await?;
-                peerstate = Some(p);
-            }
-        }
-        Ok(None) => {}
-        Err(err) => warn!(context, "Failed to parse Autocrypt header: {}", err),
-    }
+    let peerstate =
+        get_autocrypt_peerstate(context, &from, autocrypt_header.as_ref(), message_time).await?;
 
-    // Possibly perform decryption
-    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
+    Ok(DecryptionInfo {
+        from,
+        autocrypt_header,
+        peerstate,
+        message_time,
+    })
+}
 
-    if let Some(ref mut peerstate) = peerstate {
-        peerstate
-            .handle_fingerprint_change(context, message_time)
-            .await?;
-        if let Some(key) = &peerstate.public_key {
-            public_keyring_for_validate.add(key.clone());
-        } else if let Some(key) = &peerstate.gossip_key {
-            public_keyring_for_validate.add(key.clone());
-        }
-    }
-
-    let (out_mail, signatures) =
-        match decrypt_if_autocrypt_message(context, mail, public_keyring_for_validate).await? {
-            Some((out_mail, signatures)) => (Some(out_mail), signatures),
-            None => (None, Default::default()),
-        };
-
-    if let Some(mut peerstate) = peerstate {
-        // If message is not encrypted and it is not a read receipt, degrade encryption.
-        if out_mail.is_none()
-            && message_time > peerstate.last_seen_autocrypt
-            && !contains_report(mail)
-        {
-            peerstate.degrade_encryption(message_time);
-            peerstate.save_to_db(&context.sql, false).await?;
-        }
-    }
-
-    Ok((out_mail, signatures))
+#[derive(Debug)]
+pub struct DecryptionInfo {
+    /// The From address. This is the address from the unnencrypted, outer
+    /// From header.
+    pub from: String,
+    pub autocrypt_header: Option<Aheader>,
+    /// The peerstate that will be used to validate the signatures
+    pub peerstate: Option<Peerstate>,
+    /// The timestamp when the message was sent.
+    /// If this is older than the peerstate's last_seen, this probably
+    /// means out-of-order message arrival, We don't modify the
+    /// peerstate in this case.
+    pub message_time: i64,
 }
 
 /// Returns a reference to the encrypted payload of a valid PGP/MIME message.
@@ -283,39 +345,23 @@ fn get_attachment_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMai
     }
 }
 
-async fn decrypt_if_autocrypt_message(
-    context: &Context,
-    mail: &ParsedMail<'_>,
-    public_keyring_for_validate: Keyring<SignedPublicKey>,
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    let encrypted_data_part = match get_autocrypt_mime(mail)
-        .or_else(|| get_mixed_up_mime(mail))
-        .or_else(|| get_attachment_mime(mail))
-    {
-        None => {
-            // not an autocrypt mime message, abort and ignore
-            return Ok(None);
+fn keyring_from_peerstate(peerstate: &Option<Peerstate>) -> Keyring<SignedPublicKey> {
+    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
+    if let Some(ref peerstate) = *peerstate {
+        if let Some(key) = &peerstate.public_key {
+            public_keyring_for_validate.add(key.clone());
+        } else if let Some(key) = &peerstate.gossip_key {
+            public_keyring_for_validate.add(key.clone());
         }
-        Some(res) => res,
-    };
-    info!(context, "Detected Autocrypt-mime message");
-    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context)
-        .await
-        .context("failed to get own keyring")?;
-
-    decrypt_part(
-        encrypted_data_part,
-        private_keyring,
-        public_keyring_for_validate,
-    )
-    .await
+    }
+    public_keyring_for_validate
 }
 
 /// Validates signatures of Multipart/Signed message part, as defined in RFC 1847.
 ///
 /// Returns `None` if the part is not a Multipart/Signed part, otherwise retruns the set of key
 /// fingerprints for which there is a valid signature.
-async fn validate_detached_signature(
+fn validate_detached_signature(
     mail: &ParsedMail<'_>,
     public_keyring_for_validate: &Keyring<SignedPublicKey>,
 ) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
@@ -328,7 +374,7 @@ async fn validate_detached_signature(
         let content = first_part.raw_bytes;
         let signature = second_part.get_body_raw()?;
         let ret_valid_signatures =
-            pgp::pk_validate(content, &signature, public_keyring_for_validate).await?;
+            pgp::pk_validate(content, &signature, public_keyring_for_validate)?;
 
         Ok(Some((content.to_vec(), ret_valid_signatures)))
     } else {
@@ -352,12 +398,12 @@ async fn decrypt_part(
         // If decrypted part is a multipart/signed, then there is a detached signature.
         let decrypted_part = mailparse::parse_mail(&plain)?;
         if let Some((content, valid_detached_signatures)) =
-            validate_detached_signature(&decrypted_part, &public_keyring_for_validate).await?
+            validate_detached_signature(&decrypted_part, &public_keyring_for_validate)?
         {
             return Ok(Some((content, valid_detached_signatures)));
         } else {
             // If the message was wrongly or not signed, still return the plain text.
-            // The caller has to check the signatures then.
+            // The caller has to check if the signatures set is empty then.
 
             return Ok(Some((plain, ret_valid_signatures)));
         }
@@ -380,18 +426,6 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     false
 }
 
-/// Checks if a MIME structure contains a multipart/report part.
-///
-/// As reports are often unencrypted, we do not reset the Autocrypt header in
-/// this case.
-///
-/// However, Delta Chat itself has no problem with encrypted multipart/report
-/// parts and MUAs should be encouraged to encrpyt multipart/reports as well so
-/// that we could use the normal Autocrypt processing.
-fn contains_report(mail: &ParsedMail<'_>) -> bool {
-    mail.ctype.mimetype == "multipart/report"
-}
-
 /// Ensures a private key exists for the configured user.
 ///
 /// Normally the private key is generated when the first message is
@@ -411,10 +445,10 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
 #[cfg(test)]
 mod tests {
     use crate::chat;
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::message::{Message, Viewtype};
     use crate::param::Param;
     use crate::peerstate::ToSave;
+    use crate::receive_imf::receive_imf;
     use crate::test_utils::{bob_keypair, TestContext};
 
     use super::*;
@@ -422,7 +456,7 @@ mod tests {
     mod ensure_secret_key_exists {
         use super::*;
 
-        #[async_std::test]
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
         async fn test_prexisting() {
             let t = TestContext::new_alice().await;
             assert_eq!(
@@ -431,7 +465,7 @@ mod tests {
             );
         }
 
-        #[async_std::test]
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
         async fn test_not_configured() {
             let t = TestContext::new().await;
             assert!(ensure_secret_key_exists(&t).await.is_err());
@@ -480,7 +514,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert_eq!(has_decrypted_pgp_armor(data), false);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -588,7 +622,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         vec![(Some(peerstate), addr)]
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_should_encrypt() {
         let t = TestContext::new_alice().await;
         let encrypt_helper = EncryptHelper::new(&t).await.unwrap();
@@ -615,7 +649,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mixed_up_mime() -> Result<()> {
         // "Mixed Up" mail as received when sending an encrypted
         // message using Delta Chat Desktop via ProtonMail IMAP/SMTP
@@ -646,7 +680,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert!(get_attachment_mime(&mail).is_some());
 
         let bob = TestContext::new_bob().await;
-        dc_receive_imf(&bob, attachment_mime, false).await?;
+        receive_imf(&bob, attachment_mime, false).await?;
         let msg = bob.get_last_msg().await;
         assert_eq!(msg.text.as_deref(), Some("Hello from Thunderbird!"));
 

src/ephemeral.rs

@@ -62,15 +62,14 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{ensure, Context as _, Result};
-use async_std::channel::Receiver;
-use async_std::future::timeout;
+use async_channel::Receiver;
 use serde::{Deserialize, Serialize};
+use tokio::time::timeout;
 
 use crate::chat::{send_msg, ChatId};
 use crate::constants::{DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH};
 use crate::contact::ContactId;
 use crate::context::Context;
-use crate::dc_tools::{duration_to_str, time};
 use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
 use crate::log::LogExt;
@@ -78,6 +77,7 @@ use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::sql::{self, params_iter};
 use crate::stock_str;
+use crate::tools::{duration_to_str, time};
 use std::cmp::max;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
@@ -339,7 +339,7 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
         .sql
         .execute(
             // If you change which information is removed here, also change MsgId::trash() and
-            // which information dc_receive_imf::add_parts() still adds to the db if the chat_id is TRASH
+            // which information receive_imf::add_parts() still adds to the db if the chat_id is TRASH
             r#"
 UPDATE msgs
 SET 
@@ -572,16 +572,16 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
 mod tests {
     use super::*;
     use crate::config::Config;
-    use crate::dc_receive_imf::dc_receive_imf;
-    use crate::dc_tools::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::download::DownloadState;
+    use crate::receive_imf::receive_imf;
     use crate::test_utils::TestContext;
+    use crate::tools::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::{
         chat::{self, Chat, ChatItem},
-        dc_tools::IsNoneOrEmpty,
+        tools::IsNoneOrEmpty,
     };
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_stock_ephemeral_messages() {
         let context = TestContext::new().await;
 
@@ -711,7 +711,7 @@ mod tests {
     }
 
     /// Test enabling and disabling ephemeral timer remotely.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_enable_disable() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -743,7 +743,7 @@ mod tests {
     }
 
     /// Test that timer is enabled even if the message explicitly enabling the timer is lost.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_enable_lost() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -785,7 +785,7 @@ mod tests {
 
     /// Test that Alice replying to the chat without a timer at the same time as Bob enables the
     /// timer does not result in disabling the timer on the Bob's side.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_timer_rollback() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -859,7 +859,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_delete_msgs() -> Result<()> {
         let t = TestContext::new_alice().await;
         let self_chat = t.get_self_chat().await;
@@ -985,7 +985,7 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_delete_expired_imap_messages() -> Result<()> {
         let t = TestContext::new_alice().await;
         const HOUR: i64 = 60 * 60;
@@ -1096,12 +1096,12 @@ mod tests {
     }
 
     // Regression test for a bug in the timer rollback protection.
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_timer_references() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
         // Message with Message-ID <first@example.com> and no timer is received.
-        dc_receive_imf(
+        receive_imf(
             &alice,
             b"From: Bob <bob@example.com>\n\
                     To: Alice <alice@example.org>\n\
@@ -1120,7 +1120,7 @@ mod tests {
         assert_eq!(chat_id.get_ephemeral_timer(&alice).await?, Timer::Disabled);
 
         // Message with Message-ID <second@example.com> is received.
-        dc_receive_imf(
+        receive_imf(
             &alice,
             b"From: Bob <bob@example.com>\n\
                     To: Alice <alice@example.org>\n\
@@ -1155,7 +1155,7 @@ mod tests {
         //
         // The message also contains a quote of the first message to test that only References:
         // header and not In-Reply-To: is consulted by the rollback protection.
-        dc_receive_imf(
+        receive_imf(
             &alice,
             b"From: Bob <bob@example.com>\n\
                     To: Alice <alice@example.org>\n\

src/error.rs

@@ -1,31 +0,0 @@
-//! # Error handling
-
-#[macro_export]
-macro_rules! ensure_eq {
-    ($left:expr, $right:expr) => ({
-        match (&$left, &$right) {
-            (left_val, right_val) => {
-                if !(*left_val == *right_val) {
-                    bail!(r#"assertion failed: `(left == right)`
-  left: `{:?}`,
- right: `{:?}`"#, left_val, right_val)
-                }
-            }
-        }
-    });
-    ($left:expr, $right:expr,) => ({
-        ensure_eq!($left, $right)
-    });
-    ($left:expr, $right:expr, $($arg:tt)+) => ({
-        match (&($left), &($right)) {
-            (left_val, right_val) => {
-                if !(*left_val == *right_val) {
-                    bail!(r#"assertion failed: `(left == right)`
-  left: `{:?}`,
- right: `{:?}`: {}"#, left_val, right_val,
-                           format_args!($($arg)+))
-                }
-            }
-        }
-    });
-}

src/events.rs

@@ -1,7 +1,8 @@
 //! # Events specification.
 
-use async_std::channel::{self, Receiver, Sender, TrySendError};
-use async_std::path::PathBuf;
+use std::path::PathBuf;
+
+use async_channel::{self as channel, Receiver, Sender, TrySendError};
 
 use crate::chat::ChatId;
 use crate::contact::ContactId;
@@ -9,7 +10,8 @@ use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
 use crate::webxdc::StatusUpdateSerial;
 
-#[derive(Debug)]
+/// Event channel.
+#[derive(Debug, Clone)]
 pub struct Events {
     receiver: Receiver<Event>,
     sender: Sender<Event>,
@@ -17,13 +19,17 @@ pub struct Events {
 
 impl Default for Events {
     fn default() -> Self {
-        let (sender, receiver) = channel::bounded(1_000);
-
-        Self { receiver, sender }
+        Self::new()
     }
 }
 
 impl Events {
+    pub fn new() -> Self {
+        let (sender, receiver) = channel::bounded(1_000);
+
+        Self { receiver, sender }
+    }
+
     pub fn emit(&self, event: Event) {
         match self.sender.try_send(event) {
             Ok(()) => {}
@@ -56,23 +62,18 @@ impl Events {
 ///
 /// [`Context`]: crate::context::Context
 /// [`Context::get_event_emitter`]: crate::context::Context::get_event_emitter
-/// [`Stream`]: async_std::stream::Stream
+/// [`Stream`]: futures::stream::Stream
 #[derive(Debug, Clone)]
 pub struct EventEmitter(Receiver<Event>);
 
 impl EventEmitter {
-    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
-    pub fn recv_sync(&self) -> Option<Event> {
-        async_std::task::block_on(self.recv())
-    }
-
     /// Async recv of an event. Return `None` if the `Sender` has been droped.
     pub async fn recv(&self) -> Option<Event> {
         self.0.recv().await.ok()
     }
 }
 
-impl async_std::stream::Stream for EventEmitter {
+impl futures::stream::Stream for EventEmitter {
     type Item = Event;
 
     fn poll_next(

src/format_flowed.rs

@@ -51,14 +51,26 @@ fn format_line_flowed(line: &str, prefix: &str) -> String {
     result + &buffer
 }
 
-fn format_flowed_prefix(text: &str, prefix: &str) -> String {
+/// Returns text formatted according to RFC 3767 (format=flowed).
+///
+/// This function accepts text separated by LF, but returns text
+/// separated by CRLF.
+///
+/// RFC 2646 technique is used to insert soft line breaks, so DelSp
+/// SHOULD be set to "no" when sending.
+pub(crate) fn format_flowed(text: &str) -> String {
     let mut result = String::new();
 
     for line in text.split('\n') {
         if !result.is_empty() {
             result += "\r\n";
         }
-        let line = line.trim_end();
+
+        let line_no_prefix = line.strip_prefix('>');
+        let is_quote = line_no_prefix.is_some();
+        let line = line_no_prefix.unwrap_or(line).trim();
+        let prefix = if is_quote { "> " } else { "" };
+
         if prefix.len() + line.len() > 78 {
             result += &format_line_flowed(line, prefix);
         } else {
@@ -70,23 +82,23 @@ fn format_flowed_prefix(text: &str, prefix: &str) -> String {
             result += line;
         }
     }
-    result
-}
 
-/// Returns text formatted according to RFC 3767 (format=flowed).
-///
-/// This function accepts text separated by LF, but returns text
-/// separated by CRLF.
-///
-/// RFC 2646 technique is used to insert soft line breaks, so DelSp
-/// SHOULD be set to "no" when sending.
-pub fn format_flowed(text: &str) -> String {
-    format_flowed_prefix(text, "")
+    result
 }
 
 /// Same as format_flowed(), but adds "> " prefix to each line.
 pub fn format_flowed_quote(text: &str) -> String {
-    format_flowed_prefix(text, "> ")
+    let mut result = String::new();
+
+    for line in text.split('\n') {
+        if !result.is_empty() {
+            result += "\n";
+        }
+        result += "> ";
+        result += line;
+    }
+
+    format_flowed(&result)
 }
 
 /// Joins lines in format=flowed text.
@@ -129,6 +141,7 @@ pub fn unformat_flowed(text: &str, delsp: bool) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_format_flowed() {
@@ -144,18 +157,18 @@ mod tests {
                         client and enter the setup code presented on the generating device.";
         assert_eq!(format_flowed(text), expected);
 
-        let text = "> Not a quote";
-        assert_eq!(format_flowed(text), " > Not a quote");
+        let text = "> A quote";
+        assert_eq!(format_flowed(text), "> A quote");
 
         // Test space stuffing of wrapped lines
         let text = "> This is the Autocrypt Setup Message used to transfer your key between clients.\n\
                     >                               \n\
                     > To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
-        let expected = "\x20> This is the Autocrypt Setup Message used to transfer your key between \r\n\
-                        clients.\r\n\
-                        \x20>\r\n\
-                        \x20> To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
-                        client and enter the setup code presented on the generating device.";
+        let expected = "> This is the Autocrypt Setup Message used to transfer your key between \r\n\
+                        > clients.\r\n\
+                        > \r\n\
+                        > To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        > client and enter the setup code presented on the generating device.";
         assert_eq!(format_flowed(text), expected);
     }
 
@@ -175,6 +188,10 @@ mod tests {
         let expected = "> this is a quoted line";
         assert_eq!(format_flowed_quote(quote), expected);
 
+        let quote = "first quoted line\nsecond quoted line";
+        let expected = "> first quoted line\r\n> second quoted line";
+        assert_eq!(format_flowed_quote(quote), expected);
+
         let quote = "> foo bar baz";
         let expected = "> > foo bar baz";
         assert_eq!(format_flowed_quote(quote), expected);
@@ -185,4 +202,25 @@ mod tests {
             > unwrapped on the receiver";
         assert_eq!(format_flowed_quote(quote), expected);
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_send_quotes() -> anyhow::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat = alice.create_chat(&bob).await;
+
+        let sent = alice.send_text(chat.id, "> First quote").await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some("> First quote"));
+        assert!(received.quoted_text().is_none());
+        assert!(received.quoted_message(&bob).await?.is_none());
+
+        let sent = alice.send_text(chat.id, "> Second quote").await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some("> Second quote"));
+        assert!(received.quoted_text().is_none());
+        assert!(received.quoted_message(&bob).await?.is_none());
+
+        Ok(())
+    }
 }

src/html.rs

@@ -64,7 +64,7 @@ enum MimeMultipartType {
 
 /// Function takes a content type from a ParsedMail structure
 /// and checks and returns the rough mime-type.
-async fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
+fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
     let mimetype = ctype.mimetype.to_lowercase();
     if mimetype.starts_with("multipart") && ctype.params.get("boundary").is_some() {
         MimeMultipartType::Multiple
@@ -122,7 +122,7 @@ impl HtmlMsgParser {
     ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
         // Boxed future to deal with recursion
         async move {
-            match get_mime_multipart_type(&mail.ctype).await {
+            match get_mime_multipart_type(&mail.ctype) {
                 MimeMultipartType::Multiple => {
                     for cur_data in mail.subparts.iter() {
                         self.collect_texts_recursive(context, cur_data).await?
@@ -178,7 +178,7 @@ impl HtmlMsgParser {
     ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
         // Boxed future to deal with recursion
         async move {
-            match get_mime_multipart_type(&mail.ctype).await {
+            match get_mime_multipart_type(&mail.ctype) {
                 MimeMultipartType::Multiple => {
                     for cur_data in mail.subparts.iter() {
                         self.cid_to_data_recursive(context, cur_data).await?;
@@ -198,7 +198,7 @@ impl HtmlMsgParser {
                     if mimetype.type_() == mime::IMAGE {
                         if let Some(cid) = mail.headers.get_header_value(HeaderDef::ContentId) {
                             if let Ok(cid) = parse_message_id(&cid) {
-                                if let Ok(replacement) = mimepart_to_data_url(mail).await {
+                                if let Ok(replacement) = mimepart_to_data_url(mail) {
                                     let re_string = format!(
                                         "(<img[^>]*src[^>]*=[^>]*)(cid:{})([^>]*>)",
                                         regex::escape(&cid)
@@ -233,7 +233,7 @@ impl HtmlMsgParser {
 }
 
 /// Convert a mime part to a data: url as defined in [RFC 2397](https://tools.ietf.org/html/rfc2397).
-async fn mimepart_to_data_url(mail: &mailparse::ParsedMail<'_>) -> Result<String> {
+fn mimepart_to_data_url(mail: &mailparse::ParsedMail<'_>) -> Result<String> {
     let data = mail.get_body_raw()?;
     let data = base64::encode(&data);
     Ok(format!("data:{};base64,{}", mail.ctype.mimetype, data))
@@ -267,7 +267,7 @@ impl MsgId {
 ///
 /// Used on forwarding messages to avoid leaking the original mime structure
 /// and also to avoid sending too much, maybe large data.
-pub async fn new_html_mimepart(html: String) -> PartBuilder {
+pub fn new_html_mimepart(html: String) -> PartBuilder {
     PartBuilder::new()
         .content_type(&"text/html; charset=utf-8".parse::<mime::Mime>().unwrap())
         .body(html)
@@ -280,11 +280,11 @@ mod tests {
     use crate::chat::forward_msgs;
     use crate::config::Config;
     use crate::contact::ContactId;
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::message::{MessengerMessage, Viewtype};
+    use crate::receive_imf::receive_imf;
     use crate::test_utils::TestContext;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_plain_unspecified() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
@@ -300,7 +300,7 @@ This message does not have Content-Type nor Subject.<br/>
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_plain_iso88591() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_plain_iso88591.eml");
@@ -316,7 +316,7 @@ message with a non-UTF-8 encoding: äöüßÄÖÜ<br/>
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_plain_flowed() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_plain_flowed.eml");
@@ -336,7 +336,7 @@ and will be wrapped as usual.<br/>
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_alt_plain() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
@@ -355,7 +355,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_html() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_html.eml");
@@ -373,7 +373,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_alt_html() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_alt_html.eml");
@@ -388,7 +388,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_alt_plain_html() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
@@ -405,7 +405,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         );
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_htmlparse_apple_cid_jpg() {
         // load raw mime html-data with related image-part (cid:)
         // and make sure, Content-Id has angle-brackets that are removed correctly.
@@ -424,14 +424,14 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         assert!(!parser.html.contains("cid:"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_html_invalid_msgid() {
         let t = TestContext::new().await;
         let msg_id = MsgId::new(100);
         assert!(msg_id.get_html(&t).await.is_err())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_html_forwarding() {
         // alice receives a non-delta html-message
         let alice = TestContext::new_alice().await;
@@ -440,7 +440,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, false).await.unwrap();
+        receive_imf(&alice, raw, false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
         assert_ne!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
@@ -478,7 +478,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         assert!(html.contains("this is <b>html</b>"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_html_forwarding_encrypted() {
         // Alice receives a non-delta html-message
         // (`ShowEmails=1` lets Alice actually receive non-delta messages for known contacts,
@@ -489,7 +489,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, false).await.unwrap();
+        receive_imf(&alice, raw, false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
 
         // forward the message to saved-messages,
@@ -515,7 +515,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         assert!(html.contains("this is <b>html</b>"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_html() {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
@@ -547,11 +547,11 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         assert!(html.contains("<b>html</b> text"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_cp1252_html() -> Result<()> {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await?;
-        dc_receive_imf(
+        receive_imf(
             &t,
             include_bytes!("../test-data/message/cp1252-html.eml"),
             false,

src/imap.rs

@@ -11,25 +11,20 @@ use std::{
 };
 
 use anyhow::{bail, format_err, Context as _, Result};
+use async_channel::Receiver;
 use async_imap::types::{
     Fetch, Flag, Mailbox, Name, NameAttribute, Quota, QuotaRoot, UnsolicitedResponse,
 };
-use async_std::channel::Receiver;
-use async_std::prelude::*;
+use futures::StreamExt;
 use num_traits::FromPrimitive;
 
 use crate::chat::{self, ChatId, ChatIdBlocked};
 use crate::config::Config;
 use crate::constants::{
     Blocked, Chattype, ShowEmails, DC_FETCH_EXISTING_MSGS_COUNT, DC_FOLDERS_CONFIGURED_VERSION,
-    DC_LP_AUTH_OAUTH2,
 };
-use crate::contact::ContactId;
+use crate::contact::{normalize_name, Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
-use crate::dc_receive_imf::{
-    dc_receive_imf_inner, from_field_to_contact_id, get_prefetch_parent_message, ReceivedMsg,
-};
-use crate::dc_tools::dc_create_id;
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job;
@@ -38,12 +33,16 @@ use crate::login_param::{
 };
 use crate::message::{self, Message, MessageState, MessengerMessage, MsgId, Viewtype};
 use crate::mimeparser;
-use crate::oauth2::dc_get_oauth2_access_token;
+use crate::oauth2::get_oauth2_access_token;
 use crate::provider::Socket;
+use crate::receive_imf::{
+    from_field_to_contact_id, get_prefetch_parent_message, receive_imf_inner, ReceivedMsg,
+};
 use crate::scheduler::connectivity::ConnectivityStore;
 use crate::scheduler::InterruptInfo;
 use crate::sql;
 use crate::stock_str;
+use crate::tools::create_id;
 
 mod client;
 mod idle;
@@ -154,7 +153,6 @@ struct ImapConfig {
     pub lp: ServerLoginParam,
     pub socks5_config: Option<Socks5Config>,
     pub strict_tls: bool,
-    pub oauth2: bool,
     pub selected_folder: Option<String>,
     pub selected_mailbox: Option<Mailbox>,
     pub selected_folder_needs_expunge: bool,
@@ -243,7 +241,6 @@ impl Imap {
         lp: &ServerLoginParam,
         socks5_config: Option<Socks5Config>,
         addr: &str,
-        oauth2: bool,
         provider_strict_tls: bool,
         idle_interrupt: Receiver<InterruptInfo>,
     ) -> Result<Self> {
@@ -262,7 +259,6 @@ impl Imap {
             lp: lp.clone(),
             socks5_config,
             strict_tls,
-            oauth2,
             selected_folder: None,
             selected_mailbox: None,
             selected_folder_needs_expunge: false,
@@ -301,7 +297,6 @@ impl Imap {
             &param.imap,
             param.socks5_config.clone(),
             &param.addr,
-            param.server_flags & DC_LP_AUTH_OAUTH2 != 0,
             param
                 .provider
                 .map_or(param.socks5_config.is_some(), |provider| {
@@ -334,7 +329,7 @@ impl Imap {
 
         self.connectivity.set_connecting(context).await;
 
-        let oauth2 = self.config.oauth2;
+        let oauth2 = self.config.lp.oauth2;
 
         let connection_res: Result<Client> = if self.config.lp.security == Socket::Starttls
             || self.config.lp.security == Socket::Plain
@@ -395,7 +390,7 @@ impl Imap {
         let login_res = if oauth2 {
             let addr: &str = config.addr.as_ref();
 
-            let token = dc_get_oauth2_access_token(context, addr, imap_pw, true)
+            let token = get_oauth2_access_token(context, addr, imap_pw, true)
                 .await?
                 .context("IMAP could not get OAUTH token")?;
             let auth = OAuth2 {
@@ -456,7 +451,9 @@ impl Imap {
     }
 
     /// Determine server capabilities if not done yet.
-    async fn determine_capabilities(&mut self) -> Result<()> {
+    ///
+    /// If server supports ID capability, send our client ID.
+    pub(crate) async fn determine_capabilities(&mut self, context: &Context) -> Result<()> {
         if self.capabilities_determined {
             return Ok(());
         }
@@ -468,6 +465,12 @@ impl Imap {
             .capabilities()
             .await
             .context("CAPABILITY command error")?;
+        if caps.has_str("ID") {
+            let server_id = session.id([("name", Some("Delta Chat"))]).await?;
+            info!(context, "Server ID: {:?}", server_id);
+            let mut lock = context.server_id.write().await;
+            *lock = server_id;
+        }
         self.config.can_idle = caps.has_str("IDLE");
         self.config.can_move = caps.has_str("MOVE");
         self.config.can_check_quota = caps.has_str("QUOTA");
@@ -486,8 +489,8 @@ impl Imap {
             return Err(err);
         }
 
+        self.determine_capabilities(context).await?;
         self.ensure_configured_folders(context, true).await?;
-        self.determine_capabilities().await?;
         Ok(())
     }
 
@@ -792,7 +795,7 @@ impl Imap {
             };
 
             // Get the Message-ID or generate a fake one to identify the message in the database.
-            let message_id = prefetch_get_message_id(&headers).unwrap_or_else(dc_create_id);
+            let message_id = prefetch_get_message_id(&headers).unwrap_or_else(create_id);
 
             let target = match target_folder(context, folder, is_spam_folder, &headers).await? {
                 Some(config) => match context.get_config(config).await? {
@@ -880,8 +883,8 @@ impl Imap {
         received_msgs.extend(received_msgs_2);
 
         // determine which uid_next to use to update to
-        // dc_receive_imf() returns an `Err` value only on recoverable errors, otherwise it just logs an error.
-        // `largest_uid_processed` is the largest uid where dc_receive_imf() did NOT return an error.
+        // receive_imf() returns an `Err` value only on recoverable errors, otherwise it just logs an error.
+        // `largest_uid_processed` is the largest uid where receive_imf() did NOT return an error.
 
         // So: Update the uid_next to the largest uid that did NOT recoverably fail. Not perfect because if there was
         // another message afterwards that succeeded, we will not retry. The upside is that we will not retry an infinite amount of times.
@@ -905,6 +908,42 @@ impl Imap {
         Ok(read_cnt > 0)
     }
 
+    /// Read the recipients from old emails sent by the user and add them as contacts.
+    /// This way, we can already offer them some email addresses they can write to.
+    ///
+    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
+    /// and show them in the chat list.
+    pub(crate) async fn fetch_existing_msgs(&mut self, context: &Context) -> Result<()> {
+        if context.get_config_bool(Config::Bot).await? {
+            return Ok(()); // Bots don't want those messages
+        }
+        self.prepare(context).await.context("could not connect")?;
+
+        add_all_recipients_as_contacts(context, self, Config::ConfiguredSentboxFolder).await;
+        add_all_recipients_as_contacts(context, self, Config::ConfiguredMvboxFolder).await;
+        add_all_recipients_as_contacts(context, self, Config::ConfiguredInboxFolder).await;
+
+        if context.get_config_bool(Config::FetchExistingMsgs).await? {
+            for config in &[
+                Config::ConfiguredMvboxFolder,
+                Config::ConfiguredInboxFolder,
+                Config::ConfiguredSentboxFolder,
+            ] {
+                if let Some(folder) = context.get_config(*config).await? {
+                    self.fetch_new_messages(context, &folder, false, true)
+                        .await
+                        .context("could not fetch messages")?;
+                }
+            }
+        }
+
+        info!(context, "Done fetching existing messages.");
+        context
+            .set_config_bool(Config::FetchedExistingMsgs, true)
+            .await?;
+        Ok(())
+    }
+
     /// Deletes batch of messages identified by their UID from the currently
     /// selected folder.
     async fn delete_message_batch(
@@ -1400,7 +1439,7 @@ impl Imap {
                     continue;
                 }
 
-                // XXX put flags into a set and pass them to dc_receive_imf
+                // XXX put flags into a set and pass them to receive_imf
                 let context = context.clone();
 
                 // safe, as we checked above that there is a body.
@@ -1418,7 +1457,7 @@ impl Imap {
                     );
                     ""
                 };
-                match dc_receive_imf_inner(
+                match receive_imf_inner(
                     &context,
                     rfc724_mid,
                     body,
@@ -1435,7 +1474,7 @@ impl Imap {
                         last_uid = Some(server_uid)
                     }
                     Err(err) => {
-                        warn!(context, "dc_receive_imf error: {:#}", err);
+                        warn!(context, "receive_imf error: {:#}", err);
                     }
                 };
             }
@@ -1530,6 +1569,59 @@ impl Imap {
         self.configure_folders(context, create_mvbox).await
     }
 
+    /// Attempts to configure mvbox.
+    ///
+    /// Tries to find any folder in the given list of `folders`. If none is found, tries to create
+    /// any of them in the same order. This method does not use LIST command to ensure that
+    /// configuration works even if mailbox lookup is forbidden via Access Control List (see
+    /// <https://datatracker.ietf.org/doc/html/rfc4314>).
+    ///
+    /// Returns first found or created folder name.
+    async fn configure_mvbox<'a>(
+        &mut self,
+        context: &Context,
+        folders: &[&'a str],
+        create_mvbox: bool,
+    ) -> Result<Option<&'a str>> {
+        // Close currently selected folder if needed.
+        // We are going to select folders using low-level EXAMINE operations below.
+        self.select_folder(context, None).await?;
+
+        let session = self
+            .session
+            .as_mut()
+            .context("no IMAP connection established")?;
+
+        for folder in folders {
+            info!(context, "Looking for MVBOX-folder \"{}\"...", &folder);
+            let res = session.examine(&folder).await;
+            if res.is_ok() {
+                info!(
+                    context,
+                    "MVBOX-folder {:?} successfully selected, using it.", &folder
+                );
+                session.close().await?;
+                return Ok(Some(folder));
+            }
+        }
+
+        if create_mvbox {
+            for folder in folders {
+                match session.create(&folder).await {
+                    Ok(_) => {
+                        info!(context, "MVBOX-folder {} created.", &folder);
+                        return Ok(Some(folder));
+                    }
+                    Err(err) => {
+                        warn!(context, "Cannot create MVBOX-folder {:?}: {}", &folder, err);
+                    }
+                }
+            }
+        }
+
+        Ok(None)
+    }
+
     pub async fn configure_folders(&mut self, context: &Context, create_mvbox: bool) -> Result<()> {
         let session = self
             .session
@@ -1542,9 +1634,7 @@ impl Imap {
             .context("list_folders failed")?;
         let mut delimiter = ".".to_string();
         let mut delimiter_is_default = true;
-        let mut mvbox_folder = None;
         let mut folder_configs = BTreeMap::new();
-        let mut fallback_folder = get_fallback_folder(&delimiter);
 
         while let Some(folder) = folders.next().await {
             let folder = folder?;
@@ -1554,22 +1644,13 @@ impl Imap {
             if let Some(d) = folder.delimiter() {
                 if delimiter_is_default && !d.is_empty() && delimiter != d {
                     delimiter = d.to_string();
-                    fallback_folder = get_fallback_folder(&delimiter);
                     delimiter_is_default = false;
                 }
             }
 
             let folder_meaning = get_folder_meaning(&folder);
             let folder_name_meaning = get_folder_meaning_by_name(folder.name());
-            if folder.name() == "DeltaChat" {
-                // Always takes precedence
-                mvbox_folder = Some(folder.name().to_string());
-            } else if folder.name() == fallback_folder {
-                // only set if none has been already set
-                if mvbox_folder.is_none() {
-                    mvbox_folder = Some(folder.name().to_string());
-                }
-            } else if let Some(config) = folder_meaning.to_config() {
+            if let Some(config) = folder_meaning.to_config() {
                 // Always takes precedence
                 folder_configs.insert(config, folder.name().to_string());
             } else if let Some(config) = folder_name_meaning.to_config() {
@@ -1583,47 +1664,17 @@ impl Imap {
 
         info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
 
-        if mvbox_folder.is_none() && create_mvbox {
-            info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
+        let fallback_folder = format!("INBOX{}DeltaChat", delimiter);
+        let mvbox_folder = self
+            .configure_mvbox(context, &["DeltaChat", &fallback_folder], create_mvbox)
+            .await
+            .context("failed to configure mvbox")?;
 
-            match session.create("DeltaChat").await {
-                Ok(_) => {
-                    mvbox_folder = Some("DeltaChat".into());
-                    info!(context, "MVBOX-folder created.",);
-                }
-                Err(err) => {
-                    warn!(
-                        context,
-                        "Cannot create MVBOX-folder, trying to create INBOX subfolder. ({})", err
-                    );
-
-                    match session.create(&fallback_folder).await {
-                        Ok(_) => {
-                            mvbox_folder = Some(fallback_folder);
-                            info!(
-                                context,
-                                "MVBOX-folder created as INBOX subfolder. ({})", err
-                            );
-                        }
-                        Err(err) => {
-                            warn!(context, "Cannot create MVBOX-folder. ({})", err);
-                        }
-                    }
-                }
-            }
-            // SUBSCRIBE is needed to make the folder visible to the LSUB command
-            // that may be used by other MUAs to list folders.
-            // for the LIST command, the folder is always visible.
-            if let Some(ref mvbox) = mvbox_folder {
-                if let Err(err) = session.subscribe(mvbox).await {
-                    warn!(context, "could not subscribe to {:?}: {:?}", mvbox, err);
-                }
-            }
-        }
         context
             .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
             .await?;
-        if let Some(ref mvbox_folder) = mvbox_folder {
+        if let Some(mvbox_folder) = mvbox_folder {
+            info!(context, "Setting MVBOX FOLDER TO {}", &mvbox_folder);
             context
                 .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
                 .await?;
@@ -1686,11 +1737,11 @@ async fn should_move_out_of_spam(
         // the SecureJoin header. So, we always move chat messages out of Spam.
         // Two possibilities to change this would be:
         // 1. Remove the `&& !context.is_spam_folder(folder).await?` check from
-        // `fetch_new_messages()`, and then let `dc_receive_imf()` check
+        // `fetch_new_messages()`, and then let `receive_imf()` check
         // if it's a spam message and should be hidden.
         // 2. Or add a flag to the ChatVersion header that this is a securejoin
         // request, and return `true` here only if the message has this flag.
-        // `dc_receive_imf()` can then check if the securejoin request is valid.
+        // `receive_imf()` can then check if the securejoin request is valid.
         return Ok(true);
     }
 
@@ -1891,7 +1942,7 @@ fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
 
 fn get_folder_meaning(folder_name: &Name) -> FolderMeaning {
     for attr in folder_name.attributes() {
-        if let NameAttribute::Custom(ref label) = attr {
+        if let NameAttribute::Extension(ref label) = attr {
             match label.as_ref() {
                 "\\Trash" => return FolderMeaning::Other,
                 "\\Sent" => return FolderMeaning::Sent,
@@ -2013,10 +2064,6 @@ pub(crate) async fn prefetch_should_download(
     Ok(should_download)
 }
 
-fn get_fallback_folder(delimiter: &str) -> String {
-    format!("INBOX{}DeltaChat", delimiter)
-}
-
 /// Marks messages in `msgs` table as seen, searching for them by UID.
 ///
 /// Returns updated chat ID if any message was marked as seen.
@@ -2287,6 +2334,50 @@ impl std::fmt::Display for UidRange {
         }
     }
 }
+async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
+    let mailbox = if let Ok(Some(m)) = context.get_config(folder).await {
+        m
+    } else {
+        return;
+    };
+    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
+        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+        warn!(context, "Could not select {}: {:#}", mailbox, e);
+        return;
+    }
+    match imap.get_all_recipients(context).await {
+        Ok(contacts) => {
+            let mut any_modified = false;
+            for contact in contacts {
+                let display_name_normalized = contact
+                    .display_name
+                    .as_ref()
+                    .map(|s| normalize_name(s))
+                    .unwrap_or_default();
+
+                match Contact::add_or_lookup(
+                    context,
+                    &display_name_normalized,
+                    &contact.addr,
+                    Origin::OutgoingTo,
+                )
+                .await
+                {
+                    Ok((_, modified)) => {
+                        if modified != Modifier::None {
+                            any_modified = true;
+                        }
+                    }
+                    Err(e) => warn!(context, "Could not add recipient: {}", e),
+                }
+            }
+            if any_modified {
+                context.emit_event(EventType::ContactsChanged(None));
+            }
+        }
+        Err(e) => warn!(context, "Could not add recipients: {}", e),
+    };
+}
 
 #[cfg(test)]
 mod tests {
@@ -2313,7 +2404,7 @@ mod tests {
         assert_eq!(get_folder_meaning_by_name("SPAM"), FolderMeaning::Spam);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_uid_next_validity() {
         let t = TestContext::new_alice().await;
         assert_eq!(get_uid_next(&t.ctx, "Inbox").await.unwrap(), 0);
@@ -2495,7 +2586,7 @@ mod tests {
         ("Spam", true, true, "DeltaChat"),
     ];
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_target_folder_incoming_accepted() -> Result<()> {
         for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
             check_target_folder_combination(
@@ -2512,7 +2603,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_target_folder_incoming_request() -> Result<()> {
         for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_REQUEST {
             check_target_folder_combination(
@@ -2529,7 +2620,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_target_folder_outgoing() -> Result<()> {
         // Test outgoing emails
         for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
@@ -2547,7 +2638,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_target_folder_setupmsg() -> Result<()> {
         // Test setupmessages
         for (folder, mvbox_move, chat_msg, _expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
@@ -2565,7 +2656,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_imap_search_command() -> Result<()> {
         let t = TestContext::new_alice().await;
         assert_eq!(

src/imap/client.rs

@@ -8,10 +8,10 @@ use anyhow::{Context as _, Result};
 use async_imap::Client as ImapClient;
 
 use async_smtp::ServerAddress;
-use async_std::net::{self, TcpStream};
+use tokio::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::{dc_build_tls, Socks5Config};
+use crate::login_param::{build_tls, Socks5Config};
 
 use super::session::SessionStream;
 
@@ -67,7 +67,7 @@ impl Client {
         strict_tls: bool,
     ) -> Result<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(strict_tls);
+        let tls = build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> = Box::new(tls.connect(domain, stream).await?);
         let mut client = ImapClient::new(tls_stream);
 
@@ -108,7 +108,7 @@ impl Client {
                 .await?,
         );
 
-        let tls = dc_build_tls(strict_tls);
+        let tls = build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(target_addr.host.clone(), socks5_stream).await?);
         let mut client = ImapClient::new(tls_stream);
@@ -151,7 +151,7 @@ impl Client {
             Ok(self)
         } else {
             let Client { mut inner, .. } = self;
-            let tls = dc_build_tls(strict_tls);
+            let tls = build_tls(strict_tls);
             inner.run_command_and_check_ok("STARTTLS", None).await?;
 
             let stream = inner.into_inner();

src/imap/idle.rs

@@ -2,7 +2,7 @@ use super::Imap;
 
 use anyhow::{bail, Context as _, Result};
 use async_imap::extensions::idle::IdleResponse;
-use async_std::prelude::*;
+use futures_lite::FutureExt;
 use std::time::{Duration, SystemTime};
 
 use crate::{context::Context, scheduler::InterruptInfo};
@@ -87,9 +87,7 @@ impl Imap {
                 }
             }
 
-            let session = handle
-                .done()
-                .timeout(Duration::from_secs(15))
+            let session = tokio::time::timeout(Duration::from_secs(15), handle.done())
                 .await?
                 .context("IMAP IDLE protocol timed out")?;
             self.session = Some(Session { inner: session });
@@ -121,7 +119,7 @@ impl Imap {
 
         // check every minute if there are new messages
         // TODO: grow sleep durations / make them more flexible
-        let mut interval = async_std::stream::interval(Duration::from_secs(60));
+        let mut interval = tokio::time::interval(Duration::from_secs(60));
 
         enum Event {
             Tick,
@@ -131,7 +129,7 @@ impl Imap {
         let info = loop {
             use futures::future::FutureExt;
             match interval
-                .next()
+                .tick()
                 .map(|_| Event::Tick)
                 .race(
                     self.idle_interrupt

src/imap/scan_folders.rs

@@ -1,14 +1,13 @@
 use std::{collections::BTreeMap, time::Instant};
 
 use anyhow::{Context as _, Result};
+use futures::stream::StreamExt;
 
 use crate::config::Config;
 use crate::imap::Imap;
 use crate::log::LogExt;
 use crate::{context::Context, imap::FolderMeaning};
 
-use async_std::stream::StreamExt;
-
 use super::{get_folder_meaning, get_folder_meaning_by_name};
 
 impl Imap {
@@ -104,7 +103,7 @@ impl Imap {
         let list = session
             .list(Some(""), Some("*"))
             .await?
-            .filter_map(|f| f.ok_or_log_msg(context, "list_folders() can't get folder"));
+            .filter_map(|f| async { f.ok_or_log_msg(context, "list_folders() can't get folder") });
         Ok(list.collect().await)
     }
 }

src/imap/session.rs

@@ -2,8 +2,8 @@ use std::ops::{Deref, DerefMut};
 
 use async_imap::Session as ImapSession;
 use async_native_tls::TlsStream;
-use async_std::net::TcpStream;
 use fast_socks5::client::Socks5Stream;
+use tokio::net::TcpStream;
 
 #[derive(Debug)]
 pub(crate) struct Session {
@@ -11,7 +11,7 @@ pub(crate) struct Session {
 }
 
 pub(crate) trait SessionStream:
-    async_std::io::Read + async_std::io::Write + Unpin + Send + Sync + std::fmt::Debug
+    tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + Sync + std::fmt::Debug
 {
 }
 

src/imex.rs

@@ -2,26 +2,21 @@
 
 use std::any::Any;
 use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
 
 use ::pgp::types::KeyTrait;
 use anyhow::{bail, ensure, format_err, Context as _, Result};
-use async_std::{
-    fs::{self, File},
-    path::{Path, PathBuf},
-    prelude::*,
-};
-use async_tar::Archive;
+use futures::{StreamExt, TryStreamExt};
+use futures_lite::FutureExt;
 use rand::{thread_rng, Rng};
+use tokio::fs::{self, File};
+use tokio_tar::Archive;
 
 use crate::blob::BlobObject;
 use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
 use crate::config::Config;
 use crate::contact::ContactId;
 use crate::context::Context;
-use crate::dc_tools::{
-    dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
-    dc_open_file_std, dc_read_file, dc_write_file, time, EmailAddress,
-};
 use crate::e2ee;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
@@ -32,6 +27,10 @@ use crate::param::Param;
 use crate::pgp;
 use crate::sql;
 use crate::stock_str;
+use crate::tools::{
+    create_folder, delete_file, get_filesuffix_lc, open_file_std, read_file, time, write_file,
+    EmailAddress,
+};
 
 // Name of the database file in the backup.
 const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite";
@@ -59,7 +58,7 @@ pub enum ImexMode {
     ExportBackup = 11,
 
     /// `path` is the file (not: directory) to import. The file is normally
-    /// created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup
+    /// created by DC_IMEX_EXPORT_BACKUP and detected by imex_has_backup(). Importing a backup
     /// is only possible as long as the context is not configured or used in another way.
     ImportBackup = 12,
 }
@@ -86,62 +85,45 @@ pub async fn imex(
 ) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
 
-    let res = async {
-        let success = imex_inner(context, what, path, passphrase).await;
-        match success {
-            Ok(()) => {
-                info!(context, "IMEX successfully completed");
-                context.emit_event(EventType::ImexProgress(1000));
-                Ok(())
-            }
-            Err(err) => {
-                cleanup_aborted_imex(context, what).await;
-                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-                error!(context, "{:#}", err);
-                context.emit_event(EventType::ImexProgress(0));
-                bail!("IMEX FAILED to complete: {}", err);
-            }
-        }
-    }
-    .race(async {
-        cancel.recv().await.ok();
-        cleanup_aborted_imex(context, what).await;
-        Err(format_err!("canceled"))
-    })
-    .await;
+    let res = imex_inner(context, what, path, passphrase)
+        .race(async {
+            cancel.recv().await.ok();
+            Err(format_err!("canceled"))
+        })
+        .await;
 
     context.free_ongoing().await;
 
+    if let Err(err) = res.as_ref() {
+        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+        error!(context, "IMEX failed to complete: {:#}", err);
+        context.emit_event(EventType::ImexProgress(0));
+    } else {
+        info!(context, "IMEX successfully completed");
+        context.emit_event(EventType::ImexProgress(1000));
+    }
+
     res
 }
 
-async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
-    if what == ImexMode::ImportBackup {
-        dc_delete_file(context, context.get_dbfile()).await;
-        dc_delete_files_in_dir(context, context.get_blobdir()).await;
-    }
-}
-
 /// Returns the filename of the backup found (otherwise an error)
 pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result<String> {
-    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
+    let mut dir_iter = tokio::fs::read_dir(dir_name).await?;
     let mut newest_backup_name = "".to_string();
     let mut newest_backup_path: Option<PathBuf> = None;
 
-    while let Some(dirent) = dir_iter.next().await {
-        if let Ok(dirent) = dirent {
-            let path = dirent.path();
-            let name = dirent.file_name();
-            let name: String = name.to_string_lossy().into();
-            if name.starts_with("delta-chat")
-                && name.ends_with(".tar")
-                && (newest_backup_name.is_empty() || name > newest_backup_name)
-            {
-                // We just use string comparison to determine which backup is newer.
-                // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
-                newest_backup_path = Some(path);
-                newest_backup_name = name;
-            }
+    while let Ok(Some(dirent)) = dir_iter.next_entry().await {
+        let path = dirent.path();
+        let name = dirent.file_name();
+        let name: String = name.to_string_lossy().into();
+        if name.starts_with("delta-chat")
+            && name.ends_with(".tar")
+            && (newest_backup_name.is_empty() || name > newest_backup_name)
+        {
+            // We just use string comparison to determine which backup is newer.
+            // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
+            newest_backup_path = Some(path);
+            newest_backup_name = name;
         }
     }
 
@@ -192,7 +174,7 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
     info!(context, "Wait for setup message being sent ...",);
     while !context.shall_stop_ongoing().await {
-        async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
         if let Ok(msg) = Message::load_from_db(context, msg_id).await {
             if msg.is_sent() {
                 info!(context, "... setup message sent.",);
@@ -308,7 +290,7 @@ pub async fn continue_key_transfer(
     );
 
     if let Some(filename) = msg.get_file(context) {
-        let file = dc_open_file_std(context, filename)?;
+        let file = open_file_std(context, filename)?;
         let sc = normalize_setup_code(setup_code);
         let armored_key = decrypt_setup_file(&sc, file).await?;
         set_self_key(context, &armored_key, true, true).await?;
@@ -411,7 +393,7 @@ async fn imex_inner(
         if e2ee::ensure_secret_key_exists(context).await.is_err() {
             bail!("Cannot create private key or private key not available.");
         } else {
-            dc_create_folder(context, &path).await?;
+            create_folder(context, &path).await?;
         }
     }
 
@@ -461,7 +443,7 @@ async fn import_backup(
 
     context.sql.config_cache.write().await.clear();
 
-    let archive = Archive::new(backup_file);
+    let mut archive = Archive::new(backup_file);
 
     let mut entries = archive.entries()?;
     let mut last_progress = 0;
@@ -492,7 +474,7 @@ async fn import_backup(
             // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
             f.unpack_in(context.get_blobdir()).await?;
             let from_path = context.get_blobdir().join(f.path()?);
-            if from_path.is_file().await {
+            if from_path.is_file() {
                 if let Some(name) = from_path.file_name() {
                     fs::rename(&from_path, context.get_blobdir().join(name)).await?;
                 } else {
@@ -514,13 +496,10 @@ async fn import_backup(
 /// Returns Ok((temp_db_path, temp_path, dest_path)) on success. Unencrypted database can be
 /// written to temp_db_path. The backup can then be written to temp_path. If the backup succeeded,
 /// it can be renamed to dest_path. This guarantees that the backup is complete.
-async fn get_next_backup_path(
-    folder: &Path,
-    backup_time: i64,
-) -> Result<(PathBuf, PathBuf, PathBuf)> {
+fn get_next_backup_path(folder: &Path, backup_time: i64) -> Result<(PathBuf, PathBuf, PathBuf)> {
     let folder = PathBuf::from(folder);
     let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
-        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
+        // Don't change this file name format, in `dc_imex_has_backup` we use string comparison to determine which backup is newer:
         .format("delta-chat-backup-%Y-%m-%d")
         .to_string();
 
@@ -535,7 +514,7 @@ async fn get_next_backup_path(
         let mut destfile = folder.clone();
         destfile.push(format!("{}-{:02}.tar", stem, i));
 
-        if !tempdbfile.exists().await && !tempfile.exists().await && !destfile.exists().await {
+        if !tempdbfile.exists() && !tempfile.exists() && !destfile.exists() {
             return Ok((tempdbfile, tempfile, destfile));
         }
     }
@@ -545,7 +524,7 @@ async fn get_next_backup_path(
 async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
-    let (temp_db_path, temp_path, dest_path) = get_next_backup_path(dir, now).await?;
+    let (temp_db_path, temp_path, dest_path) = get_next_backup_path(dir, now)?;
     let _d1 = DeleteOnDrop(temp_db_path.clone());
     let _d2 = DeleteOnDrop(temp_path.clone());
 
@@ -598,8 +577,9 @@ struct DeleteOnDrop(PathBuf);
 impl Drop for DeleteOnDrop {
     fn drop(&mut self) {
         let file = self.0.clone();
-        // Not using dc_delete_file() here because it would send a DeletedBlobFile event
-        async_std::task::block_on(fs::remove_file(file)).ok();
+        // Not using `tools::delete_file` here because it would send a DeletedBlobFile event
+        // Hack to avoid panic in nested runtime calls of tokio
+        std::fs::remove_file(file).ok();
     }
 }
 
@@ -610,19 +590,21 @@ async fn export_backup_inner(
 ) -> Result<()> {
     let file = File::create(temp_path).await?;
 
-    let mut builder = async_tar::Builder::new(file);
+    let mut builder = tokio_tar::Builder::new(file);
 
     builder
         .append_path_with_name(temp_db_path, DBFILE_BACKUP_NAME)
         .await?;
 
-    let read_dir: Vec<_> = fs::read_dir(context.get_blobdir()).await?.collect().await;
+    let read_dir: Vec<_> =
+        tokio_stream::wrappers::ReadDirStream::new(fs::read_dir(context.get_blobdir()).await?)
+            .try_collect()
+            .await?;
     let count = read_dir.len();
     let mut written_files = 0;
 
     let mut last_progress = 0;
     for entry in read_dir.into_iter() {
-        let entry = entry?;
         let name = entry.file_name();
         if !entry.file_type().await?.is_file() {
             warn!(
@@ -663,12 +645,12 @@ async fn import_self_keys(context: &Context, dir: &Path) -> Result<()> {
     let mut imported_cnt = 0;
 
     let dir_name = dir.to_string_lossy();
-    let mut dir_handle = async_std::fs::read_dir(&dir).await?;
-    while let Some(entry) = dir_handle.next().await {
-        let entry_fn = entry?.file_name();
+    let mut dir_handle = tokio::fs::read_dir(&dir).await?;
+    while let Ok(Some(entry)) = dir_handle.next_entry().await {
+        let entry_fn = entry.file_name();
         let name_f = entry_fn.to_string_lossy();
         let path_plus_name = dir.join(&entry_fn);
-        match dc_get_filesuffix_lc(&name_f) {
+        match get_filesuffix_lc(&name_f) {
             Some(suffix) => {
                 if suffix != "asc" {
                     continue;
@@ -690,7 +672,7 @@ async fn import_self_keys(context: &Context, dir: &Path) -> Result<()> {
             path_plus_name.display()
         );
 
-        match dc_read_file(context, &path_plus_name).await {
+        match read_file(context, &path_plus_name).await {
             Ok(buf) => {
                 let armored = std::string::String::from_utf8_lossy(&buf);
                 if let Err(err) = set_self_key(context, &armored, set_default, false).await {
@@ -793,10 +775,10 @@ where
         key.key_id(),
         file_name.display()
     );
-    dc_delete_file(context, &file_name).await;
+    delete_file(context, &file_name).await;
 
     let content = key.to_asc(None).into_bytes();
-    let res = dc_write_file(context, &file_name, &content).await;
+    let res = write_file(context, &file_name, &content).await;
     if res.is_err() {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
@@ -815,7 +797,7 @@ mod tests {
 
     use ::pgp::armor::BlockType;
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_render_setup_file() {
         let t = TestContext::new_alice().await;
         let msg = render_setup_file(&t, "hello").await.unwrap();
@@ -832,7 +814,7 @@ mod tests {
         assert!(msg.contains("-----END PGP MESSAGE-----\n"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_render_setup_file_newline_replace() {
         let t = TestContext::new_alice().await;
         t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
@@ -843,7 +825,7 @@ mod tests {
         assert!(msg.contains("<p>hello<br>there</p>"));
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_create_setup_code() {
         let t = TestContext::new().await;
         let setupcode = create_setup_code(&t);
@@ -858,7 +840,7 @@ mod tests {
         assert_eq!(setupcode.chars().nth(39).unwrap(), '-');
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_public_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().public;
@@ -868,12 +850,12 @@ mod tests {
             .is_ok());
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
         let filename = format!("{}/public-key-default.asc", blobdir);
-        let bytes = async_std::fs::read(&filename).await.unwrap();
+        let bytes = tokio::fs::read(&filename).await.unwrap();
 
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_private_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().secret;
@@ -883,12 +865,12 @@ mod tests {
             .is_ok());
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
         let filename = format!("{}/private-key-default.asc", blobdir);
-        let bytes = async_std::fs::read(&filename).await.unwrap();
+        let bytes = tokio::fs::read(&filename).await.unwrap();
 
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_and_import_key() {
         let context = TestContext::new_alice().await;
         let blobdir = context.ctx.get_blobdir();
@@ -902,7 +884,7 @@ mod tests {
         }
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_and_import_backup() -> Result<()> {
         let backup_dir = tempfile::tempdir().unwrap();
 
@@ -911,26 +893,32 @@ mod tests {
 
         let context2 = TestContext::new().await;
         assert!(!context2.is_configured().await?);
-        assert!(has_backup(&context2, backup_dir.path().as_ref())
-            .await
-            .is_err());
+        assert!(has_backup(&context2, backup_dir.path()).await.is_err());
 
         // export from context1
-        assert!(imex(
-            &context1,
-            ImexMode::ExportBackup,
-            backup_dir.path().as_ref(),
-            None,
-        )
-        .await
-        .is_ok());
+        assert!(
+            imex(&context1, ImexMode::ExportBackup, backup_dir.path(), None)
+                .await
+                .is_ok()
+        );
         let _event = context1
             .evtracker
             .get_matching(|evt| matches!(evt, EventType::ImexProgress(1000)))
             .await;
 
         // import to context2
-        let backup = has_backup(&context2, backup_dir.path().as_ref()).await?;
+        let backup = has_backup(&context2, backup_dir.path()).await?;
+
+        // Import of unencrypted backup with incorrect "foobar" backup passphrase fails.
+        assert!(imex(
+            &context2,
+            ImexMode::ImportBackup,
+            backup.as_ref(),
+            Some("foobar".to_string())
+        )
+        .await
+        .is_err());
+
         assert!(
             imex(&context2, ImexMode::ImportBackup, backup.as_ref(), None)
                 .await
@@ -965,7 +953,7 @@ mod tests {
     const S_EM_SETUPCODE: &str = "1742-0185-6197-1303-7016-8412-3581-4441-0597";
     const S_EM_SETUPFILE: &str = include_str!("../test-data/message/stress.txt");
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_split_and_decrypt() {
         let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
         let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
@@ -988,20 +976,20 @@ mod tests {
         assert!(headers.get(HEADER_SETUPCODE).is_none());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_key_transfer() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
         let alice_clone = alice.clone();
-        let key_transfer_task = async_std::task::spawn(async move {
+        let key_transfer_task = tokio::task::spawn(async move {
             let ctx = alice_clone;
             initiate_key_transfer(&ctx).await
         });
 
         // Wait for the message to be added to the queue.
-        async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
         let sent = alice.pop_sent_msg().await;
-        let setup_code = key_transfer_task.await?;
+        let setup_code = key_transfer_task.await??;
 
         // Alice sets up a second device.
         let alice2 = TestContext::new().await;

src/job.rs

@@ -8,14 +8,11 @@ use anyhow::{Context as _, Result};
 use deltachat_derive::{FromSql, ToSql};
 use rand::{thread_rng, Rng};
 
-use crate::config::Config;
-use crate::contact::{normalize_name, Contact, Modifier, Origin};
 use crate::context::Context;
-use crate::dc_tools::time;
-use crate::events::EventType;
 use crate::imap::Imap;
 use crate::param::Params;
 use crate::scheduler::InterruptInfo;
+use crate::tools::time;
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
@@ -58,8 +55,6 @@ macro_rules! job_try {
 )]
 #[repr(u32)]
 pub enum Action {
-    FetchExistingMsgs = 110,
-
     // this is user initiated so it should have a fairly high priority
     UpdateRecentQuota = 140,
 
@@ -156,45 +151,6 @@ impl Job {
 
         Ok(())
     }
-
-    /// Read the recipients from old emails sent by the user and add them as contacts.
-    /// This way, we can already offer them some email addresses they can write to.
-    ///
-    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
-    /// and show them in the chat list.
-    async fn fetch_existing_msgs(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if job_try!(context.get_config_bool(Config::Bot).await) {
-            return Status::Finished(Ok(())); // Bots don't want those messages
-        }
-        if let Err(err) = imap.prepare(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
-        add_all_recipients_as_contacts(context, imap, Config::ConfiguredSentboxFolder).await;
-        add_all_recipients_as_contacts(context, imap, Config::ConfiguredMvboxFolder).await;
-        add_all_recipients_as_contacts(context, imap, Config::ConfiguredInboxFolder).await;
-
-        if job_try!(context.get_config_bool(Config::FetchExistingMsgs).await) {
-            for config in &[
-                Config::ConfiguredMvboxFolder,
-                Config::ConfiguredInboxFolder,
-                Config::ConfiguredSentboxFolder,
-            ] {
-                if let Some(folder) = job_try!(context.get_config(*config).await) {
-                    if let Err(e) = imap.fetch_new_messages(context, &folder, false, true).await {
-                        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-                        warn!(context, "Could not fetch messages, retrying: {:#}", e);
-                        return Status::RetryLater;
-                    };
-                }
-            }
-        }
-
-        info!(context, "Done fetching existing messages.");
-        Status::Finished(Ok(()))
-    }
-
     /// Synchronizes UIDs for all folders.
     async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.prepare(context).await {
@@ -250,51 +206,6 @@ pub async fn action_exists(context: &Context, action: Action) -> Result<bool> {
     Ok(exists)
 }
 
-async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
-    let mailbox = if let Ok(Some(m)) = context.get_config(folder).await {
-        m
-    } else {
-        return;
-    };
-    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
-        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-        warn!(context, "Could not select {}: {:#}", mailbox, e);
-        return;
-    }
-    match imap.get_all_recipients(context).await {
-        Ok(contacts) => {
-            let mut any_modified = false;
-            for contact in contacts {
-                let display_name_normalized = contact
-                    .display_name
-                    .as_ref()
-                    .map(|s| normalize_name(s))
-                    .unwrap_or_default();
-
-                match Contact::add_or_lookup(
-                    context,
-                    &display_name_normalized,
-                    &contact.addr,
-                    Origin::OutgoingTo,
-                )
-                .await
-                {
-                    Ok((_, modified)) => {
-                        if modified != Modifier::None {
-                            any_modified = true;
-                        }
-                    }
-                    Err(e) => warn!(context, "Could not add recipient: {}", e),
-                }
-            }
-            if any_modified {
-                context.emit_event(EventType::ContactsChanged(None));
-            }
-        }
-        Err(e) => warn!(context, "Could not add recipients: {}", e),
-    };
-}
-
 pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
 }
@@ -371,7 +282,6 @@ async fn perform_job_action(
 
     let try_res = match job.action {
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
-        Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
             Ok(status) => status,
             Err(err) => Status::Finished(Err(err)),
@@ -422,10 +332,7 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
 
     if delay_seconds == 0 {
         match action {
-            Action::ResyncFolders
-            | Action::FetchExistingMsgs
-            | Action::UpdateRecentQuota
-            | Action::DownloadMsg => {
+            Action::ResyncFolders | Action::UpdateRecentQuota | Action::DownloadMsg => {
                 info!(context, "interrupt: imap");
                 context.interrupt_inbox(InterruptInfo::new(false)).await;
             }
@@ -541,7 +448,7 @@ mod tests {
             .unwrap();
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_next_job_two() -> Result<()> {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
@@ -557,7 +464,7 @@ mod tests {
         Ok(())
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_next_job_one() -> Result<()> {
         let t = TestContext::new().await;
 

src/key.rs

@@ -11,11 +11,12 @@ use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use tokio::runtime::Handle;
 
 use crate::config::Config;
 use crate::constants::KeyGenType;
 use crate::context::Context;
-use crate::dc_tools::{time, EmailAddress};
+use crate::tools::{time, EmailAddress};
 
 // Re-export key types
 pub use crate::pgp::KeyPair;
@@ -39,7 +40,7 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     /// Create a key from a base64 string.
     fn from_base64(data: &str) -> Result<Self::KeyType> {
         // strip newlines and other whitespace
-        let cleaned: String = data.trim().split_whitespace().collect();
+        let cleaned: String = data.split_whitespace().collect();
         let bytes = base64::decode(cleaned.as_bytes())?;
         Self::from_slice(&bytes)
     }
@@ -219,9 +220,10 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await?)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
-            let keypair =
-                async_std::task::spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
-                    .await?;
+            let keypair = Handle::current()
+                .spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
+                .await??;
+
             store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
             info!(
                 context,
@@ -397,8 +399,8 @@ mod tests {
     use super::*;
     use crate::test_utils::{alice_keypair, TestContext};
 
-    use async_std::sync::Arc;
     use once_cell::sync::Lazy;
+    use std::sync::Arc;
 
     static KEYPAIR: Lazy<KeyPair> = Lazy::new(alice_keypair);
 
@@ -520,7 +522,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(key, key2);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
         let t = TestContext::new_alice().await;
@@ -530,7 +532,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(alice.secret, seckey);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_self_generate_public() {
         let t = TestContext::new().await;
         t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
@@ -540,7 +542,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert!(key.is_ok());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_self_generate_secret() {
         let t = TestContext::new().await;
         t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
@@ -550,7 +552,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert!(key.is_ok());
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_self_generate_concurrent() {
         use std::thread;
 
@@ -560,11 +562,19 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
             .unwrap();
         let thr0 = {
             let ctx = t.clone();
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+            thread::spawn(move || {
+                tokio::runtime::Runtime::new()
+                    .unwrap()
+                    .block_on(SignedPublicKey::load_self(&ctx))
+            })
         };
         let thr1 = {
             let ctx = t.clone();
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+            thread::spawn(move || {
+                tokio::runtime::Runtime::new()
+                    .unwrap()
+                    .block_on(SignedPublicKey::load_self(&ctx))
+            })
         };
         let res0 = thr0.join().unwrap();
         let res1 = thr1.join().unwrap();
@@ -577,7 +587,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(pubkey.primary_key, KEYPAIR.public.primary_key);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_save_self_key_twice() {
         // Saving the same key twice should result in only one row in
         // the keypairs table.

src/keyring.rs

@@ -76,7 +76,7 @@ mod tests {
         assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 
-    #[async_std::test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_keyring_load_self() {
         // new_self() implies load_self()
         let t = TestContext::new_alice().await;

src/lib.rs

@@ -1,20 +1,24 @@
 //! # Delta Chat Core Library.
 
+#![recursion_limit = "256"]
 #![forbid(unsafe_code)]
-#![deny(
+#![warn(
+    unused,
     clippy::correctness,
     missing_debug_implementations,
     clippy::all,
     clippy::indexing_slicing,
     clippy::wildcard_imports,
     clippy::needless_borrow,
-    clippy::cast_lossless
+    clippy::cast_lossless,
+    clippy::unused_async
 )]
 #![allow(
     clippy::match_bool,
     clippy::eval_order_dependence,
     clippy::bool_assert_comparison,
-    clippy::manual_split_once
+    clippy::manual_split_once,
+    clippy::format_push_string
 )]
 
 #[macro_use]
@@ -23,7 +27,6 @@ extern crate num_derive;
 extern crate smallvec;
 #[macro_use]
 extern crate rusqlite;
-extern crate strum;
 #[macro_use]
 extern crate strum_macros;
 
@@ -33,8 +36,6 @@ impl<T: rusqlite::ToSql + Send + Sync> ToSql for T {}
 
 #[macro_use]
 pub mod log;
-#[macro_use]
-pub mod error;
 
 #[cfg(feature = "internals")]
 #[macro_use]
@@ -94,18 +95,18 @@ mod dehtml;
 mod color;
 pub mod html;
 pub mod plaintext;
+mod ratelimit;
 pub mod summary;
 
-pub mod dc_receive_imf;
-pub mod dc_tools;
+pub mod receive_imf;
+pub mod tools;
 
 pub mod accounts;
 
 /// if set imap/incoming and smtp/outgoing MIME messages will be printed
 pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
 
-/// if set IMAP protocol commands and responses will be printed
-pub const DCC_IMAP_DEBUG: &str = "DCC_IMAP_DEBUG";
-
 #[cfg(test)]
 mod test_utils;
+#[cfg(test)]
+mod tests;