try running the upload script on python3.6

This should make colors used by Delta Chat for emails similar to colors
used by XMPP clients implementing the same specification[1], such as
Conversations and Snikket.

Previously Delta Chat used only hardcoded 16 colors, so this change
should also increase the number of colors and make it easier to
distinguish different contacts.

[1] https://xmpp.org/extensions/xep-0392.html

.circleci/config.yml

@@ -157,7 +157,7 @@ jobs:
       - attach_workspace:
           at: workspace
       - run: pyenv versions
-      - run: pyenv global 3.5.2
+      - run: pyenv global 3.6.0
       - run: ls -laR workspace
       - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 

.gitattributes

@@ -12,3 +12,6 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
+*.py diff=python
+*.rs diff=rust
+*.md diff=markdown

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.45.0
+          toolchain: 1.49.0
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: 1.45.0
+            toolchain: 1.49.0
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
@@ -40,6 +40,28 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --workspace --tests --examples
 
+  docs:
+    name: Rust doc comments
+    runs-on: ubuntu-latest
+    env:
+      RUSTDOCFLAGS: -Dwarnings
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - name: Install rust stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          components: rust-docs
+          override: true
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v1
+      - name: Rustdoc
+        uses: actions-rs/cargo@v1
+        with:
+          command: doc
+          args: --document-private-items --no-deps
 
   build_and_test:
     name: Build and test
@@ -50,7 +72,7 @@ jobs:
         # macOS disabled due to random failures related to caching
         #os: [ubuntu-latest, windows-latest, macOS-latest]
         os: [ubuntu-latest, windows-latest]
-        rust: [1.45.0]
+        rust: [1.49.0]
         experimental: [false]
 #        include:
 #          - os: ubuntu-latest

Cargo.toml

@@ -1,10 +1,13 @@
 [package]
 name = "deltachat"
-version = "1.50.0"
+version = "1.51.0-alpha.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
+[profile.dev]
+debug = 0
+
 [profile.release]
 lto = true
 
@@ -14,6 +17,7 @@ deltachat_derive = { path = "./deltachat_derive" }
 libc = "0.2.51"
 pgp = { version = "0.7.0", default-features = false } 
 hex = "0.4.0"
+sha-1 = "0.9.3"
 sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
@@ -25,7 +29,7 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.4.0"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.4", features = ["unstable"] }
+async-std = { version = "~1.8.0", features = ["unstable"] }
 base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -61,6 +65,7 @@ url = "2.1.1"
 async-std-resolver = "0.19.5"
 async-tar = "0.3.0"
 uuid = { version = "0.8", features = ["serde", "v4"] }
+rust-hsluv = "0.1.4"
 
 pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
@@ -78,6 +83,7 @@ proptest = "0.10"
 async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
 futures-lite = "1.7.0"
 criterion = "0.3"
+ansi_term = "0.12.0"
 
 [workspace]
 members = [

README.md

@@ -123,7 +123,7 @@ Language bindings are available for:
 - [C](https://c.delta.chat)
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
-- [Go](https://github.com/hugot/go-deltachat/)
+- [Go](https://github.com/deltachat/go-deltachat/)
 - [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.49.0-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share
+rm -rf /root/.rustup/toolchains/1.49.0-x86_64-unknown-linux-gnu/share

ci_scripts/remote_python_packaging.sh

@@ -1,7 +1,7 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}

ci_scripts/remote_tests_python.sh

@@ -1,7 +1,7 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}

ci_scripts/remote_tests_rust.sh

@@ -1,7 +1,7 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.50.0"
+version = "1.51.0-alpha.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/src/lib.rs

@@ -32,7 +32,7 @@ use deltachat::context::Context;
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
-use deltachat::stock::StockMessage;
+use deltachat::stock_str::StockMessage;
 use deltachat::*;
 
 mod dc_array;
@@ -296,16 +296,6 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     block_on(ctx.start_io())
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_is_io_running(context: *mut dc_context_t) -> libc::c_int {
-    if context.is_null() {
-        return 0;
-    }
-    let ctx = &*context;
-
-    block_on(ctx.is_io_running()) as libc::c_int
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_id(context: *mut dc_context_t) -> libc::c_int {
     if context.is_null() {
@@ -1146,10 +1136,15 @@ pub unsafe extern "C" fn dc_search_msgs(
         return ptr::null_mut();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
     block_on(async move {
         let arr = dc_array_t::from(
-            ctx.search_msgs(ChatId::new(chat_id), to_string_lossy(query))
+            ctx.search_msgs(chat_id, to_string_lossy(query))
                 .await
                 .iter()
                 .map(|msg_id| msg_id.to_u32())
@@ -1332,6 +1327,29 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_chat_encrinfo(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_chat_encrinfo()");
+        return "".strdup();
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        ChatId::new(chat_id)
+            .get_encryption_info(&ctx)
+            .await
+            .map(|s| s.strdup())
+            .unwrap_or_else(|e| {
+                error!(&ctx, "{}", e);
+                ptr::null_mut()
+            })
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_chat_ephemeral_timer(
     context: *mut dc_context_t,
@@ -1389,6 +1407,20 @@ pub unsafe extern "C" fn dc_get_msg_info(
     block_on(message::get_msg_info(&ctx, MsgId::new(msg_id))).strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_msg_html(
+    context: *mut dc_context_t,
+    msg_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_msg_html()");
+        return ptr::null_mut();
+    }
+    let ctx = &*context;
+
+    block_on(MsgId::new(msg_id).get_html(&ctx)).strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_mime_headers(
     context: *mut dc_context_t,
@@ -1535,6 +1567,9 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
         to_string_lossy(addr),
         Origin::IncomingReplyTo,
     ))
+    .ok()
+    .flatten()
+    .unwrap_or_default()
 }
 
 #[no_mangle]
@@ -1551,10 +1586,9 @@ pub unsafe extern "C" fn dc_create_contact(
     let name = to_string_lossy(name);
 
     block_on(async move {
-        match Contact::create(&ctx, name, to_string_lossy(addr)).await {
-            Ok(id) => id,
-            Err(_) => 0,
-        }
+        Contact::create(&ctx, name, to_string_lossy(addr))
+            .await
+            .unwrap_or(0)
     })
 }
 
@@ -1846,9 +1880,14 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
         return "".strdup();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
     block_on(async move {
-        securejoin::dc_get_securejoin_qr(&ctx, ChatId::new(chat_id))
+        securejoin::dc_get_securejoin_qr(&ctx, chat_id)
             .await
             .unwrap_or_else(|| "".to_string())
             .strdup()
@@ -1904,11 +1943,13 @@ pub unsafe extern "C" fn dc_is_sending_locations_to_chat(
         return 0;
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
-    block_on(location::is_sending_locations_to_chat(
-        &ctx,
-        ChatId::new(chat_id),
-    )) as libc::c_int
+    block_on(location::is_sending_locations_to_chat(&ctx, chat_id)) as libc::c_int
 }
 
 #[no_mangle]
@@ -1940,11 +1981,21 @@ pub unsafe extern "C" fn dc_get_locations(
         return ptr::null_mut();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
+    let contact_id = if contact_id == 0 {
+        None
+    } else {
+        Some(contact_id)
+    };
 
     block_on(async move {
         let res = location::get_range(
             &ctx,
-            ChatId::new(chat_id),
+            chat_id,
             contact_id,
             timestamp_begin as i64,
             timestamp_end as i64,
@@ -2562,6 +2613,16 @@ pub unsafe extern "C" fn dc_msg_get_chat_id(msg: *mut dc_msg_t) -> u32 {
     ffi_msg.message.get_chat_id().to_u32()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_real_chat_id(msg: *mut dc_msg_t) -> u32 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_real_chat_id()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_real_chat_id().to_u32()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_viewtype(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2724,7 +2785,7 @@ pub unsafe extern "C" fn dc_msg_get_ephemeral_timer(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_ephemeral_timer()
+    ffi_msg.message.get_ephemeral_timer().to_u32()
 }
 
 #[no_mangle]
@@ -2781,6 +2842,17 @@ pub unsafe extern "C" fn dc_msg_get_summarytext(
     .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_override_sender_name(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_override_sender_name()");
+        return "".strdup();
+    }
+    let ffi_msg = &mut *msg;
+
+    ffi_msg.message.get_override_sender_name().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_has_deviating_timestamp(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2861,6 +2933,16 @@ pub unsafe extern "C" fn dc_msg_is_setupmessage(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.is_setupmessage().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_has_html(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_has_html()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.has_html().into()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -2876,6 +2958,32 @@ pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut li
         .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_decide_on_contact_request(
+    context: *mut dc_context_t,
+    msg_id: u32,
+    decision: libc::c_int,
+) -> u32 {
+    if context.is_null() || msg_id <= constants::DC_MSG_ID_LAST_SPECIAL as u32 {
+        eprintln!("ignoring careless call to dc_decide_on_contact_request()");
+    }
+    let ctx = &*context;
+
+    match from_prim(decision) {
+        None => {
+            warn!(ctx, "{} is not a valid decision, ignoring", decision);
+            0
+        }
+        Some(d) => block_on(message::decide_on_contact_request(
+            ctx,
+            MsgId::new(msg_id),
+            d,
+        ))
+        .unwrap_or_default()
+        .to_u32(),
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_videochat_type(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2910,6 +3018,16 @@ pub unsafe extern "C" fn dc_msg_set_text(msg: *mut dc_msg_t, text: *const libc::
     ffi_msg.message.set_text(to_opt_string_lossy(text))
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_html(msg: *mut dc_msg_t, html: *const libc::c_char) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_html()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg.message.set_html(to_opt_string_lossy(html))
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_set_file(
     msg: *mut dc_msg_t,
@@ -3114,6 +3232,16 @@ pub unsafe extern "C" fn dc_contact_get_name(contact: *mut dc_contact_t) -> *mut
     ffi_contact.contact.get_name().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_auth_name(contact: *mut dc_contact_t) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_auth_name()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.get_authname().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_display_name(
     contact: *mut dc_contact_t,
@@ -3138,18 +3266,6 @@ pub unsafe extern "C" fn dc_contact_get_name_n_addr(
     ffi_contact.contact.get_name_n_addr().strdup()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_contact_get_first_name(
-    contact: *mut dc_contact_t,
-) -> *mut libc::c_char {
-    if contact.is_null() {
-        eprintln!("ignoring careless call to dc_contact_get_first_name()");
-        return "".strdup();
-    }
-    let ffi_contact = &*contact;
-    ffi_contact.contact.get_first_name().strdup()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_profile_image(
     contact: *mut dc_contact_t,
@@ -3181,6 +3297,16 @@ pub unsafe extern "C" fn dc_contact_get_color(contact: *mut dc_contact_t) -> u32
     ffi_contact.contact.get_color()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_status(contact: *mut dc_contact_t) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_status()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.get_status().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_is_blocked(contact: *mut dc_contact_t) -> libc::c_int {
     if contact.is_null() {
@@ -3358,7 +3484,7 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
         return ptr::null();
     }
     let addr = to_string_lossy(addr);
-    match provider::get_provider_info(addr.as_str()) {
+    match block_on(provider::get_provider_info(addr.as_str())) {
         Some(provider) => provider,
         None => ptr::null_mut(),
     }

examples/repl/cmdline.rs

@@ -2,7 +2,7 @@ extern crate dirs;
 
 use std::str::FromStr;
 
-use anyhow::{bail, ensure};
+use anyhow::{bail, ensure, Error};
 use async_std::path::Path;
 use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility, ProtectionStatus};
 use deltachat::chatlist::*;
@@ -11,7 +11,6 @@ use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
-use deltachat::error::Error;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::lot::LotState;
@@ -21,6 +20,7 @@ use deltachat::qr::*;
 use deltachat::sql;
 use deltachat::EventType;
 use deltachat::{config, provider};
+use std::fs;
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
@@ -185,7 +185,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -193,6 +193,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
+        if msg.has_html() { "[HAS-HTML]️" } else { "" },
         if msg.get_from_id() == 1 as libc::c_uint {
             ""
         } else if msg.get_state() == MessageState::InSeen {
@@ -368,9 +369,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  dellocations\n\
                  getlocations [<contact-id>]\n\
                  send <text>\n\
-                 send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
+                 sendhtml <file for html-part> [<text for plain-part>]\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -385,6 +386,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
+                 html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
@@ -400,6 +402,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  getqr [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
+                 setqr <qr-content>\n\
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
@@ -570,7 +573,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, ChatId::new(0)).await {
+            if location::is_sending_locations_to_chat(&context, None).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
@@ -732,7 +735,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 contacts.len(),
                 location::is_sending_locations_to_chat(
                     &context,
-                    sel_chat.as_ref().unwrap().get_id()
+                    Some(sel_chat.as_ref().unwrap().get_id())
                 )
                 .await,
             );
@@ -740,10 +743,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "getlocations" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
-            let contact_id = arg1.parse().unwrap_or_default();
+            let contact_id: Option<u32> = arg1.parse().ok();
             let locations = location::get_range(
                 &context,
-                sel_chat.as_ref().unwrap().get_id(),
+                Some(sel_chat.as_ref().unwrap().get_id()),
                 contact_id,
                 0,
                 0,
@@ -831,6 +834,22 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "sendhtml" => {
+            ensure!(sel_chat.is_some(), "No chat selected.");
+            ensure!(!arg1.is_empty(), "No html-file given.");
+            let path: &Path = arg1.as_ref();
+            let html = &*fs::read(&path)?;
+            let html = String::from_utf8_lossy(html);
+
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_html(Some(html.to_string()));
+            msg.set_text(Some(if arg2.is_empty() {
+                path.file_name().unwrap().to_string_lossy().to_string()
+            } else {
+                arg2.to_string()
+            }));
+            chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
+        }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -839,9 +858,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
             let chat = if let Some(ref sel_chat) = sel_chat {
-                sel_chat.get_id()
+                Some(sel_chat.get_id())
             } else {
-                ChatId::new(0)
+                None
             };
 
             let msglist = context.search_msgs(chat, arg1).await;
@@ -942,6 +961,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let res = message::get_msg_info(&context, id).await;
             println!("{}", res);
         }
+        "html" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            let file = dirs::home_dir()
+                .unwrap_or_default()
+                .join(format!("msg-{}.html", id.to_u32()));
+            let html = id.get_html(&context).await.unwrap_or_default();
+            fs::write(&file, html)?;
+            println!("HTML written to: {:#?}", file);
+        }
         "listfresh" => {
             let msglist = context.get_fresh_msgs().await;
 
@@ -1055,7 +1084,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            match provider::get_provider_info(arg1) {
+            match provider::get_provider_info(arg1).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);

examples/repl/main.rs

@@ -168,7 +168,7 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 27] = [
+const CHAT_COMMANDS: [&str; 28] = [
     "listchats",
     "listarchived",
     "chat",
@@ -188,6 +188,7 @@ const CHAT_COMMANDS: [&str; 27] = [
     "send",
     "sendimage",
     "sendfile",
+    "sendhtml",
     "videochat",
     "draft",
     "listmedia",
@@ -298,7 +299,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::sync::channel(100);
+    let (reader_s, reader_r) = async_std::channel::bounded(100);
     let input_loop = async_std::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
@@ -321,7 +322,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line));
+                    async_std::task::block_on(reader_s.send(line)).unwrap();
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
@@ -398,9 +399,8 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            if let Some(mut qr) =
-                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
-            {
+            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")

python/README.rst

@@ -11,10 +11,10 @@ Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
-yourself <sourceinstall>`_.
+without any "build-from-source" steps.
+Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -19,8 +19,7 @@ if __name__ == "__main__":
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
 
     if target == 'release':
-        extra = " -C lto=on -C embed-bitcode=yes"
-        os.environ["RUSTFLAGS"] = os.environ.get("RUSTFLAGS", "") + extra
+        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
 
     print("running:", " ".join(cmd))

python/src/deltachat/_build.py

@@ -145,6 +145,8 @@ def extract_defines(flags):
                 | DC_STR
                 | DC_CONTACT_ID
                 | DC_GCL
+                | DC_GCM
+                | DC_SOCKET
                 | DC_CHAT
                 | DC_PROVIDER
                 | DC_KEY_GEN

python/src/deltachat/account.py

@@ -459,8 +459,6 @@ class Account(object):
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
-        if not self.is_started():
-            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -579,12 +577,12 @@ class Account(object):
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
-    def configure(self):
+    def configure(self, reconfigure=False):
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
         """
-        assert not self.is_configured()
+        assert self.is_configured() == reconfigure
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
         configtracker = ConfigureTracker(self)
@@ -592,9 +590,6 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def is_started(self):
-        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
-
     def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
@@ -604,11 +599,8 @@ class Account(object):
         self.log("stop_ongoing")
         self.stop_ongoing()
 
-        if bool(lib.dc_is_io_running(self._dc_context)):
-            self.log("dc_stop_io (stop core IO scheduler)")
-            lib.dc_stop_io(self._dc_context)
-        else:
-            self.log("stop_scheduler called on non-running context")
+        self.log("dc_stop_io (stop core IO scheduler)")
+        lib.dc_stop_io(self._dc_context)
 
     def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove

python/src/deltachat/chat.py

@@ -167,6 +167,13 @@ class Chat(object):
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
+    def get_encryption_info(self):
+        """Return encryption info for this chat.
+
+        :returns: a string with encryption preferences of all chat members"""
+        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
+        return from_dc_charpointer(res)
+
     def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
@@ -495,18 +502,23 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
+                timestamp=datetime.utcfromtimestamp(
+                    lib.dc_array_get_timestamp(dc_array, i)
+                ),
+                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+            )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp):
+    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
         self.timestamp = timestamp
+        self.marker = marker
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__

python/src/deltachat/contact.py

@@ -77,6 +77,14 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
+    @property
+    def status(self):
+        """Get contact status.
+
+        :returns: contact status, empty string if it doesn't exist.
+        """
+        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
+
     def create_chat(self):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 

python/src/deltachat/direct_imap.py

@@ -9,7 +9,9 @@ import ssl
 import pathlib
 from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
+import imaplib
 import deltachat
+from deltachat import const
 
 
 SEEN = b'\\Seen'
@@ -24,13 +26,29 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    if not hasattr(account, "direct_imap"):
-        imap = DirectImap(account)
-        if imap.select_config_folder("mvbox"):
-            imap.delete(ALL, expunge=True)
-        assert imap.select_config_folder("inbox")
-        imap.delete(ALL, expunge=True)
-        setattr(account, "direct_imap", imap)
+    try:
+
+        if not hasattr(account, "direct_imap"):
+            imap = DirectImap(account)
+
+            for folder in imap.list_folders():
+                if folder.lower() == "inbox" or folder.lower() == "deltachat":
+                    assert imap.select_folder(folder)
+                    imap.delete(ALL, expunge=True)
+                else:
+                    imap.conn.delete_folder(folder)
+                    # We just deleted the folder, so we have to make DC forget about it, too
+                    if account.get_config("configured_sentbox_folder") == folder:
+                        account.set_config("configured_sentbox_folder", None)
+                    if account.get_config("configured_spam_folder") == folder:
+                        account.set_config("configured_spam_folder", None)
+
+            setattr(account, "direct_imap", imap)
+
+    except Exception as e:
+        # Uncaught exceptions here would lead to a timeout without any note written to the log
+        account.log("=============================== CAN'T RESET ACCOUNT: ===============================")
+        account.log("===================", e, "===================")
 
 
 @deltachat.global_hookimpl
@@ -50,18 +68,31 @@ class DirectImap:
         self.connect()
 
     def connect(self):
-        ssl_context = ssl.create_default_context()
-
-        # don't check if certificate hostname doesn't match target hostname
-        ssl_context.check_hostname = False
-
-        # don't check if the certificate is trusted by a certificate authority
-        ssl_context.verify_mode = ssl.CERT_NONE
-
         host = self.account.get_config("configured_mail_server")
+        port = int(self.account.get_config("configured_mail_port"))
+        security = int(self.account.get_config("configured_mail_security"))
+
         user = self.account.get_config("addr")
         pw = self.account.get_config("mail_pw")
-        self.conn = IMAPClient(host, ssl_context=ssl_context)
+
+        if security == const.DC_SOCKET_PLAIN:
+            ssl_context = None
+        else:
+            ssl_context = ssl.create_default_context()
+
+            # don't check if certificate hostname doesn't match target hostname
+            ssl_context.check_hostname = False
+
+            # don't check if the certificate is trusted by a certificate authority
+            ssl_context.verify_mode = ssl.CERT_NONE
+
+        if security == const.DC_SOCKET_STARTTLS:
+            self.conn = IMAPClient(host, port, ssl=False)
+            self.conn.starttls(ssl_context)
+        elif security == const.DC_SOCKET_PLAIN:
+            self.conn = IMAPClient(host, port, ssl=False)
+        elif security == const.DC_SOCKET_SSL:
+            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
@@ -76,6 +107,12 @@ class DirectImap:
         except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
+    def create_folder(self, foldername):
+        try:
+            self.conn.create_folder(foldername)
+        except imaplib.IMAP4.error as e:
+            print("Can't create", foldername, "probably it already exists:", str(e))
+
     def select_folder(self, foldername):
         assert not self._idling
         return self.conn.select_folder(foldername)
@@ -226,3 +263,9 @@ class DirectImap:
             res = self.conn.idle_done()
             self._idling = False
             return res
+
+    def append(self, folder, msg):
+        if msg.startswith("\n"):
+            msg = msg[1:]
+        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
+        self.conn.append(folder, msg)

python/src/deltachat/message.py

@@ -212,6 +212,11 @@ class Message(object):
                 return email.message_from_bytes(s)
             return email.message_from_string(s)
 
+    @property
+    def error(self):
+        """Error message"""
+        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
+
     @property
     def chat(self):
         """chat this message was posted in.

python/src/deltachat/testplugin.py

@@ -312,19 +312,19 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io([ac1])
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
             ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io([ac1, ac2])
             return ac1, ac2
 
         def get_many_online_accounts(self, num, move=True):
             accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io(accounts)
             for acc in accounts:
                 acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
@@ -356,23 +356,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac._configtracker = ac.configure()
             return ac
 
-        def wait_configure_and_start_io(self):
+        def wait_configure_and_start_io(self, accounts=None):
+            if accounts is None:
+                accounts = self._accounts[:]
             started_accounts = []
-            for acc in self._accounts:
-                if hasattr(acc, "_configtracker"):
-                    acc._configtracker.wait_finish()
-                    acc._evtracker.consume_events()
-                    acc.get_device_chat().mark_noticed()
-                    del acc._configtracker
-                acc.set_config("bcc_self", "0")
-                if acc.is_configured() and not acc.is_started():
-                    acc.start_io()
-                    started_accounts.append(acc)
-                print("{}: {} account was successfully setup".format(
-                    acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in accounts:
+                if acc not in started_accounts:
+                    self.wait_configure(acc)
+                    acc.set_config("bcc_self", "0")
+                    if acc.is_configured():
+                        acc.start_io()
+                        started_accounts.append(acc)
+                    print("{}: {} account was started".format(
+                        acc.get_config("displayname"), acc.get_config("addr")))
             for acc in started_accounts:
                 acc._evtracker.wait_all_initial_fetches()
 
+        def wait_configure(self, acc):
+            if hasattr(acc, "_configtracker"):
+                acc._configtracker.wait_finish()
+                acc._evtracker.consume_events()
+                acc.get_device_chat().mark_noticed()
+                del acc._configtracker
+
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 

python/src/deltachat/tracker.py

@@ -20,6 +20,16 @@ class ImexTracker:
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
             self._imex_events.put(ffi_event.data2)
 
+    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, int) and ev >= target_progress:
+                assert ev <= progress_upper_limit, \
+                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
+                return ev
+            if ev == 0:
+                return None
+
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
         files_written = []

python/tests/test_account.py

@@ -6,7 +6,10 @@ import queue
 import time
 from deltachat import const, Account
 from deltachat.message import Message
+from deltachat.tracker import ImexTracker
 from deltachat.hookspec import account_hookimpl
+from deltachat.capi import ffi, lib
+from deltachat.cutil import iter_array
 from datetime import datetime, timedelta
 
 
@@ -432,11 +435,11 @@ class TestOfflineChat:
         email = "hello <hello@example.org>"
         contact1 = ac1.create_contact(email)
         assert contact1.addr == "hello@example.org"
-        assert contact1.display_name == "hello"
+        assert contact1.name == "hello"
         contact1 = ac1.create_contact(email, name="world")
-        assert contact1.display_name == "world"
+        assert contact1.name == "world"
         contact2 = ac1.create_contact("display1 <x@example.org>", "real")
-        assert contact2.display_name == "real"
+        assert contact2.name == "real"
 
     def test_create_chat_simple(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
@@ -487,10 +490,6 @@ class TestOfflineChat:
         assert messages[0].text == "msg1"
         assert os.path.exists(messages[1].filename)
 
-    def test_ac_setup_message_fails(self, ac1):
-        with pytest.raises(RuntimeError):
-            ac1.initiate_key_transfer()
-
     def test_set_get_draft(self, chat1):
         msg = Message.new_empty(chat1.account, "text")
         msg1 = chat1.prepare_message(msg)
@@ -598,6 +597,28 @@ class TestOfflineChat:
         assert in_list[1][1] == chat
         assert in_list[1][2] == contacts[3]
 
+    def test_audit_log_view_without_daymarker(self, ac1, lp):
+        lp.sec("ac1: test audit log (show only system messages)")
+        chat = ac1.create_group_chat(name="audit log sample data")
+        # promote chat
+        chat.send_text("hello")
+        assert chat.is_promoted()
+
+        lp.sec("create test data")
+        chat.add_contact(ac1.create_contact("some-1@example.org"))
+        chat.set_name("audit log test group")
+        chat.send_text("a message in between")
+
+        lp.sec("check message count of all messages")
+        assert len(chat.get_messages()) == 4
+
+        lp.sec("check message count of only system messages (without daymarkers)")
+        dc_array = ffi.gc(
+            lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_INFO_ONLY, 0),
+            lib.dc_array_unref
+        )
+        assert len(list(iter_array(dc_array, lambda x: x))) == 2
+
 
 def test_basic_imap_api(acfactory, tmpdir):
     ac1, ac2 = acfactory.get_two_online_accounts()
@@ -969,6 +990,30 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
+    @pytest.mark.parametrize("mvbox_move", [True, False])
+    def test_markseen_message_and_mdn(self, acfactory, mvbox_move):
+        # Please only change this test if you are very sure that it will still catch the issues it catches now.
+        # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
+        ac1 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        acfactory.wait_configure_and_start_io()
+
+        acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
+        msg = ac2._evtracker.wait_next_incoming_message()
+
+        folder = "mvbox" if mvbox_move else "inbox"
+        ac1.direct_imap.select_config_folder(folder)
+        ac2.direct_imap.select_config_folder(folder)
+        ac1.direct_imap.idle_start()
+        ac2.direct_imap.idle_start()
+
+        ac2.mark_seen_messages([msg])
+
+        ac1.direct_imap.idle_wait_for_seen()  # Check that the mdn is marked as seen
+        ac2.direct_imap.idle_wait_for_seen()  # Check that the original message is marked as seen
+        ac1.direct_imap.idle_done()
+        ac2.direct_imap.idle_done()
+
     def test_reply_privately(self, acfactory):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -1072,6 +1117,64 @@ class TestOnlineAccount:
         assert not msg.is_encrypted()
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    def test_gossip_encryption_preference(self, acfactory, lp):
+        """Test that encryption preference of group members is gossiped to new members.
+        This is a Delta Chat extension to Autocrypt 1.1.0, which Autocrypt-Gossip headers
+        SHOULD NOT contain encryption preference.
+        """
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+
+        lp.sec("ac1 learns that ac2 prefers encryption")
+        ac1.create_chat(ac2)
+        msg = ac2.create_chat(ac1).send_text("first message")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "first message"
+        assert not msg.is_encrypted()
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config('addr'))
+        assert msg.chat.get_encryption_info() == res
+        lp.sec("ac2 learns that ac3 prefers encryption")
+        ac2.create_chat(ac3)
+        msg = ac3.create_chat(ac2).send_text("I prefer encryption")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "I prefer encryption"
+        assert not msg.is_encrypted()
+
+        lp.sec("ac3 does not know that ac1 prefers encryption")
+        ac1.create_chat(ac3)
+        chat = ac3.create_chat(ac1)
+        res = "{} No encryption.".format(ac1.get_config('addr'))
+        assert chat.get_encryption_info() == res
+        msg = chat.send_text("not encrypted")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "not encrypted"
+        assert not msg.is_encrypted()
+
+        lp.sec("ac1 creates a group chat with ac2")
+        group_chat = ac1.create_group_chat("hello")
+        group_chat.add_contact(ac2)
+        encryption_info = group_chat.get_encryption_info()
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config("addr"))
+        assert encryption_info == res
+        msg = group_chat.send_text("hi")
+
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+        assert msg.text == "hi"
+
+        lp.sec("ac2 adds ac3 to the group")
+        msg.chat.add_contact(ac3)
+        assert msg.is_encrypted()
+
+        lp.sec("ac3 learns that ac1 prefers encryption")
+        msg = ac3._evtracker.wait_next_incoming_message()
+        encryption_info = msg.chat.get_encryption_info().splitlines()
+        res = "{} End-to-end encryption preferred.".format(ac1.get_config("addr"))
+        assert res in encryption_info
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config("addr"))
+        assert res in encryption_info
+        msg = chat.send_text("encrypted")
+        assert msg.is_encrypted()
+
     def test_send_first_message_as_long_unicode_with_cr(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac2.set_config("save_mime_headers", "1")
@@ -1111,6 +1214,52 @@ class TestOnlineAccount:
         assert not device_chat.can_send()
         assert device_chat.get_draft() is None
 
+    def test_dont_show_emails_in_draft_folder(self, acfactory):
+        """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them.
+        So: If it's outgoing AND there is no Received header AND it's not in the sentbox, then ignore the email."""
+        ac1 = acfactory.get_online_configuring_account()
+        ac1.set_config("show_emails", "2")
+        ac1.create_contact("alice@example.com").create_chat()
+
+        acfactory.wait_configure(ac1)
+        ac1.direct_imap.create_folder("Drafts")
+        ac1.direct_imap.create_folder("Sent")
+
+        acfactory.wait_configure_and_start_io()
+        # Wait until each folder was selected once and we are IDLEing again:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.stop_io()
+
+        ac1.direct_imap.append("Drafts", """
+            From: ac1 <{}>
+            Subject: subj
+            To: alice@example.com
+            Message-ID: <aepiors@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            message in Drafts
+        """.format(ac1.get_config("configured_addr")))
+        ac1.direct_imap.append("Sent", """
+            From: ac1 <{}>
+            Subject: subj
+            To: alice@example.com
+            Message-ID: <hsabaeni@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            message in Sent
+        """.format(ac1.get_config("configured_addr")))
+
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+        ac1.start_io()
+
+        msg = ac1._evtracker.wait_next_messages_changed()
+
+        # Wait until each folder was scanned, this is necessary for this test to test what it should test:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+
+        assert msg.text == "subj – message in Sent"
+        assert len(msg.chat.get_messages()) == 1
+
     def test_prefer_encrypt(self, acfactory, lp):
         """Test quorum rule for encryption preference in 1:1 and group chat."""
         ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
@@ -1337,18 +1486,31 @@ class TestOnlineAccount:
         m = message_queue.get()
         assert m == msg_in
 
-    def test_import_export_online_all(self, acfactory, tmpdir, lp):
+    def test_import_export_online_all(self, acfactory, tmpdir, data, lp):
         ac1 = acfactory.get_one_online_account()
 
         lp.sec("create some chat content")
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact1.create_chat().send_text("msg1")
+        chat1 = ac1.create_contact("some1@example.org", name="some1").create_chat()
+        chat1.send_text("msg1")
         assert len(ac1.get_contacts(query="some1")) == 1
+
+        original_image_path = data.get_path("d.png")
+        chat1.send_image(original_image_path)
+
         backupdir = tmpdir.mkdir("backup")
 
         lp.sec("export all to {}".format(backupdir))
-        path = ac1.export_all(backupdir.strpath)
-        assert os.path.exists(path)
+        with ac1.temp_plugin(ImexTracker()) as imex_tracker:
+            path = ac1.export_all(backupdir.strpath)
+            assert os.path.exists(path)
+
+            # check progress events for export
+            assert imex_tracker.wait_progress(1, progress_upper_limit=249)
+            assert imex_tracker.wait_progress(250, progress_upper_limit=499)
+            assert imex_tracker.wait_progress(500, progress_upper_limit=749)
+            assert imex_tracker.wait_progress(750, progress_upper_limit=999)
+            assert imex_tracker.wait_progress(1000)
+
         t = time.time()
 
         lp.sec("get fresh empty account")
@@ -1359,15 +1521,25 @@ class TestOnlineAccount:
         assert path2 == path
 
         lp.sec("import backup and check it's proper")
-        ac2.import_all(path)
+        with ac2.temp_plugin(ImexTracker()) as imex_tracker:
+            ac2.import_all(path)
+
+            # check progress events for import
+            assert imex_tracker.wait_progress(1, progress_upper_limit=249)
+            assert imex_tracker.wait_progress(500, progress_upper_limit=749)
+            assert imex_tracker.wait_progress(750, progress_upper_limit=999)
+            assert imex_tracker.wait_progress(1000)
+
         contacts = ac2.get_contacts(query="some1")
         assert len(contacts) == 1
         contact2 = contacts[0]
         assert contact2.addr == "some1@example.org"
         chat2 = contact2.create_chat()
         messages = chat2.get_messages()
-        assert len(messages) == 1
+        assert len(messages) == 2
         assert messages[0].text == "msg1"
+        assert messages[1].filemime == "image/png"
+        assert os.stat(messages[1].filename).st_size == os.stat(original_image_path).st_size
 
         # wait until a second passed since last backup
         # because get_latest_backupfile() shall return the latest backup
@@ -1708,7 +1880,7 @@ class TestOnlineAccount:
 
         ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5)
         ac1._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED")
-        chat1.send_text("hello")
+        chat1.send_text("🍞")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
         lp.sec("ac2: wait for incoming location message")
@@ -1722,6 +1894,7 @@ class TestOnlineAccount:
         assert locations[0].longitude == 3.0
         assert locations[0].accuracy == 0.5
         assert locations[0].timestamp > now
+        assert locations[0].marker == "🍞"
 
         contact = ac2.create_contact(ac1)
         locations2 = chat2.get_locations(contact=contact)
@@ -1773,6 +1946,7 @@ class TestOnlineAccount:
         # Error message should be assigned to the chat with ac1.
         lp.sec("ac4: checking that message is assigned to the sender chat")
         error_msg = ac4._evtracker.wait_next_incoming_message()
+        assert error_msg.error  # There is an error decrypting the message
         assert error_msg.chat == chat41
 
         lp.sec("ac2: sending a reply to the chat")
@@ -1783,6 +1957,7 @@ class TestOnlineAccount:
 
         lp.sec("ac4: checking that reply is assigned to ac2 chat")
         error_reply = ac4._evtracker.wait_next_incoming_message()
+        assert error_reply.error  # There is an error decrypting the message
         assert error_reply.chat == chat42
 
         # Test that ac4 replies to error messages don't appear in the
@@ -1794,11 +1969,13 @@ class TestOnlineAccount:
         chat42.send_text("I can't decrypt your message, ac2!")
 
         msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.error is None
         assert msg.text == "I can't decrypt your message, ac1!"
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac1.create_chat(ac3)
 
         msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.error is None
         assert msg.text == "I can't decrypt your message, ac2!"
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac2.create_chat(ac4)
@@ -1959,7 +2136,9 @@ class TestOnlineAccount:
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac1.set_config("displayname", "Account 1")
 
-        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        # Similar to acfactory.get_accepted_chat, but without setting the contact name.
+        ac2.create_contact(ac1.get_config("addr")).create_chat()
+        chat12 = ac1.create_contact(ac2.get_config("addr")).create_chat()
         contact = None
 
         def update_name():
@@ -1990,12 +2169,24 @@ class TestOnlineAccount:
         # so it should not be changed.
         ac1.set_config("displayname", "Renamed again")
         updated_name = update_name()
-        if updated_name == "Renamed again":
-            # Known bug, mark as XFAIL
-            pytest.xfail("Contact was renamed after explicit rename")
-        else:
-            # No renames should happen after explicit rename
-            assert updated_name == "Renamed"
+        assert updated_name == "Renamed"
+
+    def test_status(self, acfactory):
+        """Test that status is transferred over the network."""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        ac1.set_config("selfstatus", "New status")
+        chat12.send_text("hi")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hi"
+        assert msg.get_sender_contact().status == "New status"
+
+        ac1.set_config("selfstatus", "")
+        chat12.send_text("hello")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+        assert msg.get_sender_contact().status == ""
 
     def test_group_quote(self, acfactory, lp):
         """Test quoting in a group with a new member who have not seen the quoted message."""
@@ -2036,26 +2227,83 @@ class TestOnlineAccount:
         assert received_reply.quoted_text == "hello"
         assert received_reply.quote.id == out_msg.id
 
+    @pytest.mark.parametrize("folder,move,expected_destination,", [
+        ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
+        ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
+        ("Spam", False, "INBOX")  # ...emails are moved from the spam folder to the Inbox
+    ])
+    # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
+    # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
+    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination):
+        """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
+        variant = folder + "-" + str(move) + "-" + expected_destination
+        lp.sec("Testing variant " + variant)
+        ac1 = acfactory.get_online_configuring_account(move=move)
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure(ac1)
+        ac1.direct_imap.create_folder(folder)
+
+        acfactory.wait_configure_and_start_io()
+        # Wait until each folder was selected once and we are IDLEing:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.stop_io()
+
+        # Send a message to ac1 and move it to the mvbox:
+        ac1.direct_imap.select_config_folder("inbox")
+        ac1.direct_imap.idle_start()
+        acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
+
+        lp.sec("Everything prepared, now see if DeltaChat finds the message (" + variant + ")")
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+        ac1.start_io()
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        # Wait until the message was moved (if at all) and we are IDLEing again:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.direct_imap.select_folder(expected_destination)
+        assert len(ac1.direct_imap.get_all_messages()) == 1
+        if folder != expected_destination:
+            ac1.direct_imap.select_folder(folder)
+            assert len(ac1.direct_imap.get_all_messages()) == 0
+
     @pytest.mark.parametrize("mvbox_move", [False, True])
-    def test_add_all_recipients_as_contacts(self, acfactory, lp, mvbox_move):
+    def test_fetch_existing(self, acfactory, lp, mvbox_move):
         """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
         This way, we can already offer them some email addresses they can write to.
 
-        Also test that existing emails are fetched during onboarding.
+        Also, the newest existing emails from each folder are fetched during onboarding.
 
-        Lastly, tests that bcc_self messages moved to the mvbox are marked as read."""
+        Additionally tests that bcc_self messages moved to the mvbox/sentbox are marked as read."""
         ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
+        ac1.set_config("sentbox_move", "1")
         ac2 = acfactory.get_online_configuring_account()
 
+        acfactory.wait_configure(ac1)
+
+        ac1.direct_imap.create_folder("Sent")
+        ac1.set_config("sentbox_watch", "1")
+
+        # We need to reconfigure to find the new "Sent" folder.
+        # `scan_folders()`, which runs automatically shortly after `start_io()` is invoked,
+        # would also find the "Sent" folder, but it would be too late:
+        # The sentbox thread, started by `start_io()`, would have seen that there is no
+        # ConfiguredSentboxFolder and do nothing.
+        ac1._configtracker = ac1.configure(reconfigure=True)
         acfactory.wait_configure_and_start_io()
 
-        chat = acfactory.get_accepted_chat(ac1, ac2)
-
-        lp.sec("send out message with bcc to ourselves")
         if mvbox_move:
             ac1.direct_imap.select_config_folder("mvbox")
+        else:
+            ac1.direct_imap.select_config_folder("sentbox")
         ac1.direct_imap.idle_start()
+
+        lp.sec("send out message with bcc to ourselves")
         ac1.set_config("bcc_self", "1")
+        chat = acfactory.get_accepted_chat(ac1, ac2)
         chat.send_text("message text")
 
         # now wait until the bcc_self message arrives
@@ -2074,6 +2322,53 @@ class TestOnlineAccount:
         msg = ac1_clone._evtracker.wait_next_messages_changed()
         assert msg.text == "message text"
 
+    def test_fetch_existing_msgs_group_and_single(self, acfactory, lp):
+        """There was a bug concerning fetch-existing-msgs:
+
+        A sent a message to you, adding you to a group. This created a contact request.
+        You wrote a message to A, creating a chat.
+        ...but the group stayed blocked.
+        So, after fetch-existing-msgs you have one contact request and one chat with the same person.
+
+        See https://github.com/deltachat/deltachat-core-rust/issues/2097"""
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure_and_start_io()
+
+        lp.sec("receive a message")
+        ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
+        ac1._evtracker.wait_next_messages_changed()
+
+        lp.sec("send out message with bcc to ourselves")
+        ac1.direct_imap.idle_start()
+        ac1.set_config("bcc_self", "1")
+        ac1.create_chat(ac2).send_text("outgoing, encrypted direct message, creating a chat")
+
+        # now wait until the bcc_self message arrives
+        assert ac1.direct_imap.idle_wait_for_seen()
+
+        lp.sec("Clone online account and let it fetch the existing messages")
+        ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone.set_config("fetch_existing_msgs", "1")
+        ac1_clone._configtracker.wait_finish()
+
+        ac1_clone.start_io()
+        ac1_clone._evtracker.wait_all_initial_fetches()
+        chats = ac1_clone.get_chats()
+        assert len(chats) == 4  # two newly created chats + self-chat + device-chat
+        group_chat = [c for c in chats if c.get_name() == "group name"][0]
+        assert group_chat.is_group()
+        private_chat = [c for c in chats if c.get_name() == "ac2"][0]
+        assert not private_chat.is_group()
+
+        group_messages = group_chat.get_messages()
+        assert len(group_messages) == 1
+        assert group_messages[0].text == "incoming, unencrypted group message"
+        private_messages = private_chat.get_messages()
+        # We can't decrypt the message in this chat, so the chat is empty:
+        assert len(private_messages) == 0
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):

rust-toolchain

@@ -1 +1 @@
-1.45.0
+1.49.0

set_core_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys

src/accounts.rs

@@ -6,11 +6,10 @@ use async_std::prelude::*;
 use async_std::sync::{Arc, RwLock};
 use uuid::Uuid;
 
-use anyhow::{ensure, Context as _};
+use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
-use crate::error::Result;
 use crate::events::Event;
 
 /// Account manager, that can handle multiple accounts in a single place.

src/aheader.rs

@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
-use crate::contact::*;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};

src/blob.rs

@@ -7,14 +7,17 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::Error;
 use image::GenericImageView;
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::{
+    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
+    WORSE_IMAGE_SIZE,
+};
 use crate::context::Context;
-use crate::error::Error;
 use crate::events::EventType;
 use crate::message;
 
@@ -377,27 +380,18 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
         let blob_abs = self.to_abs_path();
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
-        })?;
 
-        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
-            return Ok(());
-        }
+        let img_wh =
+            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+                .unwrap_or_default()
+            {
+                MediaQuality::Balanced => BALANCED_AVATAR_SIZE,
+                MediaQuality::Worse => WORSE_AVATAR_SIZE,
+            };
 
-        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
-        Ok(())
+        self.recode_to_size(context, blob_abs, img_wh).await
     }
 
     pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
@@ -408,39 +402,54 @@ impl<'a> BlobObject<'a> {
             return Ok(());
         }
 
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+        let img_wh =
+            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+                .unwrap_or_default()
+            {
+                MediaQuality::Balanced => BALANCED_IMAGE_SIZE,
+                MediaQuality::Worse => WORSE_IMAGE_SIZE,
+            };
+
+        self.recode_to_size(context, blob_abs, img_wh).await
+    }
+
+    async fn recode_to_size(
+        &self,
+        context: &Context,
+        blob_abs: PathBuf,
+        img_wh: u32,
+    ) -> Result<(), BlobError> {
+        let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
+        let orientation = self.get_exif_orientation(context);
 
-        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
-            .unwrap_or_default()
-            == MediaQuality::Balanced
-        {
-            BALANCED_IMAGE_SIZE
-        } else {
-            WORSE_IMAGE_SIZE
-        };
+        let do_scale = img.width() > img_wh || img.height() > img_wh;
+        let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
 
-        if img.width() <= img_wh && img.height() <= img_wh {
-            return Ok(());
+        if do_scale || do_rotate {
+            if do_scale {
+                img = img.thumbnail(img_wh, img_wh);
+            }
+
+            if do_rotate {
+                img = match orientation {
+                    Ok(90) => img.rotate90(),
+                    Ok(180) => img.rotate180(),
+                    Ok(270) => img.rotate270(),
+                    _ => img,
+                }
+            }
+
+            img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+                blobdir: context.get_blobdir().to_path_buf(),
+                blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+                cause: err.into(),
+            })?;
         }
 
-        let mut img = img.thumbnail(img_wh, img_wh);
-        match self.get_exif_orientation(context) {
-            Ok(90) => img = img.rotate90(),
-            Ok(180) => img = img.rotate180(),
-            Ok(270) => img = img.rotate270(),
-            _ => {}
-        }
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
         Ok(())
     }
 
@@ -511,69 +520,57 @@ pub enum BlobError {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_create() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
-        let fname = t.ctx.get_blobdir().join("foo");
+        let blob = BlobObject::create(&t, "foo", b"hello").await.unwrap();
+        let fname = t.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
         assert_eq!(data, b"hello");
         assert_eq!(blob.as_name(), "$BLOBDIR/foo");
-        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
+        assert_eq!(blob.to_abs_path(), t.get_blobdir().join("foo"));
     }
 
     #[async_std::test]
     async fn test_lowercase_ext() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.TXT", b"hello").await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_file_name() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_rel_path() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
     #[async_std::test]
     async fn test_suffix() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.suffix(), Some("txt"));
-        let blob = BlobObject::create(&t.ctx, "bar", b"world").await.unwrap();
+        let blob = BlobObject::create(&t, "bar", b"world").await.unwrap();
         assert_eq!(blob.suffix(), None);
     }
 
     #[async_std::test]
     async fn test_create_dup() {
         let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
-        let foo_path = t.ctx.get_blobdir().join("foo.txt");
+        BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let foo_path = t.get_blobdir().join("foo.txt");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.txt", b"world")
-            .await
-            .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
+        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -589,15 +586,15 @@ mod tests {
     #[async_std::test]
     async fn test_double_ext_preserved() {
         let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
+        BlobObject::create(&t, "foo.tar.gz", b"hello")
             .await
             .unwrap();
-        let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
+        let foo_path = t.get_blobdir().join("foo.tar.gz");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"world")
+        BlobObject::create(&t, "foo.tar.gz", b"world")
             .await
             .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -615,7 +612,7 @@ mod tests {
     async fn test_create_long_names() {
         let t = TestContext::new().await;
         let s = "1".repeat(150);
-        let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
+        let blob = BlobObject::create(&t, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
         assert!(blobname.len() < 128);
     }
@@ -625,14 +622,14 @@ mod tests {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
+        let blob = BlobObject::create_and_copy(&t, &src).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).await.is_err());
-        let whoops = t.ctx.get_blobdir().join("whoops");
+        assert!(BlobObject::create_and_copy(&t, &whoops).await.is_err());
+        let whoops = t.get_blobdir().join("whoops");
         assert!(!whoops.exists().await);
     }
 
@@ -642,14 +639,14 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
-        let src_int = t.ctx.get_blobdir().join("internal");
+        let src_int = t.get_blobdir().join("internal");
         fs::write(&src_int, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_int).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_int).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
@@ -659,7 +656,7 @@ mod tests {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"

src/chatlist.rs

@@ -1,15 +1,20 @@
 //! # Chat list module
 
+use anyhow::{bail, ensure, Result};
+
 use crate::chat;
-use crate::chat::*;
-use crate::constants::*;
-use crate::contact::*;
-use crate::context::*;
+use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
+use crate::constants::{
+    Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_DEADDROP,
+    DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT,
+    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+};
+use crate::contact::Contact;
+use crate::context::Context;
 use crate::ephemeral::delete_expired_messages;
-use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 /// An object representing a single chatlist in memory.
 ///
@@ -58,9 +63,8 @@ impl Chatlist {
     ///   messages from addresses that have no relationship to the configured account.
     ///   The last of these messages is represented by DC_CHAT_ID_DEADDROP and you can retrieve details
     ///   about it with chatlist.get_msg_id(). Typically, the UI asks the user "Do you want to chat with NAME?"
-    ///   and offers the options "Yes" (call dc_create_chat_by_msg_id()), "Never" (call dc_block_contact())
-    ///   or "Not now".
-    ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
+    ///   and offers the options "Start chat", "Block" and "Not now";
+    ///   The decision should be passed to dc_decide_on_contact_request().
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
     ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
@@ -359,28 +363,29 @@ impl Chatlist {
             return ret;
         };
 
-        let mut lastcontact = None;
-
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
-            if lastmsg.from_id != DC_CONTACT_ID_SELF && chat.typ == Chattype::Group {
-                lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
-            }
-
-            Some(lastmsg)
-        } else {
-            None
-        };
+        let (lastmsg, lastcontact) =
+            if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
+                if lastmsg.from_id == DC_CONTACT_ID_SELF {
+                    (Some(lastmsg), None)
+                } else {
+                    match chat.typ {
+                        Chattype::Group | Chattype::Mailinglist => {
+                            let lastcontact =
+                                Contact::load_from_db(context, lastmsg.from_id).await.ok();
+                            (Some(lastmsg), lastcontact)
+                        }
+                        Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
+                    }
+                }
+            } else {
+                (None, None)
+            };
 
         if chat.id.is_archived_link() {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.text2 = Some(
-                context
-                    .stock_str(StockMessage::NoMessages)
-                    .await
-                    .to_string(),
-            );
+            ret.text2 = Some(stock_str::no_messages(context).await);
         } else {
             ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
                 .await;
@@ -433,23 +438,26 @@ async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::chat::{create_group_chat, ProtectionStatus};
+    use crate::constants::Viewtype;
+    use crate::stock_str::StockMessage;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_try_load() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "b chat")
+        let chat_id2 = create_group_chat(&t, ProtectionStatus::Unprotected, "b chat")
             .await
             .unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "c chat")
+        let chat_id3 = create_group_chat(&t, ProtectionStatus::Unprotected, "c chat")
             .await
             .unwrap();
 
         // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
         assert_eq!(chats.get_chat_id(0), chat_id3);
         assert_eq!(chats.get_chat_id(1), chat_id2);
@@ -458,26 +466,24 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t.ctx, Some(&mut msg)).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        chat_id2.set_draft(&t, Some(&mut msg)).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None)
-            .await
-            .unwrap();
+        let chats = Chatlist::try_load(&t, 0, Some("b"), None).await.unwrap();
         assert_eq!(chats.len(), 1);
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
         chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .set_visibility(&t, ChatVisibility::Archived)
             .await
             .ok();
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
@@ -486,23 +492,23 @@ mod tests {
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
         let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
-        create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
+        t.update_device_chats().await.unwrap();
+        create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_FOR_FORWARDING, None, None)
             .await
             .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+        assert!(Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -511,31 +517,29 @@ mod tests {
     #[async_std::test]
     async fn test_search_special_chat_names() {
         let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
+        t.update_device_chats().await.unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
-        t.ctx
-            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+        t.set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
 
-        t.ctx
-            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+        t.set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
@@ -544,16 +548,16 @@ mod tests {
     #[async_std::test]
     async fn test_get_summary_unwrap() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t.ctx, Some(&mut msg)).await;
+        chat_id1.set_draft(&t, Some(&mut msg)).await;
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        let summary = chats.get_summary(&t.ctx, 0, None).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let summary = chats.get_summary(&t, 0, None).await;
         assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/color.rs

@@ -0,0 +1,46 @@
+//! Implementation of Consistent Color Generation
+//!
+//! Consistent Color Generation is defined in XEP-0392.
+//!
+//! Color Vision Deficiency correction is not implemented as Delta Chat does not offer
+//! corresponding settings.
+use hsluv::hsluv_to_rgb;
+use sha1::{Digest, Sha1};
+
+/// Converts an identifier to Hue angle.
+fn str_to_angle(s: impl AsRef<str>) -> f64 {
+    let bytes = s.as_ref().as_bytes();
+    let result = Sha1::digest(bytes);
+    let checksum: u16 = result.get(0).map_or(0, |&x| u16::from(x))
+        + 256 * result.get(1).map_or(0, |&x| u16::from(x));
+    f64::from(checksum) / 65536.0 * 360.0
+}
+
+/// Converts an identifier to RGB color.
+///
+/// Returns a 24-bit number with 8 least significant bits corresponding to the blue color and 8
+/// most significant bits corresponding to the red color.
+///
+/// Saturation is set to maximum (100.0) to make colors distinguishable, and lightness is set to
+/// half (50.0) to make colors suitable both for light and dark theme.
+pub(crate) fn str_to_color(s: impl AsRef<str>) -> u32 {
+    let (r, g, b) = hsluv_to_rgb((str_to_angle(s), 100.0, 50.0));
+    65536 * (r * 256.0) as u32 + 256 * (g * 256.0) as u32 + (b * 256.0) as u32
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[allow(clippy::float_cmp)]
+    #[test]
+    fn test_str_to_angle() {
+        // Test against test vectors from
+        // https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd
+        assert!((str_to_angle("Romeo") - 327.255249).abs() < 1e-6);
+        assert!((str_to_angle("juliet@capulet.lit") - 209.410400).abs() < 1e-6);
+        assert!((str_to_angle("😺") - 331.199341).abs() < 1e-6);
+        assert!((str_to_angle("council") - 359.994507).abs() < 1e-6);
+        assert!((str_to_angle("Board") - 171.430664).abs() < 1e-6);
+    }
+}

src/config.rs

@@ -7,12 +7,13 @@ use crate::blob::BlobObject;
 use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
 use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
-use crate::stock::StockMessage;
+use crate::provider::{get_provider_by_id, Provider};
+use crate::stock_str;
 
 /// The available configuration keys.
 #[derive(
@@ -35,9 +36,6 @@ pub enum Config {
     SmtpCertificateChecks,
     ServerFlags,
 
-    #[strum(props(default = "INBOX"))]
-    ImapFolder,
-
     Displayname,
     Selfstatus,
     Selfavatar,
@@ -63,6 +61,9 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MvboxMove,
 
+    #[strum(props(default = "0"))]
+    SentboxMove, // If `MvboxMove` is true, this config is ignored. Currently only used in tests.
+
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
@@ -72,8 +73,7 @@ pub enum Config {
     /// If set to "1", on the first time `start_io()` is called after configuring,
     /// the newest existing messages are fetched.
     /// Existing recipients are added to the contact database regardless of this setting.
-    #[strum(props(default = "0"))]
-    // disabled for now, we'll set this back to "1" at some point
+    #[strum(props(default = "1"))]
     FetchExistingMsgs,
 
     #[strum(props(default = "0"))]
@@ -117,6 +117,9 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
+    ConfiguredSpamFolder,
+    ConfiguredTimestamp,
+    ConfiguredProvider,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -137,6 +140,13 @@ pub enum Config {
 
     /// address to webrtc instance to use for videochats
     WebrtcInstance,
+
+    /// Timestamp of the last time housekeeping was run
+    LastHousekeeping,
+
+    /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
+    #[strum(props(default = "60"))]
+    ScanAllFoldersDebounceSecs,
 }
 
 impl Context {
@@ -163,7 +173,7 @@ impl Context {
 
         // Default values
         match key {
-            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::Selfstatus => Some(stock_str::status_line(self).await),
             Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
@@ -176,6 +186,20 @@ impl Context {
             .unwrap_or_default()
     }
 
+    pub async fn get_config_i64(&self, key: Config) -> i64 {
+        self.get_config(key)
+            .await
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
+    }
+
+    pub async fn get_config_u64(&self, key: Config) -> u64 {
+        self.get_config(key)
+            .await
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
+    }
+
     pub async fn get_config_bool(&self, key: Config) -> bool {
         self.get_config_int(key).await != 0
     }
@@ -192,6 +216,14 @@ impl Context {
         }
     }
 
+    /// Gets the configured provider, as saved in the `configured_provider` value.
+    ///
+    /// The provider is determined by `get_provider_info()` during configuration and then saved
+    /// to the db in `param.save_to_database()`, together with all the other `configured_*` values.
+    pub async fn get_configured_provider(&self) -> Option<&'static Provider> {
+        get_provider_by_id(&self.get_config(Config::ConfiguredProvider).await?)
+    }
+
     /// Gets configured "delete_device_after" value.
     ///
     /// `None` means never delete the message, `Some(x)` means delete
@@ -217,7 +249,7 @@ impl Context {
                 match value {
                     Some(value) => {
                         let blob = BlobObject::new_from_path(&self, value).await?;
-                        blob.recode_to_avatar_size(self)?;
+                        blob.recode_to_avatar_size(self).await?;
                         self.sql
                             .set_raw_config(self, key, Some(blob.as_name()))
                             .await
@@ -226,7 +258,7 @@ impl Context {
                 }
             }
             Config::Selfstatus => {
-                let def = self.stock_str(StockMessage::StatusLine).await;
+                let def = stock_str::status_line(self).await;
                 let val = if value.is_none() || value.unwrap() == def {
                     None
                 } else {
@@ -253,19 +285,14 @@ impl Context {
                 job::schedule_resync(self).await;
                 ret
             }
-            Config::InboxWatch => {
-                if self.get_config(Config::InboxWatch).await.as_deref() != value {
-                    // If Inbox-watch is disabled and enabled again, do not fetch emails from in between.
-                    // this avoids unexpected mass-downloads and -deletions (if delete_server_after is set)
-                    if let Some(inbox) = self.get_config(Config::ConfiguredInboxFolder).await {
-                        crate::imap::set_config_last_seen_uid(self, inbox, 0, 0).await;
-                    }
-                }
-                self.sql.set_raw_config(self, key, value).await
-            }
             _ => self.sql.set_raw_config(self, key, value).await,
         }
     }
+
+    pub async fn set_config_bool(&self, key: Config, value: bool) -> crate::sql::Result<()> {
+        self.set_config(key, if value { Some("1") } else { None })
+            .await
+    }
 }
 
 /// Returns all available configuration keys concated together.
@@ -287,8 +314,8 @@ mod tests {
     use std::string::ToString;
 
     use crate::constants;
-    use crate::constants::AVATAR_SIZE;
-    use crate::test_utils::*;
+    use crate::constants::BALANCED_AVATAR_SIZE;
+    use crate::test_utils::TestContext;
     use image::GenericImageView;
     use num_traits::FromPrimitive;
     use std::fs::File;
@@ -306,11 +333,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_default_prop() {
-        assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
-    }
-
     #[async_std::test]
     async fn test_selfavatar_outside_blobdir() {
         let t = TestContext::new().await;
@@ -320,15 +342,14 @@ mod tests {
             .unwrap()
             .write_all(avatar_bytes)
             .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
+        let avatar_blob = t.get_blobdir().join("avatar.jpg");
         assert!(!avatar_blob.exists().await);
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+        t.set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
             .await
             .unwrap();
         assert!(avatar_blob.exists().await);
         assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
@@ -336,14 +357,14 @@ mod tests {
         assert_eq!(img.height(), 1000);
 
         let img = image::open(avatar_blob).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
     }
 
     #[async_std::test]
     async fn test_selfavatar_in_blobdir() {
         let t = TestContext::new().await;
-        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_src = t.get_blobdir().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
             .unwrap()
@@ -354,16 +375,15 @@ mod tests {
         assert_eq!(img.width(), 900);
         assert_eq!(img.height(), 900);
 
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+        t.set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
             .await
             .unwrap();
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
     }
 
     #[async_std::test]
@@ -375,10 +395,9 @@ mod tests {
             .unwrap()
             .write_all(avatar_bytes)
             .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_blob = t.get_blobdir().join("avatar.png");
         assert!(!avatar_blob.exists().await);
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+        t.set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
             .await
             .unwrap();
         assert!(avatar_blob.exists().await);
@@ -386,24 +405,21 @@ mod tests {
             std::fs::metadata(&avatar_blob).unwrap().len(),
             avatar_bytes.len() as u64
         );
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
     }
 
     #[async_std::test]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        let media_quality = t.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
         assert_eq!(media_quality, constants::MediaQuality::Balanced);
 
-        t.ctx
-            .set_config(Config::MediaQuality, Some("1"))
-            .await
-            .unwrap();
+        t.set_config(Config::MediaQuality, Some("1")).await.unwrap();
 
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        let media_quality = t.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 1);
         assert_eq!(constants::MediaQuality::Worse as i32, 1);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure/auto_mozilla.rs

@@ -251,10 +251,10 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
 
 pub(crate) async fn moz_autoconfigure(
     context: &Context,
-    url: &str,
+    url: impl AsRef<str>,
     param_in: &LoginParam,
 ) -> Result<Vec<ServerParams>, Error> {
-    let xml_raw = read_url(context, url).await?;
+    let xml_raw = read_url(context, url.as_ref()).await?;
 
     let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {

src/configure/auto_outlook.rs

@@ -187,9 +187,8 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
 
 pub(crate) async fn outlk_autodiscover(
     context: &Context,
-    url: &str,
+    mut url: String,
 ) -> Result<Vec<ServerParams>, Error> {
-    let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
         let xml_raw = read_url(context, &url).await?;

src/configure/mod.rs

@@ -12,17 +12,20 @@ use itertools::Itertools;
 use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
-use crate::config::Config;
-use crate::dc_tools::*;
+use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
 use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
-use crate::oauth2::*;
+use crate::oauth2::dc_get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
-use crate::stock::StockMessage;
+use crate::stock_str;
 use crate::{chat, e2ee, provider};
-use crate::{constants::*, job};
+use crate::{config::Config, dc_tools::time};
+use crate::{
+    constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2},
+    job,
+};
 use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
@@ -85,7 +88,7 @@ impl Context {
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
-        if let Some(provider) = provider::get_provider_info(&param.addr) {
+        if let Some(provider) = param.provider {
             if let Some(config_defaults) = &provider.config_defaults {
                 for def in config_defaults.iter() {
                     if !self.config_exists(def.key).await {
@@ -124,9 +127,10 @@ impl Context {
                     self,
                     0,
                     Some(
-                        self.stock_string_repl_str(
-                            StockMessage::ConfigurationFailed,
-                            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                        stock_str::configuration_failed(
+                            self,
+                            // We are using Anyhow's .context() and to show the
+                            // inner error, too, we need the {:#}:
                             format!("{:#}", err),
                         )
                         .await
@@ -205,9 +209,51 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     {
         // no advanced parameters entered by the user: query provider-database or do Autoconfig
 
-        if let Some(servers) = get_offline_autoconfig(ctx, &param.addr) {
-            param_autoconfig = Some(servers);
+        info!(
+            ctx,
+            "checking internal provider-info for offline autoconfig"
+        );
+
+        if let Some(provider) = provider::get_provider_info(&param_domain).await {
+            param.provider = Some(provider);
+            match provider.status {
+                provider::Status::OK | provider::Status::PREPARATION => {
+                    if provider.server.is_empty() {
+                        info!(ctx, "offline autoconfig found, but no servers defined");
+                        param_autoconfig = None;
+                    } else {
+                        info!(ctx, "offline autoconfig found");
+                        let servers = provider
+                            .server
+                            .iter()
+                            .map(|s| ServerParams {
+                                protocol: s.protocol,
+                                socket: s.socket,
+                                hostname: s.hostname.to_string(),
+                                port: s.port,
+                                username: match s.username_pattern {
+                                    UsernamePattern::EMAIL => param.addr.to_string(),
+                                    UsernamePattern::EMAILLOCALPART => {
+                                        if let Some(at) = param.addr.find('@') {
+                                            param.addr.split_at(at).0.to_string()
+                                        } else {
+                                            param.addr.to_string()
+                                        }
+                                    }
+                                },
+                            })
+                            .collect();
+
+                        param_autoconfig = Some(servers)
+                    }
+                }
+                provider::Status::BROKEN => {
+                    info!(ctx, "offline autoconfig found, provider is broken");
+                    param_autoconfig = None;
+                }
+            }
         } else {
+            info!(ctx, "no offline autoconfig found");
             param_autoconfig =
                 get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
         }
@@ -257,6 +303,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         .filter(|params| params.protocol == Protocol::SMTP)
         .cloned()
         .collect();
+    let provider_strict_tls = param.provider.map_or(false, |provider| provider.strict_tls);
 
     let smtp_config_task = task::spawn(async move {
         let mut smtp_configured = false;
@@ -267,8 +314,15 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             smtp_param.port = smtp_server.port;
             smtp_param.security = smtp_server.socket;
 
-            match try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp)
-                .await
+            match try_smtp_one_param(
+                &context_smtp,
+                &smtp_param,
+                &smtp_addr,
+                oauth2,
+                provider_strict_tls,
+                &mut smtp,
+            )
+            .await
             {
                 Ok(_) => {
                     smtp_configured = true;
@@ -288,7 +342,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 600);
 
     // Configure IMAP
-    let (_s, r) = async_std::sync::channel(1);
+    let (_s, r) = async_std::channel::bounded(1);
     let mut imap = Imap::new(r);
 
     let mut imap_configured = false;
@@ -304,7 +358,16 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
 
-        match try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
+        match try_imap_one_param(
+            ctx,
+            &param.imap,
+            &param.addr,
+            oauth2,
+            provider_strict_tls,
+            &mut imap,
+        )
+        .await
+        {
             Ok(_) => {
                 imap_configured = true;
                 break;
@@ -351,6 +414,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     // the trailing underscore is correct
     param.save_to_database(ctx, "configured_").await?;
     ctx.sql.set_raw_config_bool(ctx, "configured", true).await?;
+    ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
+        .await?;
 
     progress!(ctx, 920);
 
@@ -369,66 +434,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     Ok(())
 }
 
-#[derive(Debug, PartialEq, Eq)]
-enum AutoconfigProvider {
-    Mozilla,
-    Outlook,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-struct AutoconfigSource {
-    provider: AutoconfigProvider,
-    url: String,
-}
-
-impl AutoconfigSource {
-    fn all(domain: &str, addr: &str) -> [Self; 5] {
-        [
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!(
-                    "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-                    domain, addr,
-                ),
-            },
-            // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!(
-                    "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
-                    domain, addr
-                ),
-            },
-            AutoconfigSource {
-                provider: AutoconfigProvider::Outlook,
-                url: format!("https://{}/autodiscover/autodiscover.xml", domain),
-            },
-            // Outlook uses always SSL but different domains (this comment describes the next two steps)
-            AutoconfigSource {
-                provider: AutoconfigProvider::Outlook,
-                url: format!(
-                    "https://autodiscover.{}/autodiscover/autodiscover.xml",
-                    domain
-                ),
-            },
-            // always SSL for Thunderbird's database
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!("https://autoconfig.thunderbird.net/v1.1/{}", domain),
-            },
-        ]
-    }
-
-    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<Vec<ServerParams>> {
-        let params = match self.provider {
-            AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
-            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url).await?,
-        };
-
-        Ok(params)
-    }
-}
-
 /// Retrieve available autoconfigurations.
 ///
 /// A Search configurations from the domain used in the email-address, prefer encrypted
@@ -439,74 +444,79 @@ async fn get_autoconfig(
     param_domain: &str,
     param_addr_urlencoded: &str,
 ) -> Option<Vec<ServerParams>> {
-    let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        format!(
+            "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
+            param_domain, param_addr_urlencoded
+        ),
+        &param,
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 300);
 
-    let mut progress = 300;
-    for source in &sources {
-        let res = source.fetch(ctx, param).await;
-        progress!(ctx, progress);
-        progress += 10;
-        if let Ok(res) = res {
-            return Some(res);
-        }
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
+        format!(
+            "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
+            &param_domain, &param_addr_urlencoded
+        ),
+        &param,
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 310);
+
+    // Outlook uses always SSL but different domains (this comment describes the next two steps)
+    if let Ok(res) = outlk_autodiscover(
+        ctx,
+        format!("https://{}/autodiscover/autodiscover.xml", &param_domain),
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 320);
+
+    if let Ok(res) = outlk_autodiscover(
+        ctx,
+        format!(
+            "https://autodiscover.{}/autodiscover/autodiscover.xml",
+            &param_domain
+        ),
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 330);
+
+    // always SSL for Thunderbird's database
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
+        &param,
+    )
+    .await
+    {
+        return Some(res);
     }
 
     None
 }
 
-fn get_offline_autoconfig(context: &Context, addr: &str) -> Option<Vec<ServerParams>> {
-    info!(
-        context,
-        "checking internal provider-info for offline autoconfig"
-    );
-
-    if let Some(provider) = provider::get_provider_info(&addr) {
-        match provider.status {
-            provider::Status::OK | provider::Status::PREPARATION => {
-                if provider.server.is_empty() {
-                    info!(context, "offline autoconfig found, but no servers defined");
-                    None
-                } else {
-                    info!(context, "offline autoconfig found");
-                    let servers = provider
-                        .server
-                        .iter()
-                        .map(|s| ServerParams {
-                            protocol: s.protocol,
-                            socket: s.socket,
-                            hostname: s.hostname.to_string(),
-                            port: s.port,
-                            username: match s.username_pattern {
-                                UsernamePattern::EMAIL => addr.to_string(),
-                                UsernamePattern::EMAILLOCALPART => {
-                                    if let Some(at) = addr.find('@') {
-                                        addr.split_at(at).0.to_string()
-                                    } else {
-                                        addr.to_string()
-                                    }
-                                }
-                            },
-                        })
-                        .collect();
-                    Some(servers)
-                }
-            }
-            provider::Status::BROKEN => {
-                info!(context, "offline autoconfig found, provider is broken");
-                None
-            }
-        }
-    } else {
-        info!(context, "no offline autoconfig found");
-        None
-    }
-}
-
 async fn try_imap_one_param(
     context: &Context,
     param: &ServerLoginParam,
     addr: &str,
     oauth2: bool,
+    provider_strict_tls: bool,
     imap: &mut Imap,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
@@ -515,7 +525,10 @@ async fn try_imap_one_param(
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = imap.connect(context, param, addr, oauth2).await {
+    if let Err(err) = imap
+        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .await
+    {
         info!(context, "failure: {}", err);
         Err(ConfigurationError {
             config: inf,
@@ -532,6 +545,7 @@ async fn try_smtp_one_param(
     param: &ServerLoginParam,
     addr: &str,
     oauth2: bool,
+    provider_strict_tls: bool,
     smtp: &mut Smtp,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
@@ -540,7 +554,10 @@ async fn try_smtp_one_param(
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
+    if let Err(err) = smtp
+        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .await
+    {
         info!(context, "failure: {}", err);
         Err(ConfigurationError {
             config: inf,
@@ -573,10 +590,7 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
         .iter()
         .all(|e| e.msg.to_lowercase().contains("could not resolve"))
     {
-        return context
-            .stock_str(StockMessage::ErrorNoNetwork)
-            .await
-            .to_string();
+        return stock_str::error_no_network(context).await;
     }
 
     if errors.iter().all(|e| e.msg == first_err.msg) {
@@ -609,37 +623,16 @@ pub enum Error {
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
-    use super::*;
-    use crate::config::*;
-    use crate::test_utils::*;
+    use crate::config::Config;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_no_panic_on_bad_credentials() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::Addr, Some("probably@unexistant.addr"))
+        t.set_config(Config::Addr, Some("probably@unexistant.addr"))
             .await
             .unwrap();
-        t.ctx
-            .set_config(Config::MailPw, Some("123456"))
-            .await
-            .unwrap();
-        assert!(t.ctx.configure().await.is_err());
-    }
-
-    #[async_std::test]
-    async fn test_get_offline_autoconfig() {
-        let context = TestContext::new().await.ctx;
-
-        let addr = "someone123@example.org";
-        assert!(get_offline_autoconfig(&context, addr).is_none());
-
-        let addr = "someone123@nauta.cu";
-        let found_params = get_offline_autoconfig(&context, addr).unwrap();
-        assert_eq!(found_params.len(), 2);
-        assert_eq!(found_params[0].protocol, Protocol::IMAP);
-        assert_eq!(found_params[0].hostname, "imap.nauta.cu".to_string());
-        assert_eq!(found_params[1].protocol, Protocol::SMTP);
-        assert_eq!(found_params[1].hostname, "smtp.nauta.cu".to_string());
+        t.set_config(Config::MailPw, Some("123456")).await.unwrap();
+        assert!(t.configure().await.is_err());
     }
 }

src/constants.rs

@@ -1,5 +1,5 @@
 //! # Constants
-use deltachat_derive::*;
+use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
@@ -99,6 +99,7 @@ pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
 pub const DC_GCL_FOR_FORWARDING: usize = 0x08;
 
 pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
+pub const DC_GCM_INFO_ONLY: u32 = 0x02;
 
 pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
 pub const DC_GCL_ADD_SELF: usize = 0x02;
@@ -143,6 +144,7 @@ pub enum Chattype {
     Undefined = 0,
     Single = 100,
     Group = 120,
+    Mailinglist = 140,
 }
 
 impl Default for Chattype {
@@ -196,7 +198,8 @@ pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
 pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
 
 // max. width/height of an avatar
-pub const AVATAR_SIZE: u32 = 192;
+pub const BALANCED_AVATAR_SIZE: u32 = 256;
+pub const WORSE_AVATAR_SIZE: u32 = 128;
 
 // max. width/height of images
 pub const BALANCED_IMAGE_SIZE: u32 = 1280;

src/context.rs

@@ -3,17 +3,21 @@
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
+use std::time::{Instant, SystemTime};
 
-use async_std::path::{Path, PathBuf};
-use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
-use async_std::task;
+use anyhow::{bail, ensure, Result};
+use async_std::{
+    channel::{self, Receiver, Sender},
+    path::{Path, PathBuf},
+    sync::{Arc, Mutex, RwLock},
+    task,
+};
 
-use crate::chat::*;
+use crate::chat::{get_chat_cnt, ChatId};
 use crate::config::Config;
-use crate::constants::*;
-use crate::contact::*;
-use crate::dc_tools::{dc_get_dirbytes, dc_get_filebytes, duration_to_str};
-use crate::error::*;
+use crate::constants::DC_VERSION_STR;
+use crate::contact::Contact;
+use crate::dc_tools::duration_to_str;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
@@ -21,7 +25,6 @@ use crate::message::{self, MsgId};
 use crate::scheduler::Scheduler;
 use crate::securejoin::Bob;
 use crate::sql::Sql;
-use std::time::SystemTime;
 
 #[derive(Clone, Debug)]
 pub struct Context {
@@ -44,7 +47,7 @@ pub struct InnerContext {
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
     pub(crate) os_name: Option<String>,
-    pub(crate) bob: RwLock<Bob>,
+    pub(crate) bob: Bob,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -59,7 +62,12 @@ pub struct InnerContext {
     pub(crate) scheduler: RwLock<Scheduler>,
     pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
-    /// Id for this context on the current device.
+    pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
+
+    /// ID for this `Context` in the current process.
+    ///
+    /// This allows for multiple `Context`s open in a single process where each context can
+    /// be identified by this ID.
     pub(crate) id: u32,
 
     creation_time: SystemTime,
@@ -121,7 +129,7 @@ impl Context {
             os_name: Some(os_name),
             running_state: RwLock::new(Default::default()),
             sql: Sql::new(),
-            bob: RwLock::new(Default::default()),
+            bob: Default::default(),
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
@@ -131,6 +139,7 @@ impl Context {
             scheduler: RwLock::new(Scheduler::Stopped),
             ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
+            last_full_folder_scan: Mutex::new(None),
         };
 
         let ctx = Context {
@@ -144,7 +153,7 @@ impl Context {
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
         info!(self, "starting IO");
-        if self.is_io_running().await {
+        if self.inner.is_io_running().await {
             info!(self, "IO is already running");
             return;
         }
@@ -155,11 +164,6 @@ impl Context {
         }
     }
 
-    /// Returns if the IO scheduler is running.
-    pub async fn is_io_running(&self) -> bool {
-        self.inner.is_io_running().await
-    }
-
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
@@ -193,7 +197,10 @@ impl Context {
         });
     }
 
-    /// Get the next queued event.
+    /// Returns a receiver for emitted events.
+    ///
+    /// Multiple emitters can be created, but note that in this case each emitted event will
+    /// only be received by one of the emitters, not by all of them.
     pub fn get_event_emitter(&self) -> EventEmitter {
         self.events.get_emitter()
     }
@@ -215,7 +222,7 @@ impl Context {
 
         s.ongoing_running = true;
         s.shall_stop_ongoing = false;
-        let (sender, receiver) = channel(1);
+        let (sender, receiver) = channel::bounded(1);
         s.cancel_sender = Some(sender);
 
         Ok(receiver)
@@ -242,7 +249,9 @@ impl Context {
         let s_a = &self.running_state;
         let mut s = s_a.write().await;
         if let Some(cancel) = s.cancel_sender.take() {
-            cancel.send(()).await;
+            if let Err(err) = cancel.send(()).await {
+                warn!(self, "could not cancel ongoing: {:?}", err);
+            }
         }
 
         if s.ongoing_running && !s.shall_stop_ongoing {
@@ -262,9 +271,6 @@ impl Context {
      ******************************************************************************/
 
     pub async fn get_info(&self) -> BTreeMap<&'static str, String> {
-        let blobdir = self.get_blobdir();
-        let (blobdir_files, blobdir_bytes) = dc_get_dirbytes(self, blobdir).await;
-
         let unset = "0";
         let l = LoginParam::from_database(self, "").await;
         let l2 = LoginParam::from_database(self, "configured_").await;
@@ -306,6 +312,7 @@ impl Context {
         let sentbox_watch = self.get_config_int(Config::SentboxWatch).await;
         let mvbox_watch = self.get_config_int(Config::MvboxWatch).await;
         let mvbox_move = self.get_config_int(Config::MvboxMove).await;
+        let sentbox_move = self.get_config_int(Config::SentboxMove).await;
         let folders_configured = self
             .sql
             .get_raw_config_int(self, "folders_configured")
@@ -322,20 +329,17 @@ impl Context {
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
+
+        // insert values
+        res.insert("bot", self.get_config_int(Config::Bot).await.to_string());
         res.insert("number_of_chats", chats.to_string());
         res.insert("number_of_chat_messages", real_msgs.to_string());
         res.insert("messages_in_contact_requests", deaddrop_msgs.to_string());
         res.insert("number_of_contacts", contacts.to_string());
-        res.insert("database", self.get_dbfile().display().to_string());
+        res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
-        res.insert(
-            "database_bytes",
-            dc_get_filebytes(self, self.get_dbfile()).await.to_string(),
-        );
         res.insert("journal_mode", journal_mode);
-        res.insert("blobdir", blobdir.display().to_string());
-        res.insert("blobdir_files", blobdir_files.to_string());
-        res.insert("blobdir_bytes", blobdir_bytes.to_string());
+        res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
         res.insert(
             "selfavatar",
@@ -346,15 +350,30 @@ impl Context {
         res.insert("is_configured", is_configured.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
+        res.insert(
+            "fetch_existing_msgs",
+            self.get_config_int(Config::FetchExistingMsgs)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "show_emails",
+            self.get_config_int(Config::ShowEmails).await.to_string(),
+        );
         res.insert("inbox_watch", inbox_watch.to_string());
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_watch", mvbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
+        res.insert("sentbox_move", sentbox_move.to_string());
         res.insert("folders_configured", folders_configured.to_string());
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
         res.insert("mdns_enabled", mdns_enabled.to_string());
         res.insert("e2ee_enabled", e2ee_enabled.to_string());
+        res.insert(
+            "key_gen_type",
+            self.get_config_int(Config::KeyGenType).await.to_string(),
+        );
         res.insert("bcc_self", bcc_self.to_string());
         res.insert(
             "private_key_count",
@@ -365,6 +384,40 @@ impl Context {
             pub_key_cnt.unwrap_or_default().to_string(),
         );
         res.insert("fingerprint", fingerprint_str);
+        res.insert(
+            "webrtc_instance",
+            self.get_config(Config::WebrtcInstance)
+                .await
+                .unwrap_or_else(|| "<unset>".to_string()),
+        );
+        res.insert(
+            "media_quality",
+            self.get_config_int(Config::MediaQuality).await.to_string(),
+        );
+        res.insert(
+            "delete_device_after",
+            self.get_config_int(Config::DeleteDeviceAfter)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "delete_server_after",
+            self.get_config_int(Config::DeleteServerAfter)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "last_housekeeping",
+            self.get_config_int(Config::LastHousekeeping)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "scan_all_folders_debounce_secs",
+            self.get_config_int(Config::ScanAllFoldersDebounceSecs)
+                .await
+                .to_string(),
+        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
@@ -404,48 +457,22 @@ impl Context {
             .unwrap_or_default()
     }
 
-    #[allow(non_snake_case)]
-    pub async fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
+    /// Searches for messages containing the query string.
+    ///
+    /// If `chat_id` is provided this searches only for messages in this chat, if `chat_id`
+    /// is `None` this searches messages from all chats.
+    pub async fn search_msgs(&self, chat_id: Option<ChatId>, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
         }
-        let strLikeInText = format!("%{}%", real_query);
-        let strLikeBeg = format!("{}%", real_query);
+        let str_like_in_text = format!("%{}%", real_query);
+        let str_like_beg = format!("{}%", real_query);
 
-        let query = if !chat_id.is_unset() {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " WHERE m.chat_id=?",
-                "   AND m.hidden=0",
-                "   AND ct.blocked=0",
-                "   AND (txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp,m.id;"
-            )
-        } else {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " LEFT JOIN chats c",
-                "        ON m.chat_id=c.id",
-                " WHERE m.chat_id>9",
-                "   AND m.hidden=0",
-                "   AND (c.blocked=0 OR c.blocked=?)",
-                "   AND ct.blocked=0",
-                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp DESC,m.id DESC;"
-            )
-        };
-
-        self.sql
-            .query_map(
+        let do_query = |query, params| {
+            self.sql.query_map(
                 query,
-                paramsv![chat_id, strLikeInText, strLikeBeg],
+                params,
                 |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
@@ -455,8 +482,42 @@ impl Context {
                     Ok(ret)
                 },
             )
+        };
+
+        if let Some(chat_id) = chat_id {
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
+                 FROM msgs m
+                 LEFT JOIN contacts ct
+                        ON m.from_id=ct.id
+                 WHERE m.chat_id=?
+                   AND m.hidden=0
+                   AND ct.blocked=0
+                   AND (txt LIKE ? OR ct.name LIKE ?)
+                 ORDER BY m.timestamp,m.id;",
+                paramsv![chat_id, str_like_in_text, str_like_beg],
+            )
             .await
             .unwrap_or_default()
+        } else {
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
+                 FROM msgs m
+                 LEFT JOIN contacts ct
+                        ON m.from_id=ct.id
+                 LEFT JOIN chats c
+                        ON m.chat_id=c.id
+                 WHERE m.chat_id>9
+                   AND m.hidden=0
+                   AND c.blocked=0
+                   AND ct.blocked=0
+                   AND (m.txt LIKE ? OR ct.name LIKE ?)
+                 ORDER BY m.timestamp DESC,m.id DESC;",
+                paramsv![str_like_in_text, str_like_beg],
+            )
+            .await
+            .unwrap_or_default()
+        }
     }
 
     pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
@@ -474,6 +535,11 @@ impl Context {
             == Some(folder_name.as_ref().to_string())
     }
 
+    pub async fn is_spam_folder(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredSpamFolder).await
+            == Some(folder_name.as_ref().to_string())
+    }
+
     pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -523,7 +589,8 @@ pub fn get_version_str() -> &'static str {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
+    use strum::IntoEnumIterator;
 
     #[async_std::test]
     async fn test_wrong_db() {
@@ -537,7 +604,7 @@ mod tests {
     #[async_std::test]
     async fn test_get_fresh_msgs() {
         let t = TestContext::new().await;
-        let fresh = t.ctx.get_fresh_msgs().await;
+        let fresh = t.get_fresh_msgs().await;
         assert!(fresh.is_empty())
     }
 
@@ -596,24 +663,65 @@ mod tests {
     #[async_std::test]
     async fn no_crashes_on_context_deref() {
         let t = TestContext::new().await;
-        std::mem::drop(t.ctx);
+        std::mem::drop(t);
     }
 
     #[async_std::test]
     async fn test_get_info() {
         let t = TestContext::new().await;
 
-        let info = t.ctx.get_info().await;
-        assert!(info.get("database").is_some());
-        assert!(info.get("database_bytes").unwrap().parse::<u64>().unwrap() > 1000);
+        let info = t.get_info().await;
+        assert!(info.get("database_dir").is_some());
     }
 
     #[test]
     fn test_get_info_no_context() {
         let info = get_info();
         assert!(info.get("deltachat_core_version").is_some());
-        assert!(info.get("database").is_none());
-        assert!(info.get("database_bytes").is_none());
+        assert!(info.get("database_dir").is_none());
         assert_eq!(info.get("level").unwrap(), "awesome");
     }
+
+    #[async_std::test]
+    async fn test_get_info_completeness() {
+        // For easier debugging,
+        // get_info() shall return all important information configurable by the Config-values.
+        //
+        // There are exceptions for Config-values considered to be unimportant,
+        // too sensitive or summarized in another item.
+        let skip_from_get_info = vec![
+            "addr",
+            "displayname",
+            "imap_certificate_checks",
+            "mail_server",
+            "mail_user",
+            "mail_pw",
+            "mail_port",
+            "mail_security",
+            "notify_about_wrong_pw",
+            "save_mime_headers",
+            "selfstatus",
+            "send_server",
+            "send_user",
+            "send_pw",
+            "send_port",
+            "send_security",
+            "server_flags",
+            "smtp_certificate_checks",
+        ];
+        let t = TestContext::new().await;
+        let info = t.get_info().await;
+        for key in Config::iter() {
+            let key: String = key.to_string();
+            if !skip_from_get_info.contains(&&*key)
+                && !key.starts_with("configured")
+                && !key.starts_with("sys.")
+            {
+                assert!(
+                    info.contains_key(&*key),
+                    format!("'{}' missing in get_info() output", key)
+                );
+            }
+        }
+    }
 }

src/dc_tools.rs

@@ -12,17 +12,17 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::{bail, Error};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
 use crate::constants::{Viewtype, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
-use crate::error::{bail, Error};
 use crate::events::EventType;
 use crate::message::Message;
 use crate::provider::get_provider_update_timestamp;
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
@@ -48,31 +48,6 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
     }
 }
 
-/// the colors must fulfill some criterions as:
-/// - contrast to black and to white
-/// - work as a text-color
-/// - being noticeable on a typical map
-/// - harmonize together while being different enough
-/// (therefore, we cannot just use random rgb colors :)
-const COLORS: [u32; 16] = [
-    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
-    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
-];
-
-#[allow(clippy::indexing_slicing)]
-pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
-    let str_lower = s.as_ref().to_lowercase();
-    let mut checksum = 0;
-    let bytes = str_lower.as_bytes();
-    for (i, byte) in bytes.iter().enumerate() {
-        checksum += (i + 1) * *byte as usize;
-        checksum %= 0x00ff_ffff;
-    }
-    let color_index = checksum % COLORS.len();
-
-    COLORS[color_index]
-}
-
 /* ******************************************************************************
  * date/time tools
  ******************************************************************************/
@@ -169,15 +144,14 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
     if now < known_past_timestamp {
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some(
-            context
-                .stock_string_repl_str(
-                    StockMessage::BadTimeMsgBody,
-                    Local
-                        .timestamp(now, 0)
-                        .format("%Y-%m-%d %H:%M:%S")
-                        .to_string(),
-                )
-                .await,
+            stock_str::bad_time_msg_body(
+                context,
+                Local
+                    .timestamp(now, 0)
+                    .format("%Y-%m-%d %H:%M:%S")
+                    .to_string(),
+            )
+            .await,
         );
         add_device_msg_with_importance(
             context,
@@ -201,12 +175,7 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
 async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time: i64) {
     if now > approx_compile_time + DC_OUTDATED_WARNING_DAYS * 24 * 60 * 60 {
         let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(
-            context
-                .stock_str(StockMessage::UpdateReminderMsgBody)
-                .await
-                .into(),
-        );
+        msg.text = Some(stock_str::update_reminder_msg_body(context).await);
         add_device_msg(
             context,
             Some(
@@ -339,24 +308,6 @@ pub(crate) async fn dc_get_filebytes(context: &Context, path: impl AsRef<Path>)
     }
 }
 
-pub(crate) async fn dc_get_dirbytes(context: &Context, path: impl AsRef<Path>) -> (usize, u64) {
-    let path_abs = dc_get_abs_path(context, &path);
-    let mut files: usize = 0;
-    let mut bytes: u64 = 0;
-    if let Ok(mut read_dir) = async_std::fs::read_dir(path_abs).await {
-        while let Some(entry) = read_dir.next().await {
-            if let Ok(entry) = entry {
-                files += 1;
-                bytes += match entry.metadata().await {
-                    Ok(meta) => meta.len(),
-                    Err(_err) => 0,
-                }
-            }
-        }
-    }
-    (files, bytes)
-}
-
 pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
     if !path_abs.exists().await {
@@ -555,29 +506,9 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn get_next_backup_path_old(
-    folder: impl AsRef<Path>,
-    backup_time: i64,
-) -> Result<PathBuf, Error> {
-    let folder = PathBuf::from(folder.as_ref());
-    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
-        .format("delta-chat-%Y-%m-%d")
-        .to_string();
-
-    // 64 backup files per day should be enough for everyone
-    for i in 0..64 {
-        let mut path = folder.clone();
-        path.push(format!("{}-{}.bak", stem, i));
-        if !path.exists().await {
-            return Ok(path);
-        }
-    }
-    bail!("could not create backup file, disk full?");
-}
-
 /// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
 /// it can be renamed to dest_path. This guarantees that the backup is complete.
-pub(crate) async fn get_next_backup_path_new(
+pub(crate) async fn get_next_backup_path(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<(PathBuf, PathBuf), Error> {
@@ -746,8 +677,8 @@ mod tests {
     use super::*;
     use std::convert::TryInto;
 
-    use crate::constants::*;
-    use crate::test_utils::*;
+    use crate::constants::{DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_rust_ftoa() {
@@ -936,7 +867,7 @@ mod tests {
     #[async_std::test]
     async fn test_file_handling() {
         let t = TestContext::new().await;
-        let context = &t.ctx;
+        let context = &t;
         macro_rules! dc_file_exist {
             ($ctx:expr, $fname:expr) => {
                 $ctx.get_blobdir()
@@ -1016,11 +947,11 @@ mod tests {
     async fn test_create_smeared_timestamp() {
         let t = TestContext::new().await;
         assert_ne!(
-            dc_create_smeared_timestamp(&t.ctx).await,
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t).await,
+            dc_create_smeared_timestamp(&t).await
         );
         assert!(
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t).await
                 >= SystemTime::now()
                     .duration_since(SystemTime::UNIX_EPOCH)
                     .unwrap()
@@ -1032,13 +963,13 @@ mod tests {
     async fn test_create_smeared_timestamps() {
         let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t, count as usize).await;
+        let next = dc_smeared_time(&t).await;
         assert!((start + count - 1) < next);
 
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t, count as usize).await;
+        let next = dc_smeared_time(&t).await;
         assert!((start + count - 1) < next);
     }
 
@@ -1112,53 +1043,53 @@ mod tests {
             / 1_000;
 
         // a correct time must not add a device message
-        maybe_warn_on_bad_time(&t.ctx, timestamp_now, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t, timestamp_now, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // we cannot find out if a date in the future is wrong - a device message is not added
-        maybe_warn_on_bad_time(&t.ctx, timestamp_future, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t, timestamp_future, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // a date in the past must add a device message
-        maybe_warn_on_bad_time(&t.ctx, timestamp_past, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t, timestamp_past, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         // the message should be added only once a day - test that an hour later and nearly a day later
         maybe_warn_on_bad_time(
-            &t.ctx,
+            &t,
             timestamp_past + 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         maybe_warn_on_bad_time(
-            &t.ctx,
+            &t,
             timestamp_past + 60 * 60 * 24 - 1,
             get_provider_update_timestamp(),
         )
         .await;
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         // next day, there should be another device message
         maybe_warn_on_bad_time(
-            &t.ctx,
+            &t,
             timestamp_past + 60 * 60 * 24,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         assert_eq!(device_chat_id, chats.get_chat_id(0));
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 2);
     }
 
@@ -1170,60 +1101,60 @@ mod tests {
         // in about 6 months, the app should not be outdated
         // (if this fails, provider-db is not updated since 6 months)
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + 180 * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // in 1 year, the app should be considered as outdated
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + 365 * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         // do not repeat the warning every day ...
         // (we test that for the 2 subsequent days, this may be the next month, so the result should be 1 or 2 device message)
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + (365 + 1) * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + (365 + 2) * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         let test_len = msgs.len();
         assert!(test_len == 1 || test_len == 2);
 
         // ... but every month
         // (forward generous 33 days to avoid being in the same month as in the previous check)
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + (365 + 33) * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), test_len + 1);
     }
 }

src/dehtml.rs

@@ -2,8 +2,13 @@
 //!
 //! A module to remove HTML tags from the email text
 
+use std::io::BufRead;
+
 use once_cell::sync::Lazy;
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+use quick_xml::{
+    events::{BytesEnd, BytesStart, BytesText},
+    Reader,
+};
 
 static LINE_RE: Lazy<regex::Regex> = Lazy::new(|| regex::Regex::new(r"(\r?\n)+").unwrap());
 
@@ -11,9 +16,37 @@ struct Dehtml {
     strbuilder: String,
     add_text: AddText,
     last_href: Option<String>,
+    /// GMX wraps a quote in `<div name="quote">`. After a `<div name="quote">`, this count is
+    /// increased at each `<div>` and decreased at each `</div>`. This way we know when the quote ends.
+    /// If this is > `0`, then we are inside a `<div name="quote">`
+    divs_since_quote_div: u32,
+    /// Everything between <div name="quote"> and <div name="quoted-content"> is usually metadata
+    /// If this is > `0`, then we are inside a `<div name="quoted-content">`.
+    divs_since_quoted_content_div: u32,
 }
 
-#[derive(Debug, PartialEq)]
+impl Dehtml {
+    fn line_prefix(&self) -> &str {
+        if self.divs_since_quoted_content_div > 0 {
+            "> "
+        } else {
+            ""
+        }
+    }
+    fn append_prefix(&self, line_end: impl AsRef<str>) -> String {
+        // line_end is e.g. "\n\n". We add "> " if necessary.
+        line_end.as_ref().to_owned() + self.line_prefix()
+    }
+    fn get_add_text(&self) -> AddText {
+        if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {
+            AddText::No // Everything between <div name="quoted"> and <div name="quoted_content"> is metadata which we don't want
+        } else {
+            self.add_text
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
 enum AddText {
     No,
     YesRemoveLineEnds,
@@ -41,6 +74,8 @@ pub fn dehtml_quick_xml(buf: &str) -> String {
         strbuilder: String::with_capacity(buf.len()),
         add_text: AddText::YesRemoveLineEnds,
         last_href: None,
+        divs_since_quote_div: 0,
+        divs_since_quoted_content_div: 0,
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
@@ -79,13 +114,16 @@ pub fn dehtml_quick_xml(buf: &str) -> String {
 }
 
 fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.add_text == AddText::YesPreserveLineEnds
-        || dehtml.add_text == AddText::YesRemoveLineEnds
+    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
+        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.add_text == AddText::YesRemoveLineEnds {
+        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
+        } else if !dehtml.line_prefix().is_empty() {
+            let l = dehtml.append_prefix("\n");
+            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -93,13 +131,16 @@ fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
 }
 
 fn dehtml_cdata_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.add_text == AddText::YesPreserveLineEnds
-        || dehtml.add_text == AddText::YesRemoveLineEnds
+    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
+        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.add_text == AddText::YesRemoveLineEnds {
+        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
+        } else if !dehtml.line_prefix().is_empty() {
+            let l = dehtml.append_prefix("\n");
+            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -110,8 +151,15 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "div" | "table" | "td" | "style" | "script" | "title" | "pre" => {
-            dehtml.strbuilder += "\n\n";
+        "p" | "table" | "td" | "style" | "script" | "title" | "pre" => {
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+            dehtml.add_text = AddText::YesRemoveLineEnds;
+        }
+        "div" => {
+            pop_tag(&mut dehtml.divs_since_quote_div);
+            pop_tag(&mut dehtml.divs_since_quoted_content_div);
+
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "a" => {
@@ -122,10 +170,14 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
             }
         }
         "b" | "strong" => {
-            dehtml.strbuilder += "*";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "*";
+            }
         }
         "i" | "em" => {
-            dehtml.strbuilder += "_";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "_";
+            }
         }
         _ => {}
     }
@@ -139,19 +191,27 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "div" | "table" | "td" => {
-            dehtml.strbuilder += "\n\n";
+        "p" | "table" | "td" => {
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+            dehtml.add_text = AddText::YesRemoveLineEnds;
+        }
+        #[rustfmt::skip]
+        "div" => {
+            maybe_push_tag(event, reader, "quote", &mut dehtml.divs_since_quote_div);
+            maybe_push_tag(event, reader, "quoted-content", &mut dehtml.divs_since_quoted_content_div);
+
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "br" => {
-            dehtml.strbuilder += "\n";
+            dehtml.strbuilder += &dehtml.append_prefix("\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "style" | "script" | "title" => {
             dehtml.add_text = AddText::No;
         }
         "pre" => {
-            dehtml.strbuilder += "\n\n";
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
@@ -172,15 +232,51 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             }
         }
         "b" | "strong" => {
-            dehtml.strbuilder += "*";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "*";
+            }
         }
         "i" | "em" => {
-            dehtml.strbuilder += "_";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "_";
+            }
         }
         _ => {}
     }
 }
 
+/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
+/// The `counts`s are stored in the `Dehtml` struct.
+fn pop_tag(count: &mut u32) {
+    if *count > 0 {
+        *count -= 1;
+    }
+}
+
+/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
+/// The `counts`s are stored in the `Dehtml` struct.
+fn maybe_push_tag(
+    event: &BytesStart,
+    reader: &Reader<impl BufRead>,
+    tag_name: &str,
+    count: &mut u32,
+) {
+    if *count > 0 || tag_contains_attr(event, reader, tag_name) {
+        *count += 1;
+    }
+}
+
+fn tag_contains_attr(event: &BytesStart, reader: &Reader<impl BufRead>, name: &str) -> bool {
+    event.attributes().any(|r| {
+        r.map(|a| {
+            a.unescape_and_decode_value(reader)
+                .map(|v| v == name)
+                .unwrap_or(false)
+        })
+        .unwrap_or(false)
+    })
+}
+
 pub fn dehtml_manually(buf: &str) -> String {
     // Just strip out everything between "<" and ">"
     let mut strbuilder = String::new();
@@ -288,4 +384,17 @@ mod tests {
         let txt = dehtml(input).unwrap();
         assert_eq!(txt.trim(), "lots of text");
     }
+
+    #[async_std::test]
+    async fn test_quote_div() {
+        let input = include_str!("../test-data/message/gmx-quote-body.eml");
+        let dehtml = dehtml(input).unwrap();
+        println!("{}", dehtml);
+        let (msg, forwarded, cut, top_quote, footer) = simplify(dehtml, false);
+        assert_eq!(msg, "Test");
+        assert_eq!(forwarded, false);
+        assert_eq!(cut, false);
+        assert_eq!(top_quote.as_deref(), Some("test"));
+        assert_eq!(footer, None);
+    }
 }

src/e2ee.rs

@@ -2,18 +2,18 @@
 
 use std::collections::HashSet;
 
+use anyhow::{bail, ensure, format_err, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
-use crate::aheader::*;
+use crate::aheader::{Aheader, EncryptPreference};
 use crate::config::Config;
 use crate::context::Context;
-use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
-use crate::keyring::*;
-use crate::peerstate::*;
+use crate::keyring::Keyring;
+use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::pgp;
 
 #[derive(Debug)]
@@ -346,10 +346,10 @@ mod tests {
 
     use crate::chat;
     use crate::constants::Viewtype;
-    use crate::contact::{Contact, Origin};
     use crate::message::Message;
     use crate::param::Param;
-    use crate::test_utils::*;
+    use crate::peerstate::ToSave;
+    use crate::test_utils::{bob_keypair, TestContext};
 
     mod ensure_secret_key_exists {
         use super::*;
@@ -358,13 +358,13 @@ mod tests {
         async fn test_prexisting() {
             let t = TestContext::new().await;
             let test_addr = t.configure_alice().await;
-            assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
+            assert_eq!(ensure_secret_key_exists(&t).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
         async fn test_not_configured() {
             let t = TestContext::new().await;
-            assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
+            assert!(ensure_secret_key_exists(&t).await.is_err());
         }
     }
 
@@ -411,27 +411,12 @@ Sent with my Delta Chat Messenger: https://delta.chat";
     }
 
     #[async_std::test]
-    async fn test_encrypted_no_autocrypt() -> crate::error::Result<()> {
+    async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
-        let (contact_alice_id, _modified) = Contact::add_or_lookup(
-            &bob.ctx,
-            "Alice",
-            "alice@example.com",
-            Origin::ManuallyCreated,
-        )
-        .await?;
-        let (contact_bob_id, _modified) = Contact::add_or_lookup(
-            &alice.ctx,
-            "Bob",
-            "bob@example.net",
-            Origin::ManuallyCreated,
-        )
-        .await?;
-
-        let chat_alice = chat::create_by_contact_id(&alice.ctx, contact_bob_id).await?;
-        let chat_bob = chat::create_by_contact_id(&bob.ctx, contact_alice_id).await?;
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
 
         // Alice sends unencrypted message to Bob
         let mut msg = Message::new(Viewtype::Text);
@@ -542,28 +527,28 @@ Sent with my Delta Chat Messenger: https://delta.chat";
     #[async_std::test]
     async fn test_should_encrypt() {
         let t = TestContext::new_alice().await;
-        let encrypt_helper = EncryptHelper::new(&t.ctx).await.unwrap();
+        let encrypt_helper = EncryptHelper::new(&t).await.unwrap();
 
         // test with EncryptPreference::NoPreference:
         // if e2ee_eguaranteed is unset, there is no encryption as not more than half of peers want encryption
-        let ps = new_peerstates(&t.ctx, EncryptPreference::NoPreference);
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = new_peerstates(&t, EncryptPreference::NoPreference);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with EncryptPreference::Reset
-        let ps = new_peerstates(&t.ctx, EncryptPreference::Reset);
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = new_peerstates(&t, EncryptPreference::Reset);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with EncryptPreference::Mutual (self is also Mutual)
-        let ps = new_peerstates(&t.ctx, EncryptPreference::Mutual);
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
-        assert!(encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = new_peerstates(&t, EncryptPreference::Mutual);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with missing peerstate
         let mut ps = Vec::new();
         ps.push((None, "bob@foo.bar"));
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).is_err());
-        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).is_err());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
     }
 }

src/ephemeral.rs

@@ -56,24 +56,26 @@
 //! the database entries which are expired either according to their
 //! ephemeral message timers or global `delete_server_after` setting.
 
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use anyhow::{ensure, Error};
+use async_std::task;
+use serde::{Deserialize, Serialize};
+
 use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
 use crate::constants::{
     Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
 };
 use crate::context::Context;
 use crate::dc_tools::time;
-use crate::error::{ensure, Error};
 use crate::events::EventType;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::sql;
-use crate::stock::StockMessage;
-use async_std::task;
-use serde::{Deserialize, Serialize};
-use std::convert::{TryFrom, TryInto};
-use std::num::ParseIntError;
-use std::str::FromStr;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use crate::stock_str;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -211,21 +213,50 @@ pub(crate) async fn stock_ephemeral_timer_changed(
     timer: Timer,
     from_id: u32,
 ) -> String {
-    let stock_message = match timer {
-        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+    match timer {
+        Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
         Timer::Enabled { duration } => match duration {
-            60 => StockMessage::MsgEphemeralTimerMinute,
-            3600 => StockMessage::MsgEphemeralTimerHour,
-            86400 => StockMessage::MsgEphemeralTimerDay,
-            604_800 => StockMessage::MsgEphemeralTimerWeek,
-            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
-            _ => StockMessage::MsgEphemeralTimerEnabled,
+            0..=59 => {
+                stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id).await
+            }
+            60 => stock_str::msg_ephemeral_timer_minute(context, from_id).await,
+            61..=3599 => {
+                stock_str::msg_ephemeral_timer_minutes(
+                    context,
+                    format!("{}", (f64::from(duration) / 6.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            3600 => stock_str::msg_ephemeral_timer_hour(context, from_id).await,
+            3601..=86399 => {
+                stock_str::msg_ephemeral_timer_hours(
+                    context,
+                    format!("{}", (f64::from(duration) / 360.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            86400 => stock_str::msg_ephemeral_timer_day(context, from_id).await,
+            86401..=604_799 => {
+                stock_str::msg_ephemeral_timer_days(
+                    context,
+                    format!("{}", (f64::from(duration) / 8640.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            604_800 => stock_str::msg_ephemeral_timer_week(context, from_id).await,
+            _ => {
+                stock_str::msg_ephemeral_timer_weeks(
+                    context,
+                    format!("{}", (f64::from(duration) / 60480.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
         },
-    };
-
-    context
-        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
-        .await
+    }
 }
 
 impl MsgId {
@@ -278,10 +309,10 @@ pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, E
         .sql
         .execute(
             "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
+             SET chat_id=?, txt='', txt_raw='', from_id=0, to_id=0, param='' \
              WHERE \
              ephemeral_timestamp != 0 \
-             AND ephemeral_timestamp < ? \
+             AND ephemeral_timestamp <= ? \
              AND chat_id != ?",
             paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
         )
@@ -417,7 +448,7 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
             "SELECT id FROM msgs \
          WHERE ( \
          timestamp < ? \
-         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
          LIMIT 1",
@@ -459,8 +490,14 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()>
 
 #[cfg(test)]
 mod tests {
+    use async_std::task::sleep;
+
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
+    use crate::{
+        chat::{self, Chat, ChatItem},
+        dc_tools::IsNoneOrEmpty,
+    };
 
     #[async_std::test]
     async fn test_stock_ephemeral_messages() {
@@ -472,24 +509,78 @@ mod tests {
         );
 
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
-            "Message deletion timer is disabled."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 1 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 s by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
-            "Message deletion timer is set to 1 s."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 30 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 30 s by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
-            "Message deletion timer is set to 30 s."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 minute by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
-            "Message deletion timer is set to 1 minute."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 90 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1.5 minutes by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
-            "Message deletion timer is set to 1 hour."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 30 * 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 30 minutes by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 60 * 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 hour by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 5400 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1.5 hours by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 2 * 60 * 60
+                },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 2 hours by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -497,10 +588,21 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 1 day."
+            "Message deletion timer is set to 1 day by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 2 * 24 * 60 * 60
+                },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 2 days by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -508,10 +610,10 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 1 week."
+            "Message deletion timer is set to 1 week by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -519,10 +621,119 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 4 weeks."
+            "Message deletion timer is set to 4 weeks by me."
         );
     }
+
+    #[async_std::test]
+    async fn test_ephemeral_timer() -> anyhow::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
+
+        // Alice sends message to Bob
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+
+        // Alice sends second message to Bob, with no timer
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Disabled
+        );
+
+        // Bob sets ephemeral timer and sends a message about timer change
+        chat_bob
+            .set_ephemeral_timer(&bob.ctx, Timer::Enabled { duration: 60 })
+            .await?;
+        let sent_timer_change = bob.pop_sent_msg().await;
+
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        // Bob receives message from Alice.
+        // Alice message has no timer. However, Bob should not disable timer,
+        // because Alice replies to old message.
+        bob.recv_msg(&sent).await;
+
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Disabled
+        );
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        // Alice receives message from Bob
+        alice.recv_msg(&sent_timer_change).await;
+
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_ephemeral_delete_msgs() {
+        let t = TestContext::new_alice().await;
+        let chat = t.get_self_chat().await;
+
+        t.send_text(chat.id, "Saved message, which we delete manually")
+            .await;
+        let msg = t.get_last_msg_in(chat.id).await;
+        msg.id.delete_from_db(&t).await.unwrap();
+        check_msg_was_deleted(&t, &chat, msg.id).await;
+
+        chat.id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
+            .await
+            .unwrap();
+        let msg = t
+            .send_text(chat.id, "Saved message, disappearing after 1s")
+            .await;
+
+        sleep(Duration::from_millis(1100)).await;
+
+        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
+    }
+
+    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+        let chat_items = chat::get_chat_msgs(&t, chat.id, 0, None).await;
+        // Check that the chat is empty except for possibly info messages:
+        for item in &chat_items {
+            if let ChatItem::Message { msg_id } = item {
+                let msg = Message::load_from_db(t, *msg_id).await.unwrap();
+                assert!(msg.is_info())
+            }
+        }
+
+        // Check that if there is a message left, the text and metadata are gone
+        if let Ok(msg) = Message::load_from_db(&t, msg_id).await {
+            assert_eq!(msg.from_id, 0);
+            assert_eq!(msg.to_id, 0);
+            assert!(msg.text.is_none_or_empty(), msg.text);
+            let rawtxt: Option<String> = t
+                .sql
+                .query_get_value(&t, "SELECT txt_raw FROM msgs WHERE id=?;", paramsv![msg_id])
+                .await;
+            assert!(rawtxt.is_none_or_empty(), rawtxt);
+        }
+    }
 }

src/error.rs

@@ -1,7 +1,5 @@
 //! # Error handling
 
-pub use anyhow::{bail, ensure, format_err, Error, Result};
-
 #[macro_export]
 macro_rules! ensure_eq {
     ($left:expr, $right:expr) => ({

src/events.rs

@@ -2,8 +2,8 @@
 
 use std::ops::Deref;
 
+use async_std::channel::{self, Receiver, Sender, TrySendError};
 use async_std::path::PathBuf;
-use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
@@ -18,7 +18,7 @@ pub struct Events {
 
 impl Default for Events {
     fn default() -> Self {
-        let (sender, receiver) = channel(1_000);
+        let (sender, receiver) = channel::bounded(1_000);
 
         Self { receiver, sender }
     }
@@ -35,7 +35,7 @@ impl Events {
                 // try again
                 self.emit(event);
             }
-            Err(TrySendError::Disconnected(_)) => {
+            Err(TrySendError::Closed(_)) => {
                 unreachable!("unable to emit event, channel disconnected");
             }
         }
@@ -47,6 +47,17 @@ impl Events {
     }
 }
 
+/// A receiver of events from a [`Context`].
+///
+/// See [`Context::get_event_emitter`] to create an instance.  If multiple instances are
+/// created events emitted by the [`Context`] will only be delivered to one of the
+/// `EventEmitter`s.
+///
+/// The `EventEmitter` is also a [`Stream`], so a typical usage is in a `while let` loop.
+///
+/// [`Context`]: crate::context::Context
+/// [`Context::get_event_emitter`]: crate::context::Context::get_event_emitter
+/// [`Stream`]: async_std::stream::Stream
 #[derive(Debug, Clone)]
 pub struct EventEmitter(Receiver<Event>);
 
@@ -73,9 +84,27 @@ impl async_std::stream::Stream for EventEmitter {
     }
 }
 
+/// The event emitted by a [`Context`] from an [`EventEmitter`].
+///
+/// Events are documented on the C/FFI API in `deltachat.h` as `DC_EVENT_*` contants.  The
+/// context emits them in relation to various operations happening, a lot of these are again
+/// documented in `deltachat.h`.
+///
+/// This struct [`Deref`]s to the [`EventType`].
+///
+/// [`Context`]: crate::context::Context
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Event {
+    /// The ID of the [`Context`] which emitted this event.
+    ///
+    /// This allows using multiple [`Context`]s in a single process as they are identified
+    /// by this ID.
+    ///
+    /// [`Context`]: crate::context::Context
     pub id: u32,
+    /// The event payload.
+    ///
+    /// These are documented in `deltachat.h` as the `DC_EVENT_*` constants.
     pub typ: EventType,
 }
 
@@ -88,7 +117,9 @@ impl Deref for Event {
 }
 
 impl EventType {
-    /// Returns the corresponding Event id.
+    /// Returns the corresponding Event ID.
+    ///
+    /// These are the IDs used in the `DC_EVENT_*` constants in `deltachat.h`.
     pub fn as_id(&self) -> i32 {
         self.get_str("id")
             .expect("missing id")
@@ -100,8 +131,9 @@ impl EventType {
 #[derive(Debug, Clone, PartialEq, Eq, EnumProperty)]
 pub enum EventType {
     /// The library-user may write an informational string to the log.
-    /// Passed to the callback given to dc_context_new().
-    /// This event should not be reported to the end-user using a popup or something like that.
+    ///
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
     #[strum(props(id = "100"))]
     Info(String),
 
@@ -134,14 +166,13 @@ pub enum EventType {
     DeletedBlobFile(String),
 
     /// The library-user should write a warning string to the log.
-    /// Passed to the callback given to dc_context_new().
     ///
-    /// This event should not be reported to the end-user using a popup or something like that.
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
     #[strum(props(id = "300"))]
     Warning(String),
 
     /// The library-user should report an error to the end-user.
-    /// Passed to the callback given to dc_context_new().
     ///
     /// As most things are asynchronous, things may go wrong at any time and the user
     /// should not be disturbed by a dialog or so.  Instead, use a bubble or so.

src/headerdef.rs

@@ -21,6 +21,7 @@ pub enum HeaderDef {
     InReplyTo,
     Precedence,
     ContentType,
+    ContentId,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -43,6 +44,7 @@ pub enum HeaderDef {
     SecureJoinInvitenumber,
     SecureJoinAuth,
     EphemeralTimer,
+    Received,
     _TestHeader,
 }
 

src/html.rs

@@ -0,0 +1,569 @@
+///! # Get message as HTML.
+///!
+///! Use `Message.has_html()` to check if the UI shall render a
+///! corresponding button and `MsgId.get_html()` to get the full message.
+///!
+///! Even when the original mime-message is not HTML,
+///! `MsgId.get_html()` will return HTML -
+///! this allows nice quoting, handling linebreaks properly etc.
+use futures::future::FutureExt;
+use std::future::Future;
+use std::pin::Pin;
+
+use anyhow::Result;
+use lettre_email::mime::{self, Mime};
+
+use crate::context::Context;
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::message::{Message, MsgId};
+use crate::mimeparser::parse_message_id;
+use crate::param::Param::SendHtml;
+use crate::plaintext::PlainText;
+use lettre_email::PartBuilder;
+use mailparse::ParsedContentType;
+
+impl Message {
+    /// Check if the message can be retrieved as HTML.
+    /// Typically, this is the case, when the mime structure of a Message is modified,
+    /// meaning that some text is cut or the original message
+    /// is in HTML and `simplify()` may hide some maybe important information.
+    /// The corresponding ffi-function is `dc_msg_has_html()`.
+    /// To get the HTML-code of the message, use `MsgId.get_html()`.
+    pub fn has_html(&self) -> bool {
+        self.mime_modified
+    }
+
+    /// Set HTML-part part of a message that is about to be sent.
+    /// The HTML-part is written to the database before sending and
+    /// used as the `text/html` part in the MIME-structure.
+    ///
+    /// Received HTML parts are handled differently,
+    /// they are saved together with the whole MIME-structure
+    /// in `mime_headers` and the HTML-part is extracted using `MsgId::get_html()`.
+    /// (To underline this asynchronicity, we are using the wording "SendHtml")
+    pub fn set_html(&mut self, html: Option<String>) {
+        if let Some(html) = html {
+            self.param.set(SendHtml, html);
+            self.mime_modified = true;
+        } else {
+            self.param.remove(SendHtml);
+            self.mime_modified = false;
+        }
+    }
+}
+
+/// Type defining a rough mime-type.
+/// This is mainly useful on iterating
+/// to decide whether a mime-part has subtypes.
+enum MimeMultipartType {
+    Multiple,
+    Single,
+    Message,
+}
+
+/// Function takes a content type from a ParsedMail structure
+/// and checks and returns the rough mime-type.
+async fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
+    let mimetype = ctype.mimetype.to_lowercase();
+    if mimetype.starts_with("multipart") && ctype.params.get("boundary").is_some() {
+        MimeMultipartType::Multiple
+    } else if mimetype == "message/rfc822" {
+        MimeMultipartType::Message
+    } else {
+        MimeMultipartType::Single
+    }
+}
+
+/// HtmlMsgParser converts a mime-message to HTML.
+#[derive(Debug)]
+struct HtmlMsgParser {
+    pub html: String,
+    pub plain: Option<PlainText>,
+}
+
+impl HtmlMsgParser {
+    /// Function takes a raw mime-message string,
+    /// searches for the main-text part
+    /// and returns that as parser.html
+    pub async fn from_bytes(context: &Context, rawmime: &[u8]) -> Result<Self> {
+        let mut parser = HtmlMsgParser {
+            html: "".to_string(),
+            plain: None,
+        };
+
+        let parsedmail = mailparse::parse_mail(rawmime)?;
+
+        parser.collect_texts_recursive(context, &parsedmail).await?;
+
+        if parser.html.is_empty() {
+            if let Some(plain) = &parser.plain {
+                parser.html = plain.to_html().await;
+            }
+        } else {
+            parser.cid_to_data_recursive(context, &parsedmail).await?;
+        }
+
+        Ok(parser)
+    }
+
+    /// Function iterates over all mime-parts
+    /// and searches for text/plain and text/html parts and saves the
+    /// last one found
+    /// in the corresponding structure fields.
+    /// Usually, there is at most one plain-text and one HTML-text part.
+    fn collect_texts_recursive<'a>(
+        &'a mut self,
+        context: &'a Context,
+        mail: &'a mailparse::ParsedMail<'a>,
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
+        // Boxed future to deal with recursion
+        async move {
+            match get_mime_multipart_type(&mail.ctype).await {
+                MimeMultipartType::Multiple => {
+                    for cur_data in mail.subparts.iter() {
+                        self.collect_texts_recursive(context, cur_data).await?
+                    }
+                    Ok(())
+                }
+                MimeMultipartType::Message => {
+                    let raw = mail.get_body_raw()?;
+                    if raw.is_empty() {
+                        return Ok(());
+                    }
+                    let mail = mailparse::parse_mail(&raw).unwrap();
+                    self.collect_texts_recursive(context, &mail).await
+                }
+                MimeMultipartType::Single => {
+                    let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+                    if mimetype == mime::TEXT_HTML {
+                        if let Ok(decoded_data) = mail.get_body() {
+                            self.html = decoded_data;
+                            return Ok(());
+                        }
+                    } else if mimetype == mime::TEXT_PLAIN {
+                        if let Ok(decoded_data) = mail.get_body() {
+                            self.plain = Some(PlainText {
+                                text: decoded_data,
+                                flowed: if let Some(format) = mail.ctype.params.get("format") {
+                                    format.as_str().to_ascii_lowercase() == "flowed"
+                                } else {
+                                    false
+                                },
+                                delsp: if let Some(delsp) = mail.ctype.params.get("delsp") {
+                                    delsp.as_str().to_ascii_lowercase() == "yes"
+                                } else {
+                                    false
+                                },
+                            });
+                            return Ok(());
+                        }
+                    }
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+
+    /// Replace cid:-protocol by the data:-protocol where appropriate.
+    /// This allows the final html-file to be self-contained.
+    fn cid_to_data_recursive<'a>(
+        &'a mut self,
+        context: &'a Context,
+        mail: &'a mailparse::ParsedMail<'a>,
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
+        // Boxed future to deal with recursion
+        async move {
+            match get_mime_multipart_type(&mail.ctype).await {
+                MimeMultipartType::Multiple => {
+                    for cur_data in mail.subparts.iter() {
+                        self.cid_to_data_recursive(context, cur_data).await?;
+                    }
+                    Ok(())
+                }
+                MimeMultipartType::Message => {
+                    let raw = mail.get_body_raw()?;
+                    if raw.is_empty() {
+                        return Ok(());
+                    }
+                    let mail = mailparse::parse_mail(&raw).unwrap();
+                    self.cid_to_data_recursive(context, &mail).await
+                }
+                MimeMultipartType::Single => {
+                    let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+                    if mimetype.type_() == mime::IMAGE {
+                        if let Some(cid) = mail.headers.get_header_value(HeaderDef::ContentId) {
+                            if let Ok(cid) = parse_message_id(&cid) {
+                                if let Ok(replacement) = mimepart_to_data_url(&mail).await {
+                                    let re_string = format!(
+                                        "(<img[^>]*src[^>]*=[^>]*)(cid:{})([^>]*>)",
+                                        regex::escape(&cid)
+                                    );
+                                    match regex::Regex::new(&re_string) {
+                                        Ok(re) => {
+                                            self.html = re
+                                                .replace_all(
+                                                    &*self.html,
+                                                    format!("${{1}}{}${{3}}", replacement).as_str(),
+                                                )
+                                                .as_ref()
+                                                .to_string()
+                                        }
+                                        Err(e) => warn!(
+                                            context,
+                                            "Cannot create regex for cid: {} throws {}",
+                                            re_string,
+                                            e
+                                        ),
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+}
+
+/// Convert a mime part to a data: url as defined in [RFC 2397](https://tools.ietf.org/html/rfc2397).
+async fn mimepart_to_data_url(mail: &mailparse::ParsedMail<'_>) -> Result<String> {
+    let data = mail.get_body_raw()?;
+    let data = base64::encode(&data);
+    Ok(format!("data:{};base64,{}", mail.ctype.mimetype, data))
+}
+
+impl MsgId {
+    /// Get HTML from a message-id.
+    /// This requires `mime_headers` field to be set for the message;
+    /// this is the case at least when `Message.has_html()` returns true
+    /// (we do not save raw mime unconditionally in the database to save space).
+    /// The corresponding ffi-function is `dc_get_msg_html()`.
+    pub async fn get_html(self, context: &Context) -> Option<String> {
+        let rawmime: Option<String> = context
+            .sql
+            .query_get_value(
+                context,
+                "SELECT mime_headers FROM msgs WHERE id=?;",
+                paramsv![self],
+            )
+            .await;
+
+        if let Some(rawmime) = rawmime {
+            if !rawmime.is_empty() {
+                match HtmlMsgParser::from_bytes(context, rawmime.as_bytes()).await {
+                    Err(err) => {
+                        warn!(context, "get_html: parser error: {}", err);
+                        None
+                    }
+                    Ok(parser) => Some(parser.html),
+                }
+            } else {
+                warn!(context, "get_html: empty mime for {}", self);
+                None
+            }
+        } else {
+            warn!(context, "get_html: no mime for {}", self);
+            None
+        }
+    }
+}
+
+/// Wraps HTML text into a new text/html mimepart structure.
+///
+/// Used on forwarding messages to avoid leaking the original mime structure
+/// and also to avoid sending too much, maybe large data.
+pub async fn new_html_mimepart(html: String) -> PartBuilder {
+    PartBuilder::new()
+        .content_type(&"text/html; charset=utf-8".parse::<mime::Mime>().unwrap())
+        .body(html)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chat;
+    use crate::chat::forward_msgs;
+    use crate::config::Config;
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::message::MessengerMessage;
+    use crate::test_utils::TestContext;
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_unspecified() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+This message does not have Content-Type nor Subject.<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_iso88591() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_iso88591.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+message with a non-UTF-8 encoding: äöüßÄÖÜ<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_flowed() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_flowed.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(parser.plain.unwrap().flowed);
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+This line ends with a space and will be merged with the next one due to format=flowed.<br/>
+<br/>
+This line does not end with a space<br/>
+and will be wrapped as usual.<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_plain() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+mime-modified should not be set set as there is no html and no special stuff;<br/>
+although not being a delta-message.<br/>
+test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x27; :)<br/>
+<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+
+        // on windows, `\r\n` linends are returned from mimeparser,
+        // however, rust multiline-strings use just `\n`;
+        // therefore, we just remove `\r` before comparison.
+        assert_eq!(
+            parser.html.replace("\r", ""),
+            r##"
+<html>
+  <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
+</html>"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            r##"<html>
+  <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
+</html>
+
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_plain_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            r##"<html>
+  <p>
+    this is <b>html</b>
+  </p>
+</html>
+
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_apple_cid_jpg() {
+        // load raw mime html-data with related image-part (cid:)
+        // and make sure, Content-Id has angle-brackets that are removed correctly.
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/apple_cid_jpg.eml");
+        let test = String::from_utf8_lossy(raw);
+        assert!(test
+            .find("Content-Id: <8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box>")
+            .is_some());
+        assert!(test
+            .find("cid:8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box")
+            .is_some());
+        assert!(test.find("data:").is_none());
+
+        // parsing converts cid: to data:
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(parser.html.find("<html>").is_some());
+        assert!(parser.html.find("Content-Id:").is_none());
+        assert!(parser
+            .html
+            .find("data:image/jpeg;base64,/9j/4AAQ")
+            .is_some());
+        assert!(parser.html.find("cid:").is_none());
+    }
+
+    #[async_std::test]
+    async fn test_get_html_empty() {
+        let t = TestContext::new().await;
+        let msg_id = MsgId::new_unset();
+        assert!(msg_id.get_html(&t).await.is_none())
+    }
+
+    #[async_std::test]
+    async fn test_html_forwarding() {
+        // alice receives a non-delta html-message
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("2")).await.ok();
+        let chat = alice
+            .create_chat_with_contact("", "sender@testrun.org")
+            .await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::No);
+        assert!(!msg.is_forwarded());
+        assert!(msg.get_text().unwrap().find("this is plain").is_some());
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.find("this is <b>html</b>").is_some());
+
+        // alice: create chat with bob and forward received html-message there
+        let chat = alice.create_chat_with_contact("", "bob@example.net").await;
+        forward_msgs(&alice, &[msg.get_id()], chat.get_id())
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().find("this is plain").is_some());
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.find("this is <b>html</b>").is_some());
+
+        // bob: check that bob also got the html-part of the forwarded message
+        let bob = TestContext::new_bob().await;
+        let chat = bob.create_chat_with_contact("", "alice@example.com").await;
+        bob.recv_msg(&alice.pop_sent_msg().await).await;
+        let msg = bob.get_last_msg_in(chat.get_id()).await;
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().find("this is plain").is_some());
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&bob).await.unwrap();
+        assert!(html.find("this is <b>html</b>").is_some());
+    }
+
+    #[async_std::test]
+    async fn test_html_forwarding_encrypted() {
+        // Alice receives a non-delta html-message
+        // (`ShowEmails=1` lets Alice actually receive non-delta messages for known contacts,
+        // the contact is marked as known by creating a chat using `chat_with_contact()`)
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("1")).await.ok();
+        let chat = alice
+            .create_chat_with_contact("", "sender@testrun.org")
+            .await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+
+        // forward the message to saved-messages,
+        // this will encrypt the message as new_alice() has set up keys
+        let chat = alice.get_self_chat().await;
+        forward_msgs(&alice, &[msg.get_id()], chat.get_id())
+            .await
+            .unwrap();
+        let msg = alice.pop_sent_msg().await;
+
+        // receive the message on another device
+        let alice = TestContext::new_alice().await;
+        assert_eq!(alice.get_config_int(Config::ShowEmails).await, 0); // set to "1" above, make sure it is another db
+        alice.recv_msg(&msg).await;
+        let chat = alice.get_self_chat().await;
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.get_showpadlock());
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().find("this is plain").is_some());
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.find("this is <b>html</b>").is_some());
+    }
+
+    #[async_std::test]
+    async fn test_set_html() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // alice sends a message with html-part to bob
+        let chat_id = alice.create_chat(&bob).await.id;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("plain text".to_string()));
+        msg.set_html(Some("<b>html</b> text".to_string()));
+        assert!(msg.mime_modified);
+        chat::send_msg(&alice, chat_id, &mut msg).await.unwrap();
+
+        // check the message is written correctly to alice's db
+        let msg = alice.get_last_msg_in(chat_id).await;
+        assert_eq!(msg.get_text(), Some("plain text".to_string()));
+        assert!(!msg.is_forwarded());
+        assert!(msg.mime_modified);
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.find("<b>html</b> text").is_some());
+
+        // let bob receive the message
+        let chat_id = bob.create_chat(&alice).await.id;
+        bob.recv_msg(&alice.pop_sent_msg().await).await;
+        let msg = bob.get_last_msg_in(chat_id).await;
+        assert_eq!(msg.get_text(), Some("plain text".to_string()));
+        assert!(!msg.is_forwarded());
+        assert!(msg.mime_modified);
+        let html = msg.get_id().get_html(&bob).await.unwrap();
+        assert!(html.find("<b>html</b> text").is_some());
+    }
+}

src/imap/idle.rs

@@ -1,11 +1,11 @@
 use super::Imap;
 
+use anyhow::{bail, format_err, Result};
 use async_imap::extensions::idle::IdleResponse;
 use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
-use crate::error::{bail, format_err, Result};
 use crate::{context::Context, scheduler::InterruptInfo};
 
 use super::session::Session;
@@ -50,6 +50,12 @@ impl Imap {
                 return Ok(info);
             }
 
+            if let Ok(info) = self.idle_interrupt.try_recv() {
+                info!(context, "skip idle, got interrupt {:?}", info);
+                self.session = Some(session);
+                return Ok(info);
+            }
+
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
                 bail!("IMAP IDLE protocol failed to init/complete: {}", err);
@@ -62,14 +68,18 @@ impl Imap {
                 Interrupt(InterruptInfo),
             }
 
-            info!(context, "Idle entering wait-on-remote state");
+            info!(
+                context,
+                "{}: Idle entering wait-on-remote state",
+                watch_folder.as_deref().unwrap_or("None")
+            );
             let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
-                let probe_network = self.idle_interrupt.recv().await;
+                let info = self.idle_interrupt.recv().await;
 
                 // cancel imap idle connection properly
                 drop(interrupt);
 
-                Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+                Ok(Event::Interrupt(info.unwrap_or_default()))
             });
 
             match fut.await {
@@ -115,10 +125,12 @@ impl Imap {
         let fake_idle_start_time = SystemTime::now();
 
         // Do not poll, just wait for an interrupt when no folder is passed in.
-        if watch_folder.is_none() {
+        let watch_folder = if let Some(watch_folder) = watch_folder {
+            watch_folder
+        } else {
             info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
             return self.idle_interrupt.recv().await.unwrap_or_default();
-        }
+        };
         info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
         // check every minute if there are new messages
@@ -160,19 +172,17 @@ impl Imap {
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
 
-                    if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_new_messages(context, watch_folder, false).await {
-                            Ok(res) => {
-                                info!(context, "fetch_new_messages returned {:?}", res);
-                                if res {
-                                    break InterruptInfo::new(false, None);
-                                }
-                            }
-                            Err(err) => {
-                                error!(context, "could not fetch from folder: {}", err);
-                                self.trigger_reconnect()
+                    match self.fetch_new_messages(context, &watch_folder, false).await {
+                        Ok(res) => {
+                            info!(context, "fetch_new_messages returned {:?}", res);
+                            if res {
+                                break InterruptInfo::new(false, None);
                             }
                         }
+                        Err(err) => {
+                            error!(context, "could not fetch from folder: {:#}", err);
+                            self.trigger_reconnect()
+                        }
                     }
                 }
                 Event::Interrupt(info) => {

src/imap/scan_folders.rs

@@ -0,0 +1,106 @@
+use std::time::Instant;
+
+use anyhow::{Context as _, Result};
+
+use crate::config::Config;
+use crate::context::Context;
+use crate::imap::Imap;
+use async_std::prelude::*;
+
+use super::{get_folder_meaning, get_folder_meaning_by_name, FolderMeaning};
+
+impl Imap {
+    pub async fn scan_folders(&mut self, context: &Context) -> Result<()> {
+        // First of all, debounce to once per minute:
+        let mut last_scan = context.last_full_folder_scan.lock().await;
+        if let Some(last_scan) = *last_scan {
+            let elapsed_secs = last_scan.elapsed().as_secs();
+            let debounce_secs = context
+                .get_config_u64(Config::ScanAllFoldersDebounceSecs)
+                .await;
+
+            if elapsed_secs < debounce_secs {
+                info!(context, "Not scanning, we scanned {}s ago", elapsed_secs);
+                return Ok(());
+            }
+        }
+        info!(context, "Starting full folder scan");
+
+        self.setup_handle(context).await?;
+        let session = self.session.as_mut();
+        let session = session.context("scan_folders(): IMAP No Connection established")?;
+        let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
+        let watched_folders = get_watched_folders(context).await;
+
+        let mut sentbox_folder = None;
+        let mut spam_folder = None;
+
+        for folder in folders {
+            let folder = match folder {
+                Ok(f) => f,
+                Err(e) => {
+                    warn!(context, "Can't get folder: {}", e);
+                    continue;
+                }
+            };
+
+            let foldername = folder.name();
+            if watched_folders.contains(&foldername.to_string()) {
+                info!(
+                    context,
+                    "Not scanning folder {} as it is watched anyway", foldername
+                );
+                continue;
+            }
+            info!(context, "Scanning folder: {}", foldername);
+
+            let folder_meaning = get_folder_meaning(&folder);
+            let folder_name_meaning = get_folder_meaning_by_name(&foldername);
+
+            if folder_meaning == FolderMeaning::SentObjects {
+                // Always takes precedent
+                sentbox_folder = Some(folder.name().to_string());
+            } else if folder_meaning == FolderMeaning::Spam {
+                spam_folder = Some(folder.name().to_string());
+            } else if folder_name_meaning == FolderMeaning::SentObjects {
+                // only set iff none has been already set
+                if sentbox_folder.is_none() {
+                    sentbox_folder = Some(folder.name().to_string());
+                }
+            } else if folder_name_meaning == FolderMeaning::Spam && spam_folder.is_none() {
+                spam_folder = Some(folder.name().to_string());
+            }
+
+            if let Err(e) = self.fetch_new_messages(context, foldername, false).await {
+                warn!(context, "Can't fetch new msgs in scanned folder: {:#}", e);
+            }
+        }
+
+        context
+            .set_config(Config::ConfiguredSentboxFolder, sentbox_folder.as_deref())
+            .await?;
+        context
+            .set_config(Config::ConfiguredSpamFolder, spam_folder.as_deref())
+            .await?;
+
+        last_scan.replace(Instant::now());
+        Ok(())
+    }
+}
+
+async fn get_watched_folders(context: &Context) -> Vec<String> {
+    let mut res = Vec::new();
+    let folder_watched_configured = &[
+        (Config::SentboxWatch, Config::ConfiguredSentboxFolder),
+        (Config::MvboxWatch, Config::ConfiguredMvboxFolder),
+        (Config::InboxWatch, Config::ConfiguredInboxFolder),
+    ];
+    for (watched, configured) in folder_watched_configured {
+        if context.get_config_bool(*watched).await {
+            if let Some(folder) = context.get_config(*configured).await {
+                res.push(folder);
+            }
+        }
+    }
+    res
+}

src/imap/select_folder.rs

@@ -61,11 +61,12 @@ impl Imap {
 
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
+    /// Returns whether a new folder was selected.
     pub(super) async fn select_folder<S: AsRef<str>>(
         &mut self,
         context: &Context,
         folder: Option<S>,
-    ) -> Result<()> {
+    ) -> Result<NewlySelected> {
         if self.session.is_none() {
             self.config.selected_folder = None;
             self.config.selected_folder_needs_expunge = false;
@@ -78,7 +79,7 @@ impl Imap {
         if let Some(ref folder) = folder {
             if let Some(ref selected_folder) = self.config.selected_folder {
                 if folder.as_ref() == selected_folder {
-                    return Ok(());
+                    return Ok(NewlySelected::No);
                 }
             }
         }
@@ -99,7 +100,7 @@ impl Imap {
                     Ok(mailbox) => {
                         self.config.selected_folder = Some(folder.as_ref().to_string());
                         self.config.selected_mailbox = Some(mailbox);
-                        Ok(())
+                        Ok(NewlySelected::Yes)
                     }
                     Err(async_imap::error::Error::ConnectionLost) => {
                         self.trigger_reconnect();
@@ -119,7 +120,15 @@ impl Imap {
                 Err(Error::NoSession)
             }
         } else {
-            Ok(())
+            Ok(NewlySelected::No)
         }
     }
 }
+#[derive(PartialEq, Debug, Copy, Clone, Eq)]
+pub(super) enum NewlySelected {
+    /// The folder was newly selected during this call to select_folder().
+    Yes,
+    /// No SELECT command was run because the folder already was selected
+    /// and self.config.selected_mailbox was not updated (so, e.g. it may contain an outdated uid_next)
+    No,
+}

src/imex.rs

@@ -1,12 +1,9 @@
 //! # Import/export module
 
 use std::any::Any;
-use std::{
-    cmp::{max, min},
-    ffi::OsStr,
-};
+use std::ffi::OsStr;
 
-use anyhow::Context as _;
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use async_std::path::{Path, PathBuf};
 use async_std::{
     fs::{self, File},
@@ -18,19 +15,21 @@ use crate::blob::BlobObject;
 use crate::chat;
 use crate::chat::delete_and_reset_all_device_msgs;
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{
+    dc_copy_file, dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
+    dc_open_file_std, dc_read_file, dc_write_file, get_next_backup_path, time, EmailAddress,
+};
 use crate::e2ee;
-use crate::error::*;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::param::*;
+use crate::param::Param;
 use crate::pgp;
 use crate::sql::{self, Sql};
-use crate::stock::StockMessage;
+use crate::stock_str;
 use ::pgp::types::KeyTrait;
 use async_tar::Archive;
 
@@ -276,8 +275,8 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
-    let msg_subj = context.stock_str(StockMessage::AcSetupMsgSubject).await;
-    let msg_body = context.stock_str(StockMessage::AcSetupMsgBody).await;
+    let msg_subj = stock_str::ac_setup_msg_subject(context).await;
+    let msg_body = stock_str::ac_setup_msg_body(context).await;
     let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
     Ok(format!(
         concat!(
@@ -428,7 +427,7 @@ async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
 pub fn normalize_setup_code(s: &str) -> String {
     let mut out = String::new();
     for c in s.chars() {
-        if c >= '0' && c <= '9' {
+        if ('0'..='9').contains(&c) {
             out.push(c);
             if let 4 | 9 | 14 | 19 | 24 | 29 | 34 | 39 = out.len() {
                 out += "-"
@@ -456,9 +455,7 @@ async fn imex_inner(context: &Context, what: ImexMode, path: impl AsRef<Path>) -
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
 
-        // TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old().
-        // (now is 07/2020)
-        ImexMode::ExportBackup => export_backup_old(context, path).await,
+        ImexMode::ExportBackup => export_backup(context, path).await,
         // import_backup() will call import_backup_old() if this is an old backup.
         ImexMode::ImportBackup => import_backup(context, path).await,
     }
@@ -494,10 +491,20 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     );
 
     let backup_file = File::open(backup_to_import).await?;
+    let file_size = backup_file.metadata().await?.len();
     let archive = Archive::new(backup_file);
+
     let mut entries = archive.entries()?;
     while let Some(file) = entries.next().await {
         let f = &mut file?;
+
+        let current_pos = f.raw_file_position();
+        let progress = 1000 * current_pos / file_size;
+        if progress > 10 && progress < 1000 {
+            // We already emitted ImexProgress(10) above
+            context.emit_event(EventType::ImexProgress(progress as usize));
+        }
+
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
             // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
             f.unpack_in(context.get_blobdir()).await?;
@@ -506,7 +513,6 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
                 context.get_dbfile(),
             )
             .await?;
-            context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
         } else {
             // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
             f.unpack_in(context.get_blobdir()).await?;
@@ -642,7 +648,7 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
 async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
-    let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+    let (temp_path, dest_path) = get_next_backup_path(dir, now).await?;
     let _d = DeleteOnDrop(temp_path.clone());
 
     context
@@ -706,131 +712,37 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
         .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
         .await?;
 
-    context.emit_event(EventType::ImexProgress(500));
+    let read_dir: Vec<_> = fs::read_dir(context.get_blobdir()).await?.collect().await;
+    let count = read_dir.len();
+    let mut written_files = 0;
 
-    builder
-        .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
-        .await?;
+    for entry in read_dir.into_iter() {
+        let entry = entry?;
+        let name = entry.file_name();
+        if !entry.file_type().await?.is_file() {
+            warn!(
+                context,
+                "Export: Found dir entry {} that is not a file, ignoring",
+                name.to_string_lossy()
+            );
+            continue;
+        }
+        let mut file = File::open(entry.path()).await?;
+        let path_in_archive = PathBuf::from(BLOBS_BACKUP_NAME).join(name);
+        builder.append_file(path_in_archive, &mut file).await?;
+
+        written_files += 1;
+        let progress = 1000 * written_files / count;
+        if progress > 10 && progress < 1000 {
+            // We already emitted ImexProgress(10) above
+            emit_event!(context, EventType::ImexProgress(progress));
+        }
+    }
 
     builder.finish().await?;
     Ok(())
 }
 
-async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
-    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
-    // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
-    // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
-    let now = time();
-    let dest_path_filename = get_next_backup_path_old(dir, now).await?;
-    let dest_path_string = dest_path_filename.to_string_lossy().to_string();
-
-    sql::housekeeping(context).await;
-
-    context.sql.execute("VACUUM;", paramsv![]).await.ok();
-
-    // we close the database during the copy of the dbfile
-    context.sql.close().await;
-    info!(
-        context,
-        "Backup '{}' to '{}'.",
-        context.get_dbfile().display(),
-        dest_path_filename.display(),
-    );
-    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename).await;
-    context
-        .sql
-        .open(&context, &context.get_dbfile(), false)
-        .await?;
-
-    if !copied {
-        bail!(
-            "could not copy file from '{}' to '{}'",
-            context.get_dbfile().display(),
-            dest_path_string
-        );
-    }
-    let dest_sql = Sql::new();
-    dest_sql
-        .open(context, &dest_path_filename, false)
-        .await
-        .with_context(|| format!("could not open exported database {}", dest_path_string))?;
-
-    let res = match add_files_to_export(context, &dest_sql).await {
-        Err(err) => {
-            dc_delete_file(context, &dest_path_filename).await;
-            error!(context, "backup failed: {}", err);
-            Err(err)
-        }
-        Ok(()) => {
-            dest_sql
-                .set_raw_config_int(context, "backup_time", now as i32)
-                .await?;
-            context.emit_event(EventType::ImexFileWritten(dest_path_filename));
-            Ok(())
-        }
-    };
-    dest_sql.close().await;
-
-    Ok(res?)
-}
-
-async fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
-    // add all files as blobs to the database copy (this does not require
-    // the source to be locked, neigher the destination as it is used only here)
-    if !sql.table_exists("backup_blobs").await? {
-        sql.execute(
-            "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
-            paramsv![],
-        )
-        .await?;
-    }
-    // copy all files from BLOBDIR into backup-db
-    let mut total_files_cnt = 0;
-    let dir = context.get_blobdir();
-    let dir_handle = async_std::fs::read_dir(&dir).await?;
-    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count().await;
-
-    info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
-
-    sql.with_conn_async(|conn| async move {
-        // scan directory, pass 2: copy files
-        let mut dir_handle = async_std::fs::read_dir(&dir).await?;
-
-        let mut processed_files_cnt = 0;
-        while let Some(entry) = dir_handle.next().await {
-            let entry = entry?;
-            if context.shall_stop_ongoing().await {
-                return Ok(());
-            }
-            processed_files_cnt += 1;
-            let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
-            context.emit_event(EventType::ImexProgress(permille));
-
-            let name_f = entry.file_name();
-            let name = name_f.to_string_lossy();
-            if name.starts_with("delta-chat") && name.ends_with(".bak") {
-                continue;
-            }
-            info!(context, "EXPORT: copying filename={}", name);
-            let curr_path_filename = context.get_blobdir().join(entry.file_name());
-            if let Ok(buf) = dc_read_file(context, &curr_path_filename).await {
-                if buf.is_empty() {
-                    continue;
-                }
-                // bail out if we can't insert
-                let mut stmt = conn.prepare_cached(
-                    "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
-                )?;
-                stmt.execute(paramsv![name, buf])?;
-            }
-        }
-        Ok(())
-    })
-    .await?;
-
-    Ok(())
-}
-
 /*******************************************************************************
  * Classic key import
  ******************************************************************************/
@@ -990,8 +902,11 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+
     use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
-    use crate::test_utils::*;
+    use crate::stock_str::StockMessage;
+    use crate::test_utils::{alice_keypair, TestContext};
+
     use ::pgp::armor::BlockType;
 
     #[async_std::test]
@@ -999,7 +914,7 @@ mod tests {
         let t = TestContext::new().await;
 
         t.configure_alice().await;
-        let msg = render_setup_file(&t.ctx, "hello").await.unwrap();
+        let msg = render_setup_file(&t, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
         // In particular note the mixing of `\r\n` and `\n` depending
@@ -1016,12 +931,11 @@ mod tests {
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
+        t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
         t.configure_alice().await;
-        let msg = render_setup_file(&t.ctx, "pw").await.unwrap();
+        let msg = render_setup_file(&t, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
     }
@@ -1029,7 +943,7 @@ mod tests {
     #[async_std::test]
     async fn test_create_setup_code() {
         let t = TestContext::new().await;
-        let setupcode = create_setup_code(&t.ctx);
+        let setupcode = create_setup_code(&t);
         assert_eq!(setupcode.len(), 44);
         assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
         assert_eq!(setupcode.chars().nth(9).unwrap(), '-');

src/job.rs

@@ -2,34 +2,33 @@
 //!
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
-
 use std::fmt;
 use std::future::Future;
 
+use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
+use async_smtp::smtp::response::{Category, Code, Detail};
 use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
 
-use async_smtp::smtp::response::Category;
-use async_smtp::smtp::response::Code;
-use async_smtp::smtp::response::Detail;
-
-use crate::chat::{self, ChatId};
-use crate::config::Config;
-use crate::contact::Contact;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_delete_file, dc_read_file, time};
 use crate::ephemeral::load_imap_deletion_msgid;
-use crate::error::{bail, ensure, format_err, Error, Result};
 use crate::events::EventType;
-use crate::imap::*;
+use crate::imap::{Imap, ImapActionResult};
 use crate::location;
 use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
-use crate::param::*;
+use crate::param::{Param, Params};
 use crate::smtp::Smtp;
 use crate::{blob::BlobObject, contact::normalize_name, contact::Modifier, contact::Origin};
+use crate::{
+    chat::{self, Chat, ChatId, ChatItem},
+    constants::DC_CHAT_ID_DEADDROP,
+};
+use crate::{config::Config, constants::Blocked};
+use crate::{constants::Chattype, contact::Contact};
 use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
@@ -248,7 +247,7 @@ impl Job {
             info!(context, "smtp-sending out mime message:");
             println!("{}", String::from_utf8_lossy(&message));
         }
-        match smtp.send(context, recipients, message, job_id).await {
+        let status = match smtp.send(context, recipients, message, job_id).await {
             Err(crate::smtp::send::Error::SendError(err)) => {
                 // Remote error, retry later.
                 warn!(context, "SMTP failed to send: {}", err);
@@ -273,7 +272,7 @@ impl Job {
                                 // Other enhanced status codes, such as Postfix
                                 // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
                                 // are not ignored.
-                                response.message.get(0) == Some(&"5.5.0".to_string())
+                                response.first_word() == Some(&"5.5.0".to_string())
                             }
                             _ => false,
                         };
@@ -285,25 +284,30 @@ impl Job {
                             // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
                             // should definitely go here, because user has to open the link to
                             // resume message sending.
-                            let msg_id = MsgId::new(self.foreign_id);
-                            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
-                            match Message::load_from_db(context, msg_id).await {
-                                Ok(message) => {
-                                    chat::add_info_msg(context, message.chat_id, err.to_string())
-                                        .await
-                                }
-                                Err(e) => error!(
-                                    context,
-                                    "couldn't load chat_id to inform user about SMTP error: {}", e
-                                ),
-                            };
                             Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                         }
                     }
-                    async_smtp::smtp::error::Error::Transient(_) => {
+                    async_smtp::smtp::error::Error::Transient(ref response) => {
                         // We got a transient 4xx response from SMTP server.
                         // Give some time until the server-side error maybe goes away.
-                        Status::RetryLater
+
+                        if let Some(first_word) = response.first_word() {
+                            if first_word.ends_with(".1.1")
+                                || first_word.ends_with(".1.2")
+                                || first_word.ends_with(".1.3")
+                            {
+                                // Sometimes we receive transient errors that should be permanent.
+                                // Any extended smtp status codes like x.1.1, x.1.2 or x.1.3 that we
+                                // receive as a transient error are misconfigurations of the smtp server.
+                                // See https://tools.ietf.org/html/rfc3463#section-3.2
+                                info!(context, "Smtp-job #{} Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", self.job_id, first_word);
+                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } else {
+                                Status::RetryLater
+                            }
+                        } else {
+                            Status::RetryLater
+                        }
                     }
                     _ => {
                         if smtp.has_maybe_stale_connection().await {
@@ -336,7 +340,14 @@ impl Job {
                 job_try!(success_cb().await);
                 Status::Finished(Ok(()))
             }
+        };
+
+        if let Status::Finished(Err(err)) = &status {
+            // We couldn't send the message, so mark it as failed
+            let msg_id = MsgId::new(self.foreign_id);
+            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
         }
+        status
     }
 
     pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
@@ -508,11 +519,27 @@ impl Job {
         }
 
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await;
+        let server_folder = &job_try!(msg
+            .server_folder
+            .context("Can't move message out of folder if we don't know the current folder"));
+
+        let move_res = msg.id.needs_move(context, server_folder).await;
+        let dest_folder = match move_res {
+            Err(e) => {
+                warn!(context, "could not load dest folder: {}", e);
+                return Status::RetryLater;
+            }
+            Ok(None) => {
+                warn!(
+                    context,
+                    "msg {} does not need to be moved from {}", msg.id, server_folder
+                );
+                return Status::Finished(Ok(()));
+            }
+            Ok(Some(config)) => context.get_config(config).await,
+        };
 
         if let Some(dest_folder) = dest_folder {
-            let server_folder = msg.server_folder.as_ref().unwrap();
-
             match imap
                 .mv(context, server_folder, msg.server_uid, &dest_folder)
                 .await
@@ -654,6 +681,45 @@ impl Job {
                 }
             }
         }
+
+        // Make sure that if there now is a chat with a contact (created by an outgoing message), then group contact requests
+        // from this contact should also be unblocked.
+        // See https://github.com/deltachat/deltachat-core-rust/issues/2097.
+        for item in chat::get_chat_msgs(context, ChatId::new(DC_CHAT_ID_DEADDROP), 0, None).await {
+            if let ChatItem::Message { msg_id } = item {
+                let msg = match Message::load_from_db(context, msg_id).await {
+                    Err(e) => {
+                        warn!(context, "can't get msg: {:#}", e);
+                        return Status::RetryLater;
+                    }
+                    Ok(m) => m,
+                };
+                let chat = match Chat::load_from_db(context, msg.chat_id).await {
+                    Err(e) => {
+                        warn!(context, "can't get chat: {:#}", e);
+                        return Status::RetryLater;
+                    }
+                    Ok(c) => c,
+                };
+                match chat.typ {
+                    Chattype::Group | Chattype::Mailinglist => {
+                        // The next lines are actually what we do in
+                        let (test_normal_chat_id, test_normal_chat_id_blocked) =
+                            chat::lookup_by_contact_id(context, msg.from_id)
+                                .await
+                                .unwrap_or_default();
+
+                        if !test_normal_chat_id.is_unset()
+                            && test_normal_chat_id_blocked == Blocked::Not
+                        {
+                            chat.id.unblock(context).await;
+                        }
+                    }
+                    Chattype::Single | Chattype::Undefined => {}
+                }
+            }
+        }
+
         info!(context, "Done fetching existing messages.");
         Status::Finished(Ok(()))
     }
@@ -714,6 +780,7 @@ impl Job {
             // retry. If the message was moved, we will create another
             // job to mark the message as seen later. If it was
             // deleted, there is nothing to do.
+            info!(context, "Can't mark message as seen: No UID");
             ImapActionResult::Failed
         } else {
             imap.set_seen(context, folder, msg.server_uid).await
@@ -805,7 +872,8 @@ async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, fold
         return;
     };
     if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
-        warn!(context, "Could not select {}: {}", mailbox, e);
+        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+        warn!(context, "Could not select {}: {:#}", mailbox, e);
         return;
     }
     match imap.get_all_recipients(context).await {
@@ -1176,6 +1244,18 @@ pub async fn add(context: &Context, job: Job) {
     }
 }
 
+async fn load_housekeeping_job(context: &Context) -> Option<Job> {
+    let last_time = context.get_config_i64(Config::LastHousekeeping).await;
+
+    let next_time = last_time + (60 * 60 * 24);
+    if next_time <= time() {
+        kill_action(context, Action::Housekeeping).await;
+        Some(Job::new(Action::Housekeeping, 0, Params::new(), 0))
+    } else {
+        None
+    }
+}
+
 /// Load jobs from the database.
 ///
 /// Load jobs for this "[Thread]", i.e. either load SMTP jobs or load
@@ -1292,8 +1372,10 @@ LIMIT 1;
                 } else {
                     Some(job)
                 }
+            } else if let Some(job) = load_imap_deletion_job(context).await.unwrap_or_default() {
+                Some(job)
             } else {
-                load_imap_deletion_job(context).await.unwrap_or_default()
+                load_housekeeping_job(context).await
             }
         }
         Thread::Smtp => job,
@@ -1304,7 +1386,7 @@ LIMIT 1;
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     async fn insert_job(context: &Context, foreign_id: i64) {
         let now = time();
@@ -1333,18 +1415,19 @@ mod tests {
         // fails to load from the database instead of failing to load
         // all jobs.
         let t = TestContext::new().await;
-        insert_job(&t.ctx, -1).await; // This can not be loaded into Job struct.
+        insert_job(&t, -1).await; // This can not be loaded into Job struct.
         let jobs = load_next(
-            &t.ctx,
+            &t,
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
         .await;
-        assert!(jobs.is_none());
+        // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
+        assert!(jobs.unwrap().action == Action::Housekeeping);
 
-        insert_job(&t.ctx, 1).await;
+        insert_job(&t, 1).await;
         let jobs = load_next(
-            &t.ctx,
+            &t,
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
@@ -1356,10 +1439,10 @@ mod tests {
     async fn test_load_next_job_one() {
         let t = TestContext::new().await;
 
-        insert_job(&t.ctx, 1).await;
+        insert_job(&t, 1).await;
 
         let jobs = load_next(
-            &t.ctx,
+            &t,
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )

src/key.rs

@@ -12,7 +12,7 @@ use pgp::types::{KeyTrait, SecretKeyTrait};
 use thiserror::Error;
 
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::KeyGenType;
 use crate::context::Context;
 use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
 use crate::sql;
@@ -407,7 +407,7 @@ impl std::str::FromStr for Fingerprint {
         let hex_repr: String = input
             .to_uppercase()
             .chars()
-            .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
+            .filter(|&c| ('0'..='9').contains(&c) || ('A'..='F').contains(&c))
             .collect();
         let v: Vec<u8> = hex::decode(hex_repr)?;
         let fp = Fingerprint::new(v)?;
@@ -426,7 +426,7 @@ pub enum FingerprintError {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::{alice_keypair, TestContext};
 
     use std::error::Error;
 
@@ -558,31 +558,29 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         let alice = alice_keypair();
         let t = TestContext::new().await;
         t.configure_alice().await;
-        let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
+        let pubkey = SignedPublicKey::load_self(&t).await.unwrap();
         assert_eq!(alice.public, pubkey);
-        let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
+        let seckey = SignedSecretKey::load_self(&t).await.unwrap();
         assert_eq!(alice.secret, seckey);
     }
 
     #[async_std::test]
     async fn test_load_self_generate_public() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
-        let key = SignedPublicKey::load_self(&t.ctx).await;
+        let key = SignedPublicKey::load_self(&t).await;
         assert!(key.is_ok());
     }
 
     #[async_std::test]
     async fn test_load_self_generate_secret() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
-        let key = SignedSecretKey::load_self(&t.ctx).await;
+        let key = SignedSecretKey::load_self(&t).await;
         assert!(key.is_ok());
     }
 
@@ -591,17 +589,17 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         use std::thread;
 
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
-        let ctx = t.ctx.clone();
-        let ctx0 = ctx.clone();
-        let thr0 =
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
-        let ctx1 = ctx;
-        let thr1 =
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
+        let thr0 = {
+            let ctx = t.clone();
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+        };
+        let thr1 = {
+            let ctx = t.clone();
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+        };
         let res0 = thr0.join().unwrap();
         let res1 = thr1.join().unwrap();
         assert_eq!(res0.unwrap(), res1.unwrap());
@@ -618,12 +616,11 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         // Saving the same key twice should result in only one row in
         // the keypairs table.
         let t = TestContext::new().await;
-        let ctx = Arc::new(t.ctx);
+        let ctx = Arc::new(t);
 
-        let ctx1 = ctx.clone();
         let nrows = || async {
-            ctx1.sql
-                .query_get_value::<u32>(&ctx1, "SELECT COUNT(*) FROM keypairs;", paramsv![])
+            ctx.sql
+                .query_get_value::<u32>(&ctx, "SELECT COUNT(*) FROM keypairs;", paramsv![])
                 .await
                 .unwrap()
         };

src/keyring.rs

@@ -62,7 +62,7 @@ where
 mod tests {
     use super::*;
     use crate::key::{SignedPublicKey, SignedSecretKey};
-    use crate::test_utils::*;
+    use crate::test_utils::{alice_keypair, TestContext};
 
     #[test]
     fn test_keyring_add_keys() {
@@ -83,10 +83,10 @@ mod tests {
         t.configure_alice().await;
         let alice = alice_keypair();
 
-        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t).await.unwrap();
         assert_eq!(pub_ring.keys(), [alice.public]);
 
-        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t).await.unwrap();
         assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 }

src/lib.rs

@@ -3,7 +3,8 @@
     clippy::correctness,
     missing_debug_implementations,
     clippy::all,
-    clippy::indexing_slicing
+    clippy::indexing_slicing,
+    clippy::wildcard_imports
 )]
 #![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
@@ -72,10 +73,13 @@ pub mod qr;
 pub mod securejoin;
 mod simplify;
 mod smtp;
-pub mod stock;
+pub mod stock_str;
 mod token;
 #[macro_use]
 mod dehtml;
+mod color;
+pub mod html;
+pub mod plaintext;
 
 pub mod dc_receive_imf;
 pub mod dc_tools;

src/location.rs

@@ -1,20 +1,20 @@
 //! Location handling
 
+use anyhow::{ensure, Error};
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::chat::{self, ChatId};
 use crate::config::Config;
-use crate::constants::*;
-use crate::context::*;
-use crate::dc_tools::*;
-use crate::error::{ensure, Error};
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+use crate::context::Context;
+use crate::dc_tools::time;
 use crate::events::EventType;
 use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::param::*;
-use crate::stock::StockMessage;
+use crate::param::Params;
+use crate::stock_str;
 
 /// Location record
 #[derive(Debug, Clone, Default)]
@@ -193,7 +193,8 @@ impl Kml {
 pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
     let now = time();
     if !(seconds < 0 || chat_id.is_special()) {
-        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id).await;
+        let is_sending_locations_before =
+            is_sending_locations_to_chat(context, Some(chat_id)).await;
         if context
             .sql
             .execute(
@@ -212,19 +213,13 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
         {
             if 0 != seconds && !is_sending_locations_before {
                 let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some(
-                    context
-                        .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
-                        .await,
-                );
+                msg.text = Some(stock_str::msg_location_enabled(context).await);
                 msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
                 chat::send_msg(context, chat_id, &mut msg)
                     .await
                     .unwrap_or_default();
             } else if 0 == seconds && is_sending_locations_before {
-                let stock_str = context
-                    .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
-                    .await;
+                let stock_str = stock_str::msg_location_disabled(context).await;
                 chat::add_info_msg(context, chat_id, stock_str).await;
             }
             context.emit_event(EventType::ChatModified(chat_id));
@@ -255,15 +250,29 @@ async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool)
     };
 }
 
-pub async fn is_sending_locations_to_chat(context: &Context, chat_id: ChatId) -> bool {
-    context
-        .sql
-        .exists(
-            "SELECT id  FROM chats  WHERE (? OR id=?)   AND locations_send_until>?;",
-            paramsv![if chat_id.is_unset() { 1 } else { 0 }, chat_id, time()],
-        )
-        .await
-        .unwrap_or_default()
+/// Returns whether `chat_id` or any chat is sending locations.
+///
+/// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
+/// is sending locations.
+pub async fn is_sending_locations_to_chat(context: &Context, chat_id: Option<ChatId>) -> bool {
+    match chat_id {
+        Some(chat_id) => context
+            .sql
+            .exists(
+                "SELECT id  FROM chats  WHERE id=?  AND locations_send_until>?;",
+                paramsv![chat_id, time()],
+            )
+            .await
+            .unwrap_or_default(),
+        None => context
+            .sql
+            .exists(
+                "SELECT id  FROM chats  WHERE locations_send_until>?;",
+                paramsv![time()],
+            )
+            .await
+            .unwrap_or_default(),
+    }
 }
 
 pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
@@ -311,14 +320,22 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
 
 pub async fn get_range(
     context: &Context,
-    chat_id: ChatId,
-    contact_id: u32,
+    chat_id: Option<ChatId>,
+    contact_id: Option<u32>,
     timestamp_from: i64,
     mut timestamp_to: i64,
 ) -> Vec<Location> {
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
+    let (disable_chat_id, chat_id) = match chat_id {
+        Some(chat_id) => (0, chat_id),
+        None => (1, ChatId::new(0)), // this ChatId is unused
+    };
+    let (disable_contact_id, contact_id) = match contact_id {
+        Some(contact_id) => (0, contact_id),
+        None => (1, 0), // this contact_id is unused
+    };
     context
         .sql
         .query_map(
@@ -329,9 +346,9 @@ pub async fn get_range(
              AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
              ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
             paramsv![
-                if chat_id.is_unset() { 1 } else { 0 },
+                disable_chat_id,
                 chat_id,
-                if contact_id == 0 { 1 } else { 0 },
+                disable_contact_id,
                 contact_id as i32,
                 timestamp_from,
                 timestamp_to,
@@ -372,7 +389,12 @@ pub async fn get_range(
 }
 
 fn is_marker(txt: &str) -> bool {
-    txt.len() == 1 && !txt.starts_with(' ')
+    let mut chars = txt.chars();
+    if let Some(c) = chars.next() {
+        !c.is_whitespace() && chars.next().is_none()
+    } else {
+        false
+    }
 }
 
 /// Deletes all locations from the database.
@@ -711,9 +733,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
                 paramsv![chat_id],
             ).await);
 
-            let stock_str = context
-                .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
-                .await;
+            let stock_str = stock_str::msg_location_disabled(context).await;
             chat::add_info_msg(context, chat_id, stock_str).await;
             context.emit_event(EventType::ChatModified(chat_id));
         }
@@ -777,4 +797,13 @@ mod tests {
         assert!(locations_ref[0].accuracy.abs() < f64::EPSILON);
         assert_eq!(locations_ref[0].timestamp, timestamp);
     }
+
+    #[test]
+    fn test_is_marker() {
+        assert!(is_marker("f"));
+        assert!(!is_marker("foo"));
+        assert!(is_marker("🏠"));
+        assert!(!is_marker(" "));
+        assert!(!is_marker("\t"));
+    }
 }

src/login_param.rs

@@ -3,6 +3,7 @@
 use std::borrow::Cow;
 use std::fmt;
 
+use crate::provider::{get_provider_by_id, Provider};
 use crate::{context::Context, provider::Socket};
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
@@ -48,6 +49,7 @@ pub struct LoginParam {
     pub imap: ServerLoginParam,
     pub smtp: ServerLoginParam,
     pub server_flags: i32,
+    pub provider: Option<&'static Provider>,
 }
 
 impl LoginParam {
@@ -130,6 +132,12 @@ impl LoginParam {
             .await
             .unwrap_or_default();
 
+        let key = format!("{}provider", prefix);
+        let provider = sql
+            .get_raw_config(context, key)
+            .await
+            .and_then(|provider_id| get_provider_by_id(&provider_id));
+
         LoginParam {
             addr,
             imap: ServerLoginParam {
@@ -148,6 +156,7 @@ impl LoginParam {
                 security: send_security,
                 certificate_checks: smtp_certificate_checks,
             },
+            provider,
             server_flags,
         }
     }
@@ -216,6 +225,11 @@ impl LoginParam {
         sql.set_raw_config_int(context, key, self.server_flags)
             .await?;
 
+        if let Some(provider) = self.provider {
+            let key = format!("{}provider", prefix);
+            sql.set_raw_config(context, key, Some(provider.id)).await?;
+        }
+
         Ok(())
     }
 }

src/mimefactory.rs

@@ -1,24 +1,28 @@
+use anyhow::{bail, ensure, format_err, Error};
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
 
 use crate::blob::BlobObject;
 use crate::chat::{self, Chat};
 use crate::config::Config;
-use crate::constants::*;
-use crate::contact::*;
+use crate::constants::{Chattype, Viewtype, DC_FROM_HANDSHAKE};
+use crate::contact::Contact;
 use crate::context::{get_version_str, Context};
-use crate::dc_tools::*;
-use crate::e2ee::*;
+use crate::dc_tools::{
+    dc_create_outgoing_rfc724_mid, dc_create_smeared_timestamp, dc_get_filebytes, time,
+};
+use crate::e2ee::EncryptHelper;
 use crate::ephemeral::Timer as EphemeralTimer;
-use crate::error::{bail, ensure, format_err, Error};
 use crate::format_flowed::{format_flowed, format_flowed_quote};
+use crate::html::new_html_mimepart;
 use crate::location;
-use crate::message::{self, Message};
+use crate::message::{self, Message, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::param::*;
+use crate::param::Param;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::simplify::escape_message_footer_marks;
-use crate::stock::StockMessage;
+use crate::stock_str;
+use std::convert::TryInto;
 
 // attachments of 25 mb brutto should work on the majority of providers
 // (brutto examples: web.de=50, 1&1=40, t-online.de=32, gmail=25, posteo=50, yahoo=25, all-inkl=100).
@@ -135,10 +139,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             )
             .await?;
 
-        let default_str = context
-            .stock_str(StockMessage::StatusLine)
-            .await
-            .to_string();
+        let default_str = stock_str::status_line(context).await;
         let factory = MimeFactory {
             from_addr,
             from_displayname,
@@ -176,10 +177,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .get_config(Config::Displayname)
             .await
             .unwrap_or_default();
-        let default_str = context
-            .stock_str(StockMessage::StatusLine)
-            .await
-            .to_string();
+        let default_str = stock_str::status_line(context).await;
         let selfstatus = context
             .get_config(Config::Selfstatus)
             .await
@@ -341,10 +339,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         match self.loaded {
             Loaded::Message { ref chat } => {
                 if self.msg.param.get_cmd() == SystemMessage::AutocryptSetupMessage {
-                    self.context
-                        .stock_str(StockMessage::AcSetupMsgSubject)
-                        .await
-                        .into_owned()
+                    stock_str::ac_setup_msg_subject(self.context).await
                 } else if chat.typ == Chattype::Group {
                     let re = if self.in_reply_to.is_empty() {
                         ""
@@ -386,21 +381,12 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                                     .unwrap_or_default(),
                             };
 
-                            self.context
-                                .stock_string_repl_str(
-                                    StockMessage::SubjectForNewContact,
-                                    self_name,
-                                )
-                                .await
+                            stock_str::subject_for_new_contact(self.context, self_name).await
                         }
                     }
                 }
             }
-            Loaded::MDN { .. } => self
-                .context
-                .stock_str(StockMessage::ReadRcpt)
-                .await
-                .into_owned(),
+            Loaded::MDN { .. } => stock_str::read_rcpt(self.context).await,
         }
     }
 
@@ -751,11 +737,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     }
                 }
                 SystemMessage::GroupNameChanged => {
-                    let value_to_add = self.msg.param.get(Param::Arg).unwrap_or_default();
-
+                    let old_name = self.msg.param.get(Param::Arg).unwrap_or_default();
                     protected_headers.push(Header::new(
                         "Chat-Group-Name-Changed".into(),
-                        value_to_add.into(),
+                        maybe_encode_words(old_name),
                     ));
                 }
                 SystemMessage::GroupImageChanged => {
@@ -805,12 +790,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 unprotected_headers
                     .push(Header::new("Autocrypt-Setup-Message".into(), "v1".into()));
 
-                placeholdertext = Some(
-                    self.context
-                        .stock_str(StockMessage::AcSetupMsgBody)
-                        .await
-                        .to_string(),
-                );
+                placeholdertext = Some(stock_str::ac_setup_msg_body(self.context).await);
             }
             SystemMessage::SecurejoinMessage => {
                 let msg = &self.msg;
@@ -863,8 +843,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         if let Some(grpimage) = grpimage {
             info!(self.context, "setting group image '{}'", grpimage);
-            let mut meta = Message::default();
-            meta.viewtype = Viewtype::Image;
+            let mut meta = Message {
+                viewtype: Viewtype::Image,
+                ..Default::default()
+            };
             meta.param.set(Param::File, grpimage);
 
             let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image").await?;
@@ -951,7 +933,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         );
 
         // Message is sent as text/plain, with charset = utf-8
-        let main_part = PartBuilder::new()
+        let mut main_part = PartBuilder::new()
             .header((
                 "Content-Type".to_string(),
                 "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
@@ -959,6 +941,22 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .body(message_text);
         let mut parts = Vec::new();
 
+        // add HTML-part, this is needed only if a HTML-message from a non-delta-client is forwarded;
+        // for simplificity and to avoid conversion errors, we're generating the HTML-part from the original message.
+        if self.msg.has_html() {
+            let html = if let Some(orig_msg_id) = self.msg.param.get_int(Param::Forwarded) {
+                MsgId::new(orig_msg_id.try_into()?).get_html(context).await
+            } else {
+                self.msg.param.get(Param::SendHtml).map(|s| s.to_string())
+            };
+            if let Some(html) = html {
+                main_part = PartBuilder::new()
+                    .message_type(MimeMultipartType::Alternative)
+                    .child(main_part.build())
+                    .child(new_html_mimepart(html).await.build());
+            }
+        }
+
         // add attachment part
         if chat::msgtype_has_file(self.msg.viewtype) {
             if !is_file_size_okay(context, &self.msg).await {
@@ -980,7 +978,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             parts.push(msg_kml_part);
         }
 
-        if location::is_sending_locations_to_chat(context, self.msg.chat_id).await {
+        if location::is_sending_locations_to_chat(context, Some(self.msg.chat_id)).await {
             match self.get_location_kml_part().await {
                 Ok(part) => parts.push(part),
                 Err(err) => {
@@ -1050,17 +1048,11 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 .get_int(Param::GuaranteeE2ee)
                 .unwrap_or_default()
         {
-            self.context
-                .stock_str(StockMessage::EncryptedMsg)
-                .await
-                .into_owned()
+            stock_str::encrypted_msg(self.context).await
         } else {
             self.msg.get_summarytext(self.context, 32).await
         };
-        let p2 = self
-            .context
-            .stock_string_repl_str(StockMessage::ReadRcptMailBody, p1)
-            .await;
+        let p2 = stock_str::read_rcpt_mail_body(self.context, p1).await;
         let message_text = format!("{}\r\n", p2);
         message = message.child(
             PartBuilder::new()
@@ -1178,14 +1170,10 @@ async fn build_body_file(
     // at least on tested Thunderbird and Gma'l in 2017.
     // But I've heard about problems with inline and outl'k, so we just use the attachment-type until we
     // run into other problems ...
-    let cd_value = if needs_encoding(&filename_to_send) {
-        format!(
-            "attachment; filename*=\"{}\"",
-            encode_words(&filename_to_send)
-        )
-    } else {
-        format!("attachment; filename=\"{}\"", &filename_to_send)
-    };
+    let cd_value = format!(
+        "attachment; filename=\"{}\"",
+        maybe_encode_words(&filename_to_send)
+    );
 
     let body = std::fs::read(blob.to_abs_path())?;
     let encoded_body = wrapped_base64_encode(&body);
@@ -1268,18 +1256,27 @@ fn encode_words(word: &str) -> String {
     encoded_words::encode(word, None, encoded_words::EncodingFlag::Shortest, None)
 }
 
-pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
+fn needs_encoding(to_check: impl AsRef<str>) -> bool {
     !to_check.as_ref().chars().all(|c| {
         c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '~' || c == '%'
     })
 }
 
+fn maybe_encode_words(words: &str) -> String {
+    if needs_encoding(words) {
+        encode_words(words)
+    } else {
+        words.to_string()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::chatlist::Chatlist;
+    use crate::contact::Origin;
     use crate::dc_receive_imf::dc_receive_imf;
-    use crate::mimeparser::*;
+    use crate::mimeparser::MimeMessage;
     use crate::test_utils::TestContext;
 
     #[test]
@@ -1364,12 +1361,20 @@ mod tests {
         assert!(needs_encoding("foo bar"));
     }
 
+    #[test]
+    fn test_maybe_encode_words() {
+        assert_eq!(maybe_encode_words("foobar"), "foobar");
+        assert_eq!(maybe_encode_words("-_.~%"), "-_.~%");
+        assert_eq!(maybe_encode_words("äöü"), "=?utf-8?b?w6TDtsO8?=");
+    }
+
     #[async_std::test]
     async fn test_subject() {
         // 1.: Receive a mail from an MUA or Delta Chat
         assert_eq!(
             msg_to_subject_str(
-                b"From: Bob <bob@example.com>\n\
+                b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: Bob <bob@example.com>\n\
                 To: alice@example.com\n\
                 Subject: Antw: Chat: hello\n\
                 Message-ID: <2222@example.com>\n\
@@ -1383,7 +1388,8 @@ mod tests {
 
         assert_eq!(
             msg_to_subject_str(
-                b"From: Bob <bob@example.com>\n\
+                b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: Bob <bob@example.com>\n\
                 To: alice@example.com\n\
                 Subject: Infos: 42\n\
                 Message-ID: <2222@example.com>\n\
@@ -1398,7 +1404,8 @@ mod tests {
         // 2. Receive a message from Delta Chat when we did not send any messages before
         assert_eq!(
             msg_to_subject_str(
-                b"From: Charlie <charlie@example.com>\n\
+                b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: Charlie <charlie@example.com>\n\
                 To: alice@example.com\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
@@ -1417,15 +1424,15 @@ mod tests {
         assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
 
         let t = TestContext::new_alice().await;
-        t.ctx
-            .set_config(Config::Displayname, Some("Alice"))
+        t.set_config(Config::Displayname, Some("Alice"))
             .await
             .unwrap();
         assert_eq!(first_subject_str(t).await, "Message from Alice");
 
         // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
         msg_to_subject_str(
-            "From: Charlie <charlie@example.com>\n\
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+            From: Charlie <charlie@example.com>\n\
             To: alice@example.com\n\
             Subject: äääää\n\
             Chat-Version: 1.0\n\
@@ -1438,7 +1445,8 @@ mod tests {
         .await;
 
         msg_to_subject_str(
-            "From: Charlie <charlie@example.com>\n\
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+            From: Charlie <charlie@example.com>\n\
             To: alice@example.com\n\
             Subject: aäääää\n\
             Chat-Version: 1.0\n\
@@ -1453,8 +1461,9 @@ mod tests {
         // 5. Receive an mdn (read receipt) and make sure the mdn's subject is not used
         let t = TestContext::new_alice().await;
         dc_receive_imf(
-            &t.ctx,
-            b"From: alice@example.com\n\
+            &t,
+            b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+            From: alice@example.com\n\
             To: Charlie <charlie@example.com>\n\
             Subject: Hello, Charlie\n\
             Chat-Version: 1.0\n\
@@ -1468,7 +1477,9 @@ mod tests {
         )
         .await
         .unwrap();
-        let new_msg = incoming_msg_to_reply_msg(b"From: charlie@example.com\n\
+        let new_msg = incoming_msg_to_reply_msg(
+            b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                 From: charlie@example.com\n\
                  To: alice@example.com\n\
                  Subject: message opened\n\
                  Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
@@ -1491,45 +1502,35 @@ mod tests {
                  Final-Recipient: rfc822;charlie@example.com\n\
                  Original-Message-ID: <2893@example.com>\n\
                  Disposition: manual-action/MDN-sent-automatically; displayed\n\
-                 \n", &t.ctx).await;
-        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
-            .await
-            .unwrap();
+                 \n", &t).await;
+        let mf = MimeFactory::from_msg(&t, &new_msg, false).await.unwrap();
         // The subject string should not be "Re: message opened"
         assert_eq!("Re: Hello, Charlie", mf.subject_str().await);
     }
 
     async fn first_subject_str(t: TestContext) -> String {
         let contact_id =
-            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.com", Origin::ManuallyCreated)
+            Contact::add_or_lookup(&t, "Dave", "dave@example.com", Origin::ManuallyCreated)
                 .await
                 .unwrap()
                 .0;
 
-        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
-            .await
-            .unwrap();
+        let chat_id = chat::create_by_contact_id(&t, contact_id).await.unwrap();
 
         let mut new_msg = Message::new(Viewtype::Text);
         new_msg.set_text(Some("Hi".to_string()));
         new_msg.chat_id = chat_id;
-        chat::prepare_msg(&t.ctx, chat_id, &mut new_msg)
-            .await
-            .unwrap();
+        chat::prepare_msg(&t, chat_id, &mut new_msg).await.unwrap();
 
-        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
-            .await
-            .unwrap();
+        let mf = MimeFactory::from_msg(&t, &new_msg, false).await.unwrap();
 
         mf.subject_str().await
     }
 
     async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
         let t = TestContext::new_alice().await;
-        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
-        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
-            .await
-            .unwrap();
+        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t).await;
+        let mf = MimeFactory::from_msg(&t, &new_msg, false).await.unwrap();
         mf.subject_str().await
     }
 
@@ -1564,10 +1565,11 @@ mod tests {
     // This test could still be extended
     async fn test_render_reply() {
         let t = TestContext::new_alice().await;
-        let context = &t.ctx;
+        let context = &t;
 
         let msg = incoming_msg_to_reply_msg(
-            b"From: Charlie <charlie@example.com>\n\
+            b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: Charlie <charlie@example.com>\n\
                 To: alice@example.com\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
@@ -1579,7 +1581,7 @@ mod tests {
         )
         .await;
 
-        let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
+        let mimefactory = MimeFactory::from_msg(&t, &msg, false).await.unwrap();
 
         let recipients = mimefactory.recipients();
         assert_eq!(recipients, vec!["charlie@example.com"]);

src/mimeparser.rs

@@ -2,30 +2,32 @@ use std::collections::{HashMap, HashSet};
 use std::future::Future;
 use std::pin::Pin;
 
+use anyhow::{bail, Result};
+use charset::Charset;
 use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 use once_cell::sync::Lazy;
+use percent_encoding::percent_decode_str;
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
 use crate::constants::Viewtype;
-use crate::contact::*;
+use crate::contact::addr_normalize;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::dc_get_filemeta;
 use crate::dehtml::dehtml;
 use crate::e2ee;
-use crate::error::{bail, Result};
 use crate::events::EventType;
 use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
 use crate::location;
 use crate::message;
-use crate::param::*;
+use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
-use crate::simplify::*;
-use crate::stock::StockMessage;
+use crate::simplify::simplify;
+use crate::stock_str;
 
 /// A parsed MIME message.
 ///
@@ -63,6 +65,19 @@ pub struct MimeMessage {
     pub(crate) group_avatar: Option<AvatarAction>,
     pub(crate) mdn_reports: Vec<Report>,
     pub(crate) failure_report: Option<FailureReport>,
+
+    /// Standard USENET signature, if any.
+    pub(crate) footer: Option<String>,
+
+    // if this flag is set, the parts/text/etc. are just close to the original mime-message;
+    // clients should offer a way to view the original message in this case
+    pub is_mime_modified: bool,
+
+    /// The decrypted, raw mime structure.
+    ///
+    /// This is non-empty only if the message was actually encrypted.  It is used
+    /// for e.g. late-parsing HTML.
+    pub decoded_data: Vec<u8>,
 }
 
 #[derive(Debug, PartialEq)]
@@ -130,7 +145,7 @@ impl MimeMessage {
         headers.remove("chat-verified");
 
         // Memory location for a possible decrypted message.
-        let mail_raw;
+        let mut mail_raw = Vec::new();
         let mut gossipped_addr = Default::default();
 
         let (mail, signatures, warn_empty_signature) =
@@ -221,9 +236,12 @@ impl MimeMessage {
             user_avatar: None,
             group_avatar: None,
             failure_report: None,
+            footer: None,
+            is_mime_modified: false,
+            decoded_data: Vec::new(),
         };
         parser.parse_mime_recursive(context, &mail).await?;
-        parser.maybe_remove_bad_parts().await;
+        parser.maybe_remove_bad_parts();
         parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context)?;
 
@@ -233,6 +251,10 @@ impl MimeMessage {
             }
         }
 
+        if parser.is_mime_modified {
+            parser.decoded_data = mail_raw;
+        }
+
         Ok(parser)
     }
 
@@ -383,23 +405,21 @@ impl MimeMessage {
                     prepend_subject = false
                 }
             }
-            if prepend_subject {
-                let subj = subject
-                    .find('[')
-                    .and_then(|n| subject.get(..n))
-                    .unwrap_or(subject)
-                    .trim();
 
-                if !subj.is_empty() {
-                    for part in self.parts.iter_mut() {
-                        if part.typ == Viewtype::Text {
-                            part.msg = format!("{} – {}", subj, part.msg);
-                            break;
-                        }
-                    }
+            // For mailing lists, always add the subject because sometimes there are different topics
+            // and otherwise it might be hard to keep track:
+            if self.get(HeaderDef::ListId).is_some() {
+                prepend_subject = true;
+            }
+
+            if prepend_subject && !subject.is_empty() {
+                let part_with_text = self.parts.iter_mut().find(|part| !part.msg.is_empty());
+                if let Some(mut part) = part_with_text {
+                    part.msg = format!("{} – {}", subject, part.msg);
                 }
             }
         }
+
         if self.is_forwarded {
             for part in self.parts.iter_mut() {
                 part.param.set_int(Param::Forwarded, 1);
@@ -412,10 +432,15 @@ impl MimeMessage {
         if !self.decrypting_failed && !self.parts.is_empty() {
             if let Some(ref dn_to) = self.chat_disposition_notification_to {
                 if let Some(ref from) = self.from.get(0) {
-                    if from.addr == dn_to.addr {
+                    if from.addr.to_lowercase() == dn_to.addr.to_lowercase() {
                         if let Some(part) = self.parts.last_mut() {
                             part.param.set_int(Param::WantsMdn, 1);
                         }
+                    } else {
+                        warn!(
+                            context,
+                            "{} requested a read receipt to {}, ignoring", from.addr, dn_to.addr
+                        );
                     }
                 }
             }
@@ -426,8 +451,10 @@ impl MimeMessage {
         // Besides, we want to show something in case our incoming-processing
         // failed to properly handle an incoming message.
         if self.parts.is_empty() && self.mdn_reports.is_empty() {
-            let mut part = Part::default();
-            part.typ = Viewtype::Text;
+            let mut part = Part {
+                typ: Viewtype::Text,
+                ..Default::default()
+            };
 
             if let Some(ref subject) = self.get_subject() {
                 if !self.has_chat_version() {
@@ -591,20 +618,27 @@ impl MimeMessage {
                         }
                     }
                 }
+                if any_part_added && mail.subparts.len() > 1 {
+                    // there are other alternative parts, likely HTML,
+                    // so we might have missed some content on simplifying.
+                    // set mime-modified to force the ui to display a show-message button.
+                    self.is_mime_modified = true;
+                }
             }
             (mime::MULTIPART, "encrypted") => {
                 // we currently do not try to decrypt non-autocrypt messages
                 // at all. If we see an encrypted part, we set
                 // decrypting_failed.
-                let msg_body = context.stock_str(StockMessage::CantDecryptMsgBody).await;
+                let msg_body = stock_str::cant_decrypt_msg_body(context).await;
                 let txt = format!("[{}]", msg_body);
 
-                let mut part = Part::default();
-                part.typ = Viewtype::Text;
-                part.msg_raw = Some(txt.clone());
-                part.msg = txt;
-                part.error = Some("Decryption failed".to_string());
-
+                let part = Part {
+                    typ: Viewtype::Text,
+                    msg_raw: Some(txt.clone()),
+                    msg: txt,
+                    error: Some("Decryption failed".to_string()),
+                    ..Default::default()
+                };
                 self.parts.push(part);
 
                 any_part_added = true;
@@ -636,8 +670,10 @@ impl MimeMessage {
                             // downloading the message again and
                             // delete if automatic message deletion is
                             // enabled.
-                            let mut part = Part::default();
-                            part.typ = Viewtype::Unknown;
+                            let part = Part {
+                                typ: Viewtype::Unknown,
+                                ..Default::default()
+                            };
                             self.parts.push(part);
 
                             any_part_added = true;
@@ -648,7 +684,7 @@ impl MimeMessage {
                                 self.failure_report = Some(report);
                             }
 
-                            // Add all parts (we need another part, preferrably text/plain, to show as an error message)
+                            // Add all parts (we need another part, preferably text/plain, to show as an error message)
                             for cur_data in mail.subparts.iter() {
                                 if self.parse_mime_recursive(context, cur_data).await? {
                                     any_part_added = true;
@@ -686,7 +722,7 @@ impl MimeMessage {
         let (mime_type, msg_type) = get_mime_type(mail)?;
         let raw_mime = mail.ctype.mimetype.to_lowercase();
 
-        let filename = get_attachment_filename(mail)?;
+        let filename = get_attachment_filename(context, mail)?;
 
         let old_part_count = self.parts.len();
 
@@ -720,20 +756,26 @@ impl MimeMessage {
 
                         let mut dehtml_failed = false;
 
-                        let (simplified_txt, is_forwarded, top_quote) = if decoded_data.is_empty() {
-                            ("".to_string(), false, None)
-                        } else {
-                            let is_html = mime_type == mime::TEXT_HTML;
-                            let out = if is_html {
-                                dehtml(&decoded_data).unwrap_or_else(|| {
-                                    dehtml_failed = true;
-                                    decoded_data.clone()
-                                })
+                        let (simplified_txt, is_forwarded, is_cut, top_quote, footer) =
+                            if decoded_data.is_empty() {
+                                ("".to_string(), false, false, None, None)
                             } else {
-                                decoded_data.clone()
+                                let is_html = mime_type == mime::TEXT_HTML;
+                                let out = if is_html {
+                                    self.is_mime_modified = true;
+                                    dehtml(&decoded_data).unwrap_or_else(|| {
+                                        dehtml_failed = true;
+                                        decoded_data.clone()
+                                    })
+                                } else {
+                                    decoded_data.clone()
+                                };
+                                simplify(out, self.has_chat_version())
                             };
-                            simplify(out, self.has_chat_version())
-                        };
+
+                        self.is_mime_modified = self.is_mime_modified
+                            || ((is_forwarded || is_cut || top_quote.is_some())
+                                && !self.has_chat_version());
 
                         let is_format_flowed = if let Some(format) = mail.ctype.params.get("format")
                         {
@@ -759,11 +801,13 @@ impl MimeMessage {
                         };
 
                         if !simplified_txt.is_empty() || simplified_quote.is_some() {
-                            let mut part = Part::default();
-                            part.dehtlm_failed = dehtml_failed;
-                            part.typ = Viewtype::Text;
-                            part.mimetype = Some(mime_type);
-                            part.msg = simplified_txt;
+                            let mut part = Part {
+                                dehtml_failed,
+                                typ: Viewtype::Text,
+                                mimetype: Some(mime_type),
+                                msg: simplified_txt,
+                                ..Default::default()
+                            };
                             if let Some(quote) = simplified_quote {
                                 part.param.set(Param::Quote, quote);
                             }
@@ -774,6 +818,8 @@ impl MimeMessage {
                         if is_forwarded {
                             self.is_forwarded = true;
                         }
+
+                        self.footer = footer;
                     }
                     _ => {}
                 }
@@ -1003,13 +1049,13 @@ impl MimeMessage {
         Ok(None)
     }
 
-    async fn maybe_remove_bad_parts(&mut self) {
-        let good_parts = self.parts.iter().filter(|p| !p.dehtlm_failed).count();
+    fn maybe_remove_bad_parts(&mut self) {
+        let good_parts = self.parts.iter().filter(|p| !p.dehtml_failed).count();
         if good_parts == 0 {
             // We have no good part but show at least one bad part in order to show anything at all
             self.parts.truncate(1);
         } else if good_parts < self.parts.len() {
-            self.parts.retain(|p| !p.dehtlm_failed);
+            self.parts.retain(|p| !p.dehtml_failed);
         }
     }
 
@@ -1216,7 +1262,7 @@ pub struct Part {
     pub param: Params,
     org_filename: Option<String>,
     pub error: Option<String>,
-    dehtlm_failed: bool,
+    dehtml_failed: bool,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1270,51 +1316,96 @@ fn is_attachment_disposition(mail: &mailparse::ParsedMail<'_>) -> bool {
 
 /// Tries to get attachment filename.
 ///
-/// If filename is explitictly specified in Content-Disposition, it is
+/// If filename is explicitly specified in Content-Disposition, it is
 /// returned. If Content-Disposition is "attachment" but filename is
 /// not specified, filename is guessed. If Content-Disposition cannot
 /// be parsed, returns an error.
-fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String>> {
-    // try to get file name from
-    //    `Content-Disposition: ... filename*=...`
-    // or `Content-Disposition: ... filename*0*=... filename*1*=... filename*2*=...`
-    // or `Content-Disposition: ... filename=...`
-
+fn get_attachment_filename(
+    context: &Context,
+    mail: &mailparse::ParsedMail,
+) -> Result<Option<String>> {
     let ct = mail.get_content_disposition();
 
-    let desired_filename: Option<String> = ct
-        .params
-        .iter()
-        .filter(|(key, _value)| key.starts_with("filename"))
-        .fold(None, |acc, (_key, value)| {
-            if let Some(acc) = acc {
-                Some(acc + value)
-            } else {
-                Some(value.to_string())
-            }
-        });
+    // try to get file name as "encoded-words" from
+    // `Content-Disposition: ... filename=...`
+    let mut desired_filename = ct.params.get("filename").map(|s| s.to_string());
 
-    let desired_filename =
-        desired_filename.or_else(|| ct.params.get("name").map(|s| s.to_string()));
+    // try to get file name from
+    // `Content-Disposition: ... filename*0*=... filename*1*=... filename*2*=...`
+    // encoded as CHARSET'LANG'test%2E%70%64%66 (key ends with `*`)
+    // or as "encoded-words" (key does not end with `*`)
+    if desired_filename.is_none() {
+        let mut apostrophe_encoded = false;
+        desired_filename = ct
+            .params
+            .iter()
+            .filter(|(key, _value)| key.starts_with("filename*"))
+            .fold(None, |acc, (key, value)| {
+                if key.ends_with('*') {
+                    apostrophe_encoded = true;
+                }
+                if let Some(acc) = acc {
+                    Some(acc + value)
+                } else {
+                    Some(value.to_string())
+                }
+            });
+        if apostrophe_encoded {
+            if let Some(name) = desired_filename {
+                let mut parts = name.splitn(3, '\'');
+                desired_filename =
+                    if let (Some(charset), Some(value)) = (parts.next(), parts.last()) {
+                        let decoded_bytes = percent_decode_str(&value);
+                        if charset.to_lowercase() == "utf-8" {
+                            Some(decoded_bytes.decode_utf8_lossy().to_string())
+                        } else {
+                            // encoded_words crate say, latin-1 is not reported; moreover, latin1 is a good default
+                            if let Some(charset) = Charset::for_label(charset.as_bytes())
+                                .or_else(|| Charset::for_label(b"latin1"))
+                            {
+                                let decoded_bytes = decoded_bytes.collect::<Vec<u8>>();
+                                let (utf8_str, _, _) = charset.decode(&*decoded_bytes);
+                                Some(utf8_str.into())
+                            } else {
+                                warn!(context, "latin1 encoding does not exist");
+                                None
+                            }
+                        }
+                    } else {
+                        warn!(context, "apostrophed encoding invalid: {}", name);
+                        // be graceful and just use the original name.
+                        // some MUA, including Delta Chat up to core1.50,
+                        // use `filename*` mistakenly for simple encoded-words without following rfc2231
+                        Some(name)
+                    }
+            }
+        }
+    }
+
+    // if no filename is set, try `Content-Disposition: ... name=...`
+    if desired_filename.is_none() {
+        desired_filename = ct.params.get("name").map(|s| s.to_string());
+    }
 
     // MS Outlook is known to specify filename in the "name" attribute of
     // Content-Type and omit Content-Disposition.
-    let desired_filename =
-        desired_filename.or_else(|| mail.ctype.params.get("name").map(|s| s.to_string()));
+    if desired_filename.is_none() {
+        desired_filename = mail.ctype.params.get("name").map(|s| s.to_string());
+    }
 
     // If there is no filename, but part is an attachment, guess filename
-    if ct.disposition == DispositionType::Attachment && desired_filename.is_none() {
+    if desired_filename.is_none() && ct.disposition == DispositionType::Attachment {
         if let Some(subtype) = mail.ctype.mimetype.split('/').nth(1) {
-            Ok(Some(format!("file.{}", subtype,)))
+            desired_filename = Some(format!("file.{}", subtype,));
         } else {
             bail!(
                 "could not determine attachment filename: {:?}",
                 ct.disposition
             );
-        }
-    } else {
-        Ok(desired_filename)
+        };
     }
+
+    Ok(desired_filename)
 }
 
 /// Returned addresses are normalized and lowercased.
@@ -1368,7 +1459,15 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
-    use crate::test_utils::*;
+    use crate::{
+        chatlist::Chatlist,
+        config::Config,
+        constants::Blocked,
+        dc_receive_imf::dc_receive_imf,
+        message::{Message, MessageState, MessengerMessage},
+        test_utils::TestContext,
+    };
+    use mailparse::ParsedMail;
 
     impl AvatarAction {
         pub fn is_change(&self) -> bool {
@@ -1379,6 +1478,64 @@ mod tests {
         }
     }
 
+    #[async_std::test]
+    async fn test_mimeparser_fromheader() {
+        let ctx = TestContext::new_alice().await;
+
+        let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi")
+            .await
+            .unwrap();
+        let contact = mimemsg.from.first().unwrap();
+        assert_eq!(contact.addr, "g@c.de");
+        assert_eq!(contact.display_name, None);
+
+        let mimemsg = MimeMessage::from_bytes(&ctx, b"From:   g@c.de  \n\nhi")
+            .await
+            .unwrap();
+        let contact = mimemsg.from.first().unwrap();
+        assert_eq!(contact.addr, "g@c.de");
+        assert_eq!(contact.display_name, None);
+
+        let mimemsg = MimeMessage::from_bytes(&ctx, b"From: <g@c.de>\n\nhi")
+            .await
+            .unwrap();
+        let contact = mimemsg.from.first().unwrap();
+        assert_eq!(contact.addr, "g@c.de");
+        assert_eq!(contact.display_name, None);
+
+        let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C <g@c.de>\n\nhi")
+            .await
+            .unwrap();
+        let contact = mimemsg.from.first().unwrap();
+        assert_eq!(contact.addr, "g@c.de");
+        assert_eq!(contact.display_name, Some("Goetz C".to_string()));
+
+        let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" <g@c.de>\n\nhi")
+            .await
+            .unwrap();
+        let contact = mimemsg.from.first().unwrap();
+        assert_eq!(contact.addr, "g@c.de");
+        assert_eq!(contact.display_name, Some("Goetz C".to_string()));
+
+        let mimemsg =
+            MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C <g@c.de>\n\nhi")
+                .await
+                .unwrap();
+        let contact = mimemsg.from.first().unwrap();
+        assert_eq!(contact.addr, "g@c.de");
+        assert_eq!(contact.display_name, Some("Götz C".to_string()));
+
+        // although RFC 2047 says, encoded-words shall not appear inside quoted-string,
+        // this combination is used in the wild eg. by MailMate
+        let mimemsg =
+            MimeMessage::from_bytes(&ctx, b"From: \"=?utf-8?q?G=C3=B6tz?= C\" <g@c.de>\n\nhi")
+                .await
+                .unwrap();
+        let contact = mimemsg.from.first().unwrap();
+        assert_eq!(contact.addr, "g@c.de");
+        assert_eq!(contact.display_name, Some("Götz C".to_string()));
+    }
+
     #[async_std::test]
     async fn test_dc_mimeparser_crash() {
         let context = TestContext::new().await;
@@ -1438,18 +1595,157 @@ mod tests {
         assert!(is_attachment_disposition(&mail.subparts[1]));
     }
 
-    #[test]
-    fn test_get_attachment_filename() {
-        let raw = include_bytes!("../test-data/message/html_attach.eml");
+    fn load_mail_with_attachment<'a>(t: &'a TestContext, raw: &'a [u8]) -> ParsedMail<'a> {
         let mail = mailparse::parse_mail(raw).unwrap();
-        assert!(get_attachment_filename(&mail).unwrap().is_none());
-        assert!(get_attachment_filename(&mail.subparts[0])
+        assert!(get_attachment_filename(&t, &mail).unwrap().is_none());
+        assert!(get_attachment_filename(&t, &mail.subparts[0])
             .unwrap()
             .is_none());
-        let filename = get_attachment_filename(&mail.subparts[1]).unwrap();
+        mail
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_simple.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
         assert_eq!(filename, Some("test.html".to_string()))
     }
 
+    #[async_std::test]
+    async fn test_get_attachment_filename_encoded_words() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_encoded_words.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some("Maßnahmen Okt. 2020.html".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_encoded_words_binary() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_encoded_words_binary.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some(" § 165 Abs".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_encoded_words_windows1251() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_encoded_words_windows1251.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some("file Что нового 2020.pdf".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_encoded_words_cont() {
+        // test continued encoded-words and also test apostropes work that way
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_encoded_words_cont.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some("Maßn'ah'men Okt. 2020.html".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_encoded_words_bad_delimiter() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_encoded_words_bad_delimiter.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        // not decoded as a space is missing after encoded-words part
+        assert_eq!(filename, Some("=?utf-8?q?foo?=.bar".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_apostrophed() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_apostrophed.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some("Maßnahmen Okt. 2021.html".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_apostrophed_cont() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_apostrophed_cont.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some("Maßnahmen März 2022.html".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_apostrophed_windows1251() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_apostrophed_windows1251.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some("программирование.HTM".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_apostrophed_cp1252() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_apostrophed_cp1252.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some("Auftragsbestätigung.pdf".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_apostrophed_invalid() {
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_apostrophed_invalid.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some("somedäüta.html.zip".to_string()))
+    }
+
+    #[async_std::test]
+    async fn test_get_attachment_filename_combined() {
+        // test that if `filename` and `filename*0` are given, the filename is not doubled
+        let t = TestContext::new().await;
+        let mail = load_mail_with_attachment(
+            &t,
+            include_bytes!("../test-data/message/attach_filename_combined.eml"),
+        );
+        let filename = get_attachment_filename(&t, &mail.subparts[1]).unwrap();
+        assert_eq!(filename, Some("Maßnahmen Okt. 2020.html".to_string()))
+    }
+
+    #[test]
+    fn test_charset_latin1() {
+        // make sure, latin1 exists under this name
+        // as we're using it as default in get_attachment_filename() for non-utf-8
+        assert!(Charset::for_label(b"latin1").is_some());
+    }
+
     #[test]
     fn test_mailparse_content_type() {
         let ctype =
@@ -1572,26 +1868,26 @@ mod tests {
         let t = TestContext::new().await;
 
         let raw = include_bytes!("../test-data/message/mail_attach_txt.eml");
-        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
         assert_eq!(mimeparser.user_avatar, None);
         assert_eq!(mimeparser.group_avatar, None);
 
         let raw = include_bytes!("../test-data/message/mail_with_user_avatar.eml");
-        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
         assert!(mimeparser.user_avatar.unwrap().is_change());
         assert_eq!(mimeparser.group_avatar, None);
 
         let raw = include_bytes!("../test-data/message/mail_with_user_avatar_deleted.eml");
-        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
         assert_eq!(mimeparser.user_avatar, Some(AvatarAction::Delete));
         assert_eq!(mimeparser.group_avatar, None);
 
         let raw = include_bytes!("../test-data/message/mail_with_user_and_group_avatars.eml");
-        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
         assert!(mimeparser.user_avatar.unwrap().is_change());
@@ -1601,9 +1897,7 @@ mod tests {
         let raw = include_bytes!("../test-data/message/mail_with_user_and_group_avatars.eml");
         let raw = String::from_utf8_lossy(raw).to_string();
         let raw = raw.replace("Chat-User-Avatar:", "Xhat-Xser-Xvatar:");
-        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw.as_bytes())
-            .await
-            .unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t, raw.as_bytes()).await.unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Image);
         assert_eq!(mimeparser.user_avatar, None);
@@ -1615,7 +1909,7 @@ mod tests {
         let t = TestContext::new().await;
 
         let raw = include_bytes!("../test-data/message/videochat_invitation.eml");
-        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::VideochatInvitation);
         assert_eq!(
@@ -1895,7 +2189,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
 
         assert_eq!(message.parts.len(), 1);
         assert_eq!(message.parts[0].typ, Viewtype::File);
-        assert_eq!(message.parts[0].msg, "Hello!");
+        assert_eq!(message.parts[0].msg, "Mail with inline attachment – Hello!");
     }
 
     #[async_std::test]
@@ -1929,7 +2223,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
 ------=_Part_25_46172632.1581201680436--
 "#;
 
-        let message = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        let message = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
 
         assert_eq!(message.parts.len(), 1);
         assert_eq!(message.parts[0].typ, Viewtype::File);
@@ -1938,7 +2232,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
         // Make sure the file is there even though the html is wrong:
         let param = &message.parts[0].param;
         let blob: BlobObject = param
-            .get_blob(Param::File, &t.ctx, false)
+            .get_blob(Param::File, &t, false)
             .await
             .unwrap()
             .unwrap();
@@ -1990,7 +2284,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
 
         assert_eq!(message.parts.len(), 1);
         assert_eq!(message.parts[0].typ, Viewtype::Image);
-        assert_eq!(message.parts[0].msg, "Test");
+        assert_eq!(message.parts[0].msg, "example – Test");
     }
 
     #[async_std::test]
@@ -2062,7 +2356,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
 
         assert_eq!(message.parts.len(), 1);
         assert_eq!(message.parts[0].typ, Viewtype::Image);
-        assert_eq!(message.parts[0].msg, "Test");
+        assert_eq!(message.parts[0].msg, "Test subject – Test");
     }
 
     // Outlook specifies filename in the "name" attribute of Content-Type
@@ -2136,7 +2430,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
 
         assert_eq!(message.parts.len(), 1);
         assert_eq!(message.parts[0].typ, Viewtype::Image);
-        assert_eq!(message.parts[0].msg, "Test");
+        assert_eq!(message.parts[0].msg, "Delta Chat is great stuff! – Test");
     }
 
     #[test]
@@ -2321,4 +2615,111 @@ On 2020-10-25, Bob wrote:
         );
         assert_eq!(mimeparser.parts[0].typ, Viewtype::File);
     }
+
+    #[async_std::test]
+    async fn test_quote_div() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/gmx-quote.eml");
+        let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap();
+        assert_eq!(mimeparser.parts[0].msg, "YIPPEEEEEE\n\nMulti-line");
+        assert_eq!(mimeparser.parts[0].param.get(Param::Quote).unwrap(), "Now?");
+    }
+
+    #[async_std::test]
+    async fn test_add_subj_to_multimedia_msg() {
+        let t = TestContext::new_alice().await;
+        t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
+        dc_receive_imf(
+            &t.ctx,
+            include_bytes!("../test-data/message/subj_with_multimedia_msg.eml"),
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+
+        assert_eq!(
+            msg.text.as_ref().unwrap(),
+            "subj with important info – body text"
+        );
+        assert_eq!(msg.viewtype, Viewtype::Image);
+        assert_eq!(msg.error(), None);
+        assert_eq!(msg.is_dc_message, MessengerMessage::No);
+        assert_eq!(msg.chat_blocked, Blocked::Deaddrop);
+        assert_eq!(msg.state, MessageState::InFresh);
+        assert_eq!(msg.get_filebytes(&t).await, 2115);
+        assert!(msg.get_file(&t).is_some());
+        assert_eq!(msg.get_filename().unwrap(), "avatar64x64.png");
+        assert_eq!(msg.get_width(), 64);
+        assert_eq!(msg.get_height(), 64);
+        assert_eq!(msg.get_filemime().unwrap(), "image/png");
+    }
+
+    #[async_std::test]
+    async fn test_mime_modified_plain() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(!mimeparser.is_mime_modified);
+        assert_eq!(
+            mimeparser.parts[0].msg,
+            "This message does not have Content-Type nor Subject."
+        );
+    }
+
+    #[async_std::test]
+    async fn test_mime_modified_alt_plain_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(mimeparser.is_mime_modified);
+        assert_eq!(
+            mimeparser.parts[0].msg,
+            "mime-modified test – this is plain"
+        );
+    }
+
+    #[async_std::test]
+    async fn test_mime_modified_alt_plain() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(!mimeparser.is_mime_modified);
+        assert_eq!(
+            mimeparser.parts[0].msg,
+            "mime-modified test – \
+        mime-modified should not be set set as there is no html and no special stuff;\n\
+        although not being a delta-message.\n\
+        test some special html-characters as < > and & but also \" and ' :)"
+        );
+    }
+
+    #[async_std::test]
+    async fn test_mime_modified_alt_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_html.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(mimeparser.is_mime_modified);
+        assert_eq!(
+            mimeparser.parts[0].msg,
+            "mime-modified test – mime-modified *set*; simplify is always regarded as lossy."
+        );
+    }
+
+    #[async_std::test]
+    async fn test_mime_modified_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_html.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(mimeparser.is_mime_modified);
+        assert_eq!(
+            mimeparser.parts[0].msg,
+            "mime-modified test – mime-modified *set*; simplify is always regarded as lossy."
+        );
+    }
 }

src/oauth2.rs

@@ -1,14 +1,12 @@
 //! OAuth 2 module
 
-use regex::Regex;
 use std::collections::HashMap;
 
-use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::time;
 use crate::provider;
 use crate::provider::Oauth2Authorizer;
 
@@ -19,7 +17,6 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
-    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -29,11 +26,8 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
-    mx_pattern: None,
 };
 
-const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
-
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -41,7 +35,6 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
-    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -276,47 +269,16 @@ impl Oauth2 {
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            if let Some(provider) = provider::get_provider_info(&addr_normalized) {
-                match &provider.oauth2_authorizer {
-                    Some(Oauth2Authorizer::Gmail) => Some(OAUTH2_GMAIL),
-                    Some(Oauth2Authorizer::Yandex) => Some(OAUTH2_YANDEX),
-                    None => None, // provider known to not support oauth2, no mx-lookup required
-                }
-            } else {
-                Oauth2::lookup_mx(domain).await
-            }
-        } else {
-            None
-        }
-    }
-
-    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
-        if let Ok(resolver) = resolver(
-            config::ResolverConfig::default(),
-            config::ResolverOpts::default(),
-        )
-        .await
-        {
-            for provider in OAUTH2_PROVIDERS.iter() {
-                if let Some(pattern) = provider.mx_pattern {
-                    let re = Regex::new(pattern).unwrap();
-
-                    let mut fqdn: String = String::from(domain.as_ref());
-                    if !fqdn.ends_with('.') {
-                        fqdn.push('.');
-                    }
-
-                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
-                        for rr in res.iter() {
-                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
-                                return Some(provider.clone());
-                            }
-                        }
-                    }
-                }
+            if let Some(oauth2_authorizer) = provider::get_provider_info(&domain)
+                .await
+                .and_then(|provider| provider.oauth2_authorizer.as_ref())
+            {
+                return Some(match oauth2_authorizer {
+                    Oauth2Authorizer::Gmail => OAUTH2_GMAIL,
+                    Oauth2Authorizer::Yandex => OAUTH2_YANDEX,
+                });
             }
         }
-
         None
     }
 
@@ -386,7 +348,7 @@ fn normalize_addr(addr: &str) -> &str {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_normalize_addr() {

src/param.rs

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
 use std::fmt;
 use std::str;
 
+use anyhow::{bail, Error};
 use async_std::path::PathBuf;
 use itertools::Itertools;
 use num_traits::FromPrimitive;
@@ -9,7 +10,6 @@ use serde::{Deserialize, Serialize};
 
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
-use crate::error::{self, bail};
 use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 
@@ -22,6 +22,10 @@ pub enum Param {
     /// For messages and jobs
     File = b'f',
 
+    /// For messages: This name should be shown instead of contact.get_display_name()
+    /// (used if this is a mailinglist)
+    OverrideSenderDisplayname = b'O',
+
     /// For Messages
     Width = b'w',
 
@@ -34,6 +38,11 @@ pub enum Param {
     /// For Messages
     MimeType = b'm',
 
+    /// For Messages: HTML to be written to the database and to be send.
+    /// `SendHtml` param is not used for received messages.
+    /// Use `MsgId::get_html()` to get HTML of received messages.
+    SendHtml = b'T',
+
     /// For Messages: message is encrypted, outgoing: guarantee E2EE or the message is not send
     GuaranteeE2ee = b'c',
 
@@ -50,7 +59,8 @@ pub enum Param {
     /// For Messages
     WantsMdn = b'r',
 
-    /// For Messages
+    /// For Messages: unset or 0=not forwarded,
+    /// 1=forwarded from unknown msg_id, >9 forwarded from msg_id
     Forwarded = b'a',
 
     /// For Messages: quoted text.
@@ -121,15 +131,6 @@ pub enum Param {
     /// For Chats
     Devicetalk = b'D',
 
-    /// For QR
-    Auth = b's',
-
-    /// For QR
-    GroupId = b'x',
-
-    /// For QR
-    GroupName = b'g',
-
     /// For MDN-sending job
     MsgId = b'I',
 }
@@ -162,7 +163,7 @@ impl fmt::Display for Params {
 }
 
 impl str::FromStr for Params {
-    type Err = error::Error;
+    type Err = Error;
 
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let mut inner = BTreeMap::new();
@@ -379,7 +380,7 @@ mod tests {
     use async_std::fs;
     use async_std::path::Path;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_dc_param() {
@@ -425,18 +426,10 @@ mod tests {
         assert_eq!(params.to_string().parse::<Params>().unwrap(), params);
     }
 
-    #[test]
-    fn test_regression() {
-        let p1: Params = "a=cli%40deltachat.de\nn=\ni=TbnwJ6lSvD5\ns=0ejvbdFSQxB"
-            .parse()
-            .unwrap();
-        assert_eq!(p1.get(Param::Forwarded).unwrap(), "cli%40deltachat.de");
-    }
-
     #[async_std::test]
     async fn test_params_file_fs_path() {
         let t = TestContext::new().await;
-        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
+        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t, "/foo/bar/baz").unwrap() {
             assert_eq!(p, Path::new("/foo/bar/baz"));
         } else {
             panic!("Wrong enum variant");
@@ -446,7 +439,7 @@ mod tests {
     #[async_std::test]
     async fn test_params_file_blob() {
         let t = TestContext::new().await;
-        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
+        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t, "$BLOBDIR/foo").unwrap() {
             assert_eq!(b.as_name(), "$BLOBDIR/foo");
         } else {
             panic!("Wrong enum variant");
@@ -461,51 +454,33 @@ mod tests {
         let mut p = Params::new();
         p.set(Param::File, fname.to_str().unwrap());
 
-        let file = p.get_file(Param::File, &t.ctx).unwrap().unwrap();
+        let file = p.get_file(Param::File, &t).unwrap().unwrap();
         assert_eq!(file, ParamsFile::FsPath(fname.clone().into()));
 
-        let path: PathBuf = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        let path: PathBuf = p.get_path(Param::File, &t).unwrap().unwrap();
         let fname: PathBuf = fname.into();
         assert_eq!(path, fname);
 
         // Blob does not exist yet, expect BlobError.
-        let err = p.get_blob(Param::File, &t.ctx, false).await.unwrap_err();
+        let err = p.get_blob(Param::File, &t, false).await.unwrap_err();
         match err {
             BlobError::WrongBlobdir { .. } => (),
             _ => panic!("wrong error type/variant: {:?}", err),
         }
 
         fs::write(fname, b"boo").await.unwrap();
-        let blob = p
-            .get_blob(Param::File, &t.ctx, true)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(
-            blob,
-            BlobObject::from_name(&t.ctx, "foo".to_string()).unwrap()
-        );
+        let blob = p.get_blob(Param::File, &t, true).await.unwrap().unwrap();
+        assert_eq!(blob, BlobObject::from_name(&t, "foo".to_string()).unwrap());
 
         // Blob in blobdir, expect blob.
-        let bar_path = t.ctx.get_blobdir().join("bar");
+        let bar_path = t.get_blobdir().join("bar");
         p.set(Param::File, bar_path.to_str().unwrap());
-        let blob = p
-            .get_blob(Param::File, &t.ctx, false)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(
-            blob,
-            BlobObject::from_name(&t.ctx, "bar".to_string()).unwrap()
-        );
+        let blob = p.get_blob(Param::File, &t, false).await.unwrap().unwrap();
+        assert_eq!(blob, BlobObject::from_name(&t, "bar".to_string()).unwrap());
 
         p.remove(Param::File);
-        assert!(p.get_file(Param::File, &t.ctx).unwrap().is_none());
-        assert!(p.get_path(Param::File, &t.ctx).unwrap().is_none());
-        assert!(p
-            .get_blob(Param::File, &t.ctx, false)
-            .await
-            .unwrap()
-            .is_none());
+        assert!(p.get_file(Param::File, &t).unwrap().is_none());
+        assert!(p.get_path(Param::File, &t).unwrap().is_none());
+        assert!(p.get_blob(Param::File, &t, false).await.unwrap().is_none());
     }
 }

src/peerstate.rs

@@ -1,18 +1,19 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
+
 use std::collections::HashSet;
 use std::fmt;
 
+use anyhow::{bail, Result};
 use num_traits::FromPrimitive;
 
-use crate::aheader::*;
+use crate::aheader::{Aheader, EncryptPreference};
 use crate::chat;
 use crate::constants::Blocked;
 use crate::context::Context;
-use crate::error::{bail, Result};
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 #[derive(Debug)]
 pub enum PeerstateKeyType {
@@ -119,7 +120,15 @@ impl<'a> Peerstate<'a> {
             addr: gossip_header.addr.clone(),
             last_seen: 0,
             last_seen_autocrypt: 0,
-            prefer_encrypt: Default::default(),
+
+            // Non-standard extension. According to Autocrypt 1.1.0 gossip headers SHOULD NOT
+            // contain encryption preference.
+            //
+            // Delta Chat includes encryption preference to ensure new users introduced to a group
+            // learn encryption preferences of other members immediately and don't send unencrypted
+            // messages to a group where everyone prefers encryption.
+            prefer_encrypt: gossip_header.prefer_encrypt,
+
             public_key: None,
             public_key_fingerprint: None,
             gossip_key: Some(gossip_header.public_key.clone()),
@@ -272,9 +281,7 @@ impl<'a> Peerstate<'a> {
                         .await
                         .unwrap_or_default();
 
-                let msg = context
-                    .stock_string_repl_str(StockMessage::ContactSetupChanged, self.addr.clone())
-                    .await;
+                let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
 
                 chat::add_info_msg(context, contact_chat_id, msg).await;
                 emit_event!(context, EventType::ChatModified(contact_chat_id));
@@ -487,7 +494,7 @@ impl From<crate::key::FingerprintError> for rusqlite::Error {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::alice_keypair;
     use pretty_assertions::assert_eq;
 
     #[async_std::test]

src/pgp.rs

@@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashSet};
 use std::io;
 use std::io::Cursor;
 
+use anyhow::{bail, ensure, format_err, Result};
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
@@ -17,7 +18,6 @@ use rand::{thread_rng, CryptoRng, Rng};
 
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
-use crate::error::{bail, ensure, format_err, Result};
 use crate::key::{DcKey, Fingerprint};
 use crate::keyring::Keyring;
 
@@ -380,7 +380,7 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::{alice_keypair, bob_keypair};
     use once_cell::sync::Lazy;
 
     #[test]

src/plaintext.rs

@@ -0,0 +1,252 @@
+///! Handle plain text together with some attributes.
+use crate::simplify::split_lines;
+use once_cell::sync::Lazy;
+
+#[derive(Debug)]
+pub struct PlainText {
+    pub text: String,
+
+    /// Text may "flowed" as defined in [RFC 2646](https://tools.ietf.org/html/rfc2646).
+    /// At a glance, that means, if a line ends with a space, it is merged with the next one
+    /// and the first leading spaces is ignored
+    /// (to allow lines starting with `>` that normally indicates a quote)
+    pub flowed: bool,
+
+    /// If set together with "flowed",
+    /// The space indicating merging two lines is removed.
+    pub delsp: bool,
+}
+
+impl PlainText {
+    /// Convert plain text to HTML.
+    /// The function handles quotes, links, fixed and floating text paragraphs.
+    pub async fn to_html(&self) -> String {
+        static LINKIFY_MAIL_RE: Lazy<regex::Regex> =
+            Lazy::new(|| regex::Regex::new(r#"\b([\w.\-+]+@[\w.\-]+)\b"#).unwrap());
+
+        static LINKIFY_URL_RE: Lazy<regex::Regex> = Lazy::new(|| {
+            regex::Regex::new(r#"\b((http|https|ftp|ftps):[\w.,:;$/@!?&%\-~=#+]+)"#).unwrap()
+        });
+
+        let lines = split_lines(&self.text);
+
+        let mut ret =
+            "<!DOCTYPE html>\n<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" /></head><body>\n".to_string();
+
+        for line in lines {
+            let is_quote = line.starts_with('>');
+
+            // we need to do html-entity-encoding after linkify, as otherwise encapsulated links
+            // as <http://example.org> cannot be handled correctly
+            // (they would become &lt;http://example.org&gt; where the trailing &gt; would become a valid url part).
+            // to avoid double encoding, we escape our html-entities by \r that must not be used in the string elsewhere.
+            let line = line.to_string().replace("\r", "");
+
+            let mut line = LINKIFY_MAIL_RE
+                .replace_all(&*line, "\rLTa href=\rQUOTmailto:$1\rQUOT\rGT$1\rLT/a\rGT")
+                .as_ref()
+                .to_string();
+
+            line = LINKIFY_URL_RE
+                .replace_all(&*line, "\rLTa href=\rQUOT$1\rQUOT\rGT$1\rLT/a\rGT")
+                .as_ref()
+                .to_string();
+
+            // encode html-entities after linkify the raw string
+            line = escaper::encode_minimal(&line);
+
+            // make our escaped html-entities real after encoding all others
+            line = line.replace("\rLT", "<");
+            line = line.replace("\rGT", ">");
+            line = line.replace("\rQUOT", "\"");
+
+            if self.flowed {
+                // flowed text as of RFC 3676 -
+                // a leading space shall be removed
+                // and is only there to allow > at the beginning of a line that is no quote.
+                line = line.strip_prefix(" ").unwrap_or(&line).to_string();
+                if is_quote {
+                    line = "<em>".to_owned() + &line + "</em>";
+                }
+
+                // a trailing space indicates that the line can be merged with the next one;
+                // for sake of simplicity, we skip merging for quotes (quotes may be combined with
+                // delsp, so `> >` is different from `>>` etc. see RFC 3676 for details)
+                if line.ends_with(' ') && !is_quote {
+                    if self.delsp {
+                        line.pop();
+                    }
+                } else {
+                    line += "<br/>\n";
+                }
+            } else {
+                // normal, fixed text
+                if is_quote {
+                    line = "<em>".to_owned() + &line + "</em>";
+                }
+                line += "<br/>\n";
+            }
+
+            ret += &*line;
+        }
+        ret += "</body></html>\n";
+        ret
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[async_std::test]
+    async fn test_plain_to_html() {
+        let html = PlainText {
+            text: r##"line 1
+line 2
+line with https://link-mid-of-line.org and http://link-end-of-line.com/file?foo=bar%20
+http://link-at-start-of-line.org
+"##
+            .to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line 1<br/>
+line 2<br/>
+line with <a href="https://link-mid-of-line.org">https://link-mid-of-line.org</a> and <a href="http://link-end-of-line.com/file?foo=bar%20">http://link-end-of-line.com/file?foo=bar%20</a><br/>
+<a href="http://link-at-start-of-line.org">http://link-at-start-of-line.org</a><br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_encapsulated() {
+        let html = PlainText {
+            text: r#"line with <http://encapsulated.link/?foo=_bar> here!"#.to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line with &lt;<a href="http://encapsulated.link/?foo=_bar">http://encapsulated.link/?foo=_bar</a>&gt; here!<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_nolink() {
+        let html = PlainText {
+            text: r#"line with nohttp://no.link here"#.to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line with nohttp://no.link here<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_mailto() {
+        let html = PlainText {
+            text: r#"just an address: foo@bar.org another@one.de"#.to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+just an address: <a href="mailto:foo@bar.org">foo@bar.org</a> <a href="mailto:another@one.de">another@one.de</a><br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_flowed() {
+        let html = PlainText {
+            text: "line \nstill line\n>quote \n>still quote\n >no quote".to_string(),
+            flowed: true,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line still line<br/>
+<em>&gt;quote </em><br/>
+<em>&gt;still quote</em><br/>
+&gt;no quote<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_flowed_delsp() {
+        let html = PlainText {
+            text: "line \nstill line\n>quote \n>still quote\n >no quote".to_string(),
+            flowed: true,
+            delsp: true,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+linestill line<br/>
+<em>&gt;quote </em><br/>
+<em>&gt;still quote</em><br/>
+&gt;no quote<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_fixed() {
+        let html = PlainText {
+            text: "line \nstill line\n>quote \n>still quote\n >no quote".to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line <br/>
+still line<br/>
+<em>&gt;quote </em><br/>
+<em>&gt;still quote</em><br/>
+ &gt;no quote<br/>
+</body></html>
+"#
+        );
+    }
+}

src/provider/data.rs

@@ -3,13 +3,14 @@
 use crate::provider::Protocol::*;
 use crate::provider::Socket::*;
 use crate::provider::UsernamePattern::*;
-use crate::provider::*;
+use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status};
 use std::collections::HashMap;
 
 use once_cell::sync::Lazy;
 
 // aktivix.org.md: aktivix.org
 static P_AKTIVIX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "aktivix.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -31,7 +32,7 @@ static P_AKTIVIX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
@@ -39,14 +40,17 @@ static P_AKTIVIX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
 // aol.md: aol.com
 static P_AOL: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "aol",
     status: Status::PREPARATION,
     before_login_hint: "To log in to AOL with Delta Chat, you need to set up an app password in the AOL web interface.",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/aol",
     server: vec![
+        Server { protocol: IMAP, socket: SSL, hostname: "imap.aol.com", port: 993, username_pattern: EMAIL },
+        Server { protocol: SMTP, socket: SSL, hostname: "smtp.aol.com", port: 465, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
@@ -54,6 +58,7 @@ static P_AOL: Lazy<Provider> = Lazy::new(|| {
 
 // arcor.de.md: arcor.de
 static P_ARCOR_DE: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "arcor.de",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -75,13 +80,14 @@ static P_ARCOR_DE: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // autistici.org.md: autistici.org
 static P_AUTISTICI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "autistici.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -103,13 +109,14 @@ static P_AUTISTICI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // bluewin.ch.md: bluewin.ch
 static P_BLUEWIN_CH: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "bluewin.ch",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -131,13 +138,14 @@ static P_BLUEWIN_CH: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // buzon.uy.md: buzon.uy
 static P_BUZON_UY: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "buzon.uy",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -166,6 +174,7 @@ static P_BUZON_UY: Lazy<Provider> = Lazy::new(|| Provider {
 
 // chello.at.md: chello.at
 static P_CHELLO_AT: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "chello.at",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -187,39 +196,42 @@ static P_CHELLO_AT: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // comcast.md: xfinity.com, comcast.net
 static P_COMCAST: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "comcast",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/comcast",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // dismail.de.md: dismail.de
 static P_DISMAIL_DE: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "dismail.de",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/dismail-de",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // disroot.md: disroot.org
 static P_DISROOT: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "disroot",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -233,6 +245,7 @@ static P_DISROOT: Lazy<Provider> = Lazy::new(|| Provider {
 
 // dubby.org.md: dubby.org
 static P_DUBBY_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "dubby.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -283,9 +296,24 @@ static P_DUBBY_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// example.com.md: example.com, example.org
+// espiv.net.md: espiv.net
+static P_ESPIV_NET: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "espiv.net",
+    status: Status::OK,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/espiv-net",
+    server: vec![],
+    config_defaults: None,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
+    oauth2_authorizer: None,
+});
+
+// example.com.md: example.com, example.org, example.net
 static P_EXAMPLE_COM: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "example.com",
     status: Status::BROKEN,
     before_login_hint: "Hush this provider doesn't exist!",
     after_login_hint: "This provider doesn't really exist, so you can't use it :/ If you need an email provider for Delta Chat, take a look at providers.delta.chat!",
@@ -295,7 +323,7 @@ static P_EXAMPLE_COM: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
@@ -303,6 +331,7 @@ static P_EXAMPLE_COM: Lazy<Provider> = Lazy::new(|| {
 
 // fastmail.md: fastmail.com
 static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "fastmail",
     status: Status::PREPARATION,
     before_login_hint:
         "You must create an app-specific password for Delta Chat before you can log in.",
@@ -310,7 +339,7 @@ static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
     overview_page: "https://providers.delta.chat/fastmail",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
@@ -318,6 +347,7 @@ static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
 // firemail.de.md: firemail.at, firemail.de
 static P_FIREMAIL_DE: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "firemail.de",
     status: Status::PREPARATION,
     before_login_hint: "Firemail erlaubt nur bei bezahlten Accounts den vollen Zugriff auf das E-Mail-Protokoll. Wenn Sie nicht für Firemail bezahlen, verwenden Sie bitte einen anderen E-Mail-Anbieter.",
     after_login_hint: "Leider schränkt Firemail die maximale Gruppengröße ein. Je nach Bezahlmodell sind nur 5 bis 30 Gruppenmitglieder erlaubt.",
@@ -325,7 +355,7 @@ static P_FIREMAIL_DE: Lazy<Provider> = Lazy::new(|| {
     server: vec![
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
@@ -333,6 +363,7 @@ static P_FIREMAIL_DE: Lazy<Provider> = Lazy::new(|| {
 
 // five.chat.md: five.chat
 static P_FIVE_CHAT: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "five.chat",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -363,6 +394,7 @@ static P_FIVE_CHAT: Lazy<Provider> = Lazy::new(|| Provider {
 
 // freenet.de.md: freenet.de
 static P_FREENET_DE: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "freenet.de",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -384,14 +416,15 @@ static P_FREENET_DE: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
-// gmail.md: gmail.com, googlemail.com
+// gmail.md: gmail.com, googlemail.com, google.com
 static P_GMAIL: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "gmail",
     status: Status::PREPARATION,
     before_login_hint: "For Gmail accounts, you need to create an app-password if you have \"2-Step Verification\" enabled. If this setting is not available, you need to enable \"less secure apps\".",
     after_login_hint: "",
@@ -409,6 +442,7 @@ static P_GMAIL: Lazy<Provider> = Lazy::new(|| {
 
 // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
 static P_GMX_NET: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "gmx.net",
     status: Status::PREPARATION,
     before_login_hint: "You must allow IMAP access to your account before you can login.",
     after_login_hint: "",
@@ -437,13 +471,14 @@ static P_GMX_NET: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // hermes.radio.md: hermes.radio
 static P_HERMES_RADIO: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "hermes.radio",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -475,6 +510,7 @@ static P_HERMES_RADIO: Lazy<Provider> = Lazy::new(|| Provider {
 // hey.com.md: hey.com
 static P_HEY_COM: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "hey.com",
     status: Status::BROKEN,
     before_login_hint: "hey.com does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to hey.com.",
     after_login_hint: "",
@@ -482,7 +518,7 @@ static P_HEY_COM: Lazy<Provider> = Lazy::new(|| {
     server: vec![
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
@@ -490,19 +526,21 @@ static P_HEY_COM: Lazy<Provider> = Lazy::new(|| {
 
 // i.ua.md: i.ua
 static P_I_UA: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "i.ua",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/i-ua",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // icloud.md: icloud.com, me.com, mac.com
 static P_ICLOUD: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "icloud",
     status: Status::PREPARATION,
     before_login_hint:
         "You must create an app-specific password for Delta Chat before you can login.",
@@ -525,52 +563,56 @@ static P_ICLOUD: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // kolst.com.md: kolst.com
 static P_KOLST_COM: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "kolst.com",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/kolst-com",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // kontent.com.md: kontent.com
 static P_KONTENT_COM: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "kontent.com",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/kontent-com",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // mail.ru.md: mail.ru, inbox.ru, bk.ru, list.ru
 static P_MAIL_RU: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "mail.ru",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/mail-ru",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // mailbox.org.md: mailbox.org, secure.mailbox.org
 static P_MAILBOX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "mailbox.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -585,6 +627,7 @@ static P_MAILBOX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
 // nauta.cu.md: nauta.cu
 static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "nauta.cu",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "Atención - con nauta.cu, puede enviar mensajes sólo a un máximo de 20 personas a la vez. En grupos más grandes, no puede enviar mensajes o abandonar el grupo.",
@@ -612,6 +655,7 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| {
 // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
 static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "outlook.com",
     status: Status::BROKEN,
     before_login_hint: "Outlook.com email addresses will not work as expected as these servers remove some important transport information. Hopefully sooner or later there will be a fix, for now we suggest to use another email address.",
     after_login_hint: "Outlook.com email addresses will not work as expected as these servers remove some important transport information. Unencrypted 1-on-1 chats kind of work, but groups and encryption don't. Hopefully sooner or later there will be a fix, for now we suggest to use another email address.",
@@ -621,7 +665,7 @@ static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
@@ -629,6 +673,7 @@ static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| {
 
 // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
 static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "posteo",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -658,6 +703,7 @@ static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
 // protonmail.md: protonmail.com, protonmail.ch
 static P_PROTONMAIL: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "protonmail",
     status: Status::BROKEN,
     before_login_hint: "Protonmail does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to Protonmail.",
     after_login_hint: "To use Delta Chat with Protonmail, the IMAP bridge must be running in the background. If you have connectivity issues, double check whether it works as expected.",
@@ -665,7 +711,7 @@ static P_PROTONMAIL: Lazy<Provider> = Lazy::new(|| {
     server: vec![
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
@@ -673,6 +719,7 @@ static P_PROTONMAIL: Lazy<Provider> = Lazy::new(|| {
 
 // riseup.net.md: riseup.net
 static P_RISEUP_NET: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "riseup.net",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -686,19 +733,21 @@ static P_RISEUP_NET: Lazy<Provider> = Lazy::new(|| Provider {
 
 // rogers.com.md: rogers.com
 static P_ROGERS_COM: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "rogers.com",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/rogers-com",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // systemli.org.md: systemli.org
 static P_SYSTEMLI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "systemli.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -713,14 +762,17 @@ static P_SYSTEMLI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
 // t-online.md: t-online.de, magenta.de
 static P_T_ONLINE: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "t-online",
     status: Status::PREPARATION,
     before_login_hint: "To use Delta Chat with a T-Online email address, you need to create an app password in the web interface.",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/t-online",
     server: vec![
+        Server { protocol: IMAP, socket: SSL, hostname: "secureimap.t-online.de", port: 993, username_pattern: EMAIL },
+        Server { protocol: SMTP, socket: SSL, hostname: "securesmtp.t-online.de", port: 465, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
@@ -728,6 +780,7 @@ static P_T_ONLINE: Lazy<Provider> = Lazy::new(|| {
 
 // testrun.md: testrun.org
 static P_TESTRUN: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "testrun",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -780,6 +833,7 @@ static P_TESTRUN: Lazy<Provider> = Lazy::new(|| Provider {
 
 // tiscali.it.md: tiscali.it
 static P_TISCALI_IT: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "tiscali.it",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -801,26 +855,28 @@ static P_TISCALI_IT: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // ukr.net.md: ukr.net
 static P_UKR_NET: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "ukr.net",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/ukr-net",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // undernet.uy.md: undernet.uy
 static P_UNDERNET_UY: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "undernet.uy",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -849,19 +905,21 @@ static P_UNDERNET_UY: Lazy<Provider> = Lazy::new(|| Provider {
 
 // vfemail.md: vfemail.net
 static P_VFEMAIL: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "vfemail",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/vfemail",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // vodafone.de.md: vodafone.de, vodafonemail.de
 static P_VODAFONE_DE: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "vodafone.de",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -883,7 +941,7 @@ static P_VODAFONE_DE: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
@@ -891,6 +949,7 @@ static P_VODAFONE_DE: Lazy<Provider> = Lazy::new(|| Provider {
 // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms
 static P_WEB_DE: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "web.de",
     status: Status::PREPARATION,
     before_login_hint: "You must allow IMAP access to your account before you can login.",
     after_login_hint: "Note: if you have your web.de spam settings too strict, you won't receive contact requests from new people. If you want to receive contact requests, you should disable the \"3-Wege-Spamschutz\" in the web.de settings.  Read how: https://hilfe.web.de/email/spam-und-viren/spamschutz-einstellungen.html",
@@ -901,7 +960,7 @@ static P_WEB_DE: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
@@ -910,6 +969,7 @@ static P_WEB_DE: Lazy<Provider> = Lazy::new(|| {
 // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
 static P_YAHOO: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "yahoo",
     status: Status::PREPARATION,
     before_login_hint: "To use Delta Chat with your Yahoo email address you have to create an \"App-Password\" in the account security screen.",
     after_login_hint: "",
@@ -919,7 +979,7 @@ static P_YAHOO: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
@@ -927,6 +987,7 @@ static P_YAHOO: Lazy<Provider> = Lazy::new(|| {
 
 // yandex.ru.md: yandex.com, yandex.by, yandex.kz, yandex.ru, yandex.ua, ya.ru, narod.ru
 static P_YANDEX_RU: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "yandex.ru",
     status: Status::PREPARATION,
     before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
     after_login_hint: "",
@@ -955,6 +1016,7 @@ static P_YANDEX_RU: Lazy<Provider> = Lazy::new(|| Provider {
 
 // ziggo.nl.md: ziggo.nl
 static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "ziggo.nl",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -976,12 +1038,12 @@ static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
-pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
+pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
     [
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol.com", &*P_AOL),
@@ -995,8 +1057,10 @@ pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy:
         ("dismail.de", &*P_DISMAIL_DE),
         ("disroot.org", &*P_DISROOT),
         ("dubby.org", &*P_DUBBY_ORG),
+        ("espiv.net", &*P_ESPIV_NET),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
+        ("example.net", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
         ("firemail.at", &*P_FIREMAIL_DE),
         ("firemail.de", &*P_FIREMAIL_DE),
@@ -1004,6 +1068,7 @@ pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy:
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
+        ("google.com", &*P_GMAIL),
         ("gmx.net", &*P_GMX_NET),
         ("gmx.de", &*P_GMX_NET),
         ("gmx.at", &*P_GMX_NET),
@@ -1152,5 +1217,58 @@ pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy:
     .collect()
 });
 
+pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
+    [
+        ("aktivix.org", &*P_AKTIVIX_ORG),
+        ("aol", &*P_AOL),
+        ("arcor.de", &*P_ARCOR_DE),
+        ("autistici.org", &*P_AUTISTICI_ORG),
+        ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("buzon.uy", &*P_BUZON_UY),
+        ("chello.at", &*P_CHELLO_AT),
+        ("comcast", &*P_COMCAST),
+        ("dismail.de", &*P_DISMAIL_DE),
+        ("disroot", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
+        ("espiv.net", &*P_ESPIV_NET),
+        ("example.com", &*P_EXAMPLE_COM),
+        ("fastmail", &*P_FASTMAIL),
+        ("firemail.de", &*P_FIREMAIL_DE),
+        ("five.chat", &*P_FIVE_CHAT),
+        ("freenet.de", &*P_FREENET_DE),
+        ("gmail", &*P_GMAIL),
+        ("gmx.net", &*P_GMX_NET),
+        ("hermes.radio", &*P_HERMES_RADIO),
+        ("hey.com", &*P_HEY_COM),
+        ("i.ua", &*P_I_UA),
+        ("icloud", &*P_ICLOUD),
+        ("kolst.com", &*P_KOLST_COM),
+        ("kontent.com", &*P_KONTENT_COM),
+        ("mail.ru", &*P_MAIL_RU),
+        ("mailbox.org", &*P_MAILBOX_ORG),
+        ("nauta.cu", &*P_NAUTA_CU),
+        ("outlook.com", &*P_OUTLOOK_COM),
+        ("posteo", &*P_POSTEO),
+        ("protonmail", &*P_PROTONMAIL),
+        ("riseup.net", &*P_RISEUP_NET),
+        ("rogers.com", &*P_ROGERS_COM),
+        ("systemli.org", &*P_SYSTEMLI_ORG),
+        ("t-online", &*P_T_ONLINE),
+        ("testrun", &*P_TESTRUN),
+        ("tiscali.it", &*P_TISCALI_IT),
+        ("ukr.net", &*P_UKR_NET),
+        ("undernet.uy", &*P_UNDERNET_UY),
+        ("vfemail", &*P_VFEMAIL),
+        ("vodafone.de", &*P_VODAFONE_DE),
+        ("web.de", &*P_WEB_DE),
+        ("yahoo", &*P_YAHOO),
+        ("yandex.ru", &*P_YANDEX_RU),
+        ("ziggo.nl", &*P_ZIGGO_NL),
+    ]
+    .iter()
+    .copied()
+    .collect()
+});
+
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd(2020, 10, 30));
+    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 1, 8));

src/provider/mod.rs

@@ -3,8 +3,8 @@
 mod data;
 
 use crate::config::Config;
-use crate::dc_tools::EmailAddress;
-use crate::provider::data::{PROVIDER_DATA, PROVIDER_UPDATED};
+use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED};
+use async_std_resolver::{config, resolver};
 use chrono::{NaiveDateTime, NaiveTime};
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
@@ -68,6 +68,8 @@ pub struct ConfigDefault {
 
 #[derive(Debug)]
 pub struct Provider {
+    /// Unique ID, corresponding to provider database filename.
+    pub id: &'static str,
     pub status: Status,
     pub before_login_hint: &'static str,
     pub after_login_hint: &'static str,
@@ -79,20 +81,84 @@ pub struct Provider {
     pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
-pub fn get_provider_info(addr: &str) -> Option<&Provider> {
-    let domain = match addr.parse::<EmailAddress>() {
-        Ok(addr) => addr.domain,
-        Err(_err) => return None,
-    }
-    .to_lowercase();
+/// Returns provider for the given domain.
+///
+/// This function looks up domain in offline database first. If not
+/// found, it queries MX record for the domain and looks up offline
+/// database for MX domains.
+///
+/// For compatibility, email address can be passed to this function
+/// instead of the domain.
+pub async fn get_provider_info(domain: &str) -> Option<&'static Provider> {
+    let domain = domain.rsplitn(2, '@').next()?;
 
-    if let Some(provider) = PROVIDER_DATA.get(domain.as_str()) {
+    if let Some(provider) = get_provider_by_domain(domain) {
+        return Some(provider);
+    }
+
+    if let Some(provider) = get_provider_by_mx(domain).await {
+        return Some(provider);
+    }
+
+    None
+}
+
+/// Finds a provider in offline database based on domain.
+pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
+    if let Some(provider) = PROVIDER_DATA.get(domain.to_lowercase().as_str()) {
         return Some(*provider);
     }
 
     None
 }
 
+/// Finds a provider based on MX record for the given domain.
+///
+/// For security reasons, only Gmail can be configured this way.
+pub async fn get_provider_by_mx(domain: impl AsRef<str>) -> Option<&'static Provider> {
+    if let Ok(resolver) = resolver(
+        config::ResolverConfig::default(),
+        config::ResolverOpts::default(),
+    )
+    .await
+    {
+        let mut fqdn: String = String::from(domain.as_ref());
+        if !fqdn.ends_with('.') {
+            fqdn.push('.');
+        }
+
+        if let Ok(mx_domains) = resolver.mx_lookup(fqdn).await {
+            for (provider_domain, provider) in PROVIDER_DATA.iter() {
+                if provider.id != "gmail" {
+                    // MX lookup is limited to Gmail for security reasons
+                    continue;
+                }
+
+                let provider_fqdn = provider_domain.to_string() + ".";
+                let provider_fqdn_dot = ".".to_string() + &provider_fqdn;
+
+                for mx_domain in mx_domains.iter() {
+                    let mx_domain = mx_domain.exchange().to_lowercase().to_utf8();
+
+                    if mx_domain == provider_fqdn || mx_domain.ends_with(&provider_fqdn_dot) {
+                        return Some(provider);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+pub fn get_provider_by_id(id: &str) -> Option<&'static Provider> {
+    if let Some(provider) = PROVIDER_IDS.get(id) {
+        Some(&provider)
+    } else {
+        None
+    }
+}
+
 // returns update timestamp in seconds, UTC, compatible for comparison with time() and database times
 pub fn get_provider_update_timestamp() -> i64 {
     NaiveDateTime::new(*PROVIDER_UPDATED, NaiveTime::from_hms(0, 0, 0)).timestamp_millis() / 1_000
@@ -107,24 +173,21 @@ mod tests {
     use chrono::NaiveDate;
 
     #[test]
-    fn test_get_provider_info_unexistant() {
-        let provider = get_provider_info("user@unexistant.org");
+    fn test_get_provider_by_domain_unexistant() {
+        let provider = get_provider_by_domain("unexistant.org");
         assert!(provider.is_none());
     }
 
     #[test]
-    fn test_get_provider_info_mixed_case() {
-        let provider = get_provider_info("uSer@nAUta.Cu").unwrap();
+    fn test_get_provider_by_domain_mixed_case() {
+        let provider = get_provider_by_domain("nAUta.Cu").unwrap();
         assert!(provider.status == Status::OK);
     }
 
     #[test]
-    fn test_get_provider_info() {
-        let provider = get_provider_info("nauta.cu"); // this is no email address
-        assert!(provider.is_none());
-
-        let addr = "user@nauta.cu";
-        let provider = get_provider_info(addr).unwrap();
+    fn test_get_provider_by_domain() {
+        let addr = "nauta.cu";
+        let provider = get_provider_by_domain(addr).unwrap();
         assert!(provider.status == Status::OK);
         let server = &provider.server[0];
         assert_eq!(server.protocol, Protocol::IMAP);
@@ -139,15 +202,30 @@ mod tests {
         assert_eq!(server.port, 25);
         assert_eq!(server.username_pattern, UsernamePattern::EMAIL);
 
-        let provider = get_provider_info("user@gmail.com").unwrap();
+        let provider = get_provider_by_domain("gmail.com").unwrap();
         assert!(provider.status == Status::PREPARATION);
         assert!(!provider.before_login_hint.is_empty());
         assert!(!provider.overview_page.is_empty());
 
-        let provider = get_provider_info("user@googlemail.com").unwrap();
+        let provider = get_provider_by_domain("googlemail.com").unwrap();
         assert!(provider.status == Status::PREPARATION);
     }
 
+    #[test]
+    fn test_get_provider_by_id() {
+        let provider = get_provider_by_id("gmail").unwrap();
+        assert!(provider.id == "gmail");
+    }
+
+    #[async_std::test]
+    async fn test_get_provider_info() {
+        assert!(get_provider_info("").await.is_none());
+        assert!(get_provider_info("google.com").await.unwrap().id == "gmail");
+
+        // get_provider_info() accepts email addresses for backwards compatibility
+        assert!(get_provider_info("example@google.com").await.unwrap().id == "gmail");
+    }
+
     #[test]
     fn test_get_provider_update_timestamp() {
         let timestamp_past = NaiveDateTime::new(

src/provider/update.py

@@ -8,7 +8,8 @@ import datetime
 
 out_all = ""
 out_domains = ""
-domains_dict = {}
+out_ids = ""
+domains_set = set()
 
 def camel(name):
     words = name.split("_")
@@ -22,15 +23,19 @@ def cleanstr(s):
     return s
 
 
+def file2id(f):
+    return os.path.basename(f).replace(".md", "")
+
+
 def file2varname(f):
-    f = f[f.rindex("/")+1:].replace(".md", "")
+    f = file2id(f)
     f = f.replace(".", "_")
     f = f.replace("-", "_")
     return "P_" + f.upper()
 
 
 def file2url(f):
-    f = f[f.rindex("/")+1:].replace(".md", "")
+    f = file2id(f)
     f = f.replace(".", "-")
     return "https://providers.delta.chat/" + f
 
@@ -61,14 +66,16 @@ def process_data(data, file):
         if domain == "" or domain.count(".") < 1 or domain.lower() != domain:
             raise TypeError("bad domain: " + domain)
 
-        global domains_dict
-        if domains_dict.get(domain, False):
+        global domains_set
+        if domain in domains_set:
             raise TypeError("domain used twice: " + domain)
-        domains_dict[domain] = True
+        domains_set.add(domain)
 
         domains += "    (\"" + domain + "\", &*" + file2varname(file) + "),\n"
         comment += domain + ", "
 
+    ids = ""
+    ids += "    (\"" + file2id(file) + "\", &*" + file2varname(file) + "),\n"
 
     server = ""
     has_imap = False
@@ -101,7 +108,7 @@ def process_data(data, file):
 
     config_defaults = process_config_defaults(data)
 
-    strict_tls = data.get("strict_tls", False)
+    strict_tls = data.get("strict_tls", True)
     strict_tls = "true" if strict_tls else "false"
 
     max_smtp_rcpt_to = data.get("max_smtp_rcpt_to", 0)
@@ -115,6 +122,7 @@ def process_data(data, file):
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
     if (not has_imap and not has_smtp) or (has_imap and has_smtp):
         provider += "static " + file2varname(file) + ": Lazy<Provider> = Lazy::new(|| Provider {\n"
+        provider += "    id: \"" + file2id(file) + "\",\n"
         provider += "    status: Status::" + status + ",\n"
         provider += "    before_login_hint: \"" + before_login_hint + "\",\n"
         provider += "    after_login_hint: \"" + after_login_hint + "\",\n"
@@ -132,13 +140,14 @@ def process_data(data, file):
         raise TypeError("status PREPARATION or BROKEN requires before_login_hint: " + file)
 
     # finally, add the provider
-    global out_all, out_domains
+    global out_all, out_domains, out_ids
     out_all += "// " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
 
     # also add provider with no special things to do -
     # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
     out_all += provider
     out_domains += domains
+    out_ids += ids
 
 
 def process_file(file):
@@ -172,10 +181,14 @@ if __name__ == "__main__":
 
     process_dir(sys.argv[1])
 
-    out_all += "pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
+    out_all += "pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
     out_all += out_domains;
     out_all += "].iter().copied().collect());\n\n"
 
+    out_all += "pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
+    out_all += out_ids;
+    out_all += "].iter().copied().collect());\n\n"
+
     now = datetime.datetime.utcnow()
     out_all += "pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> = "\
                "Lazy::new(|| chrono::NaiveDate::from_ymd("+str(now.year)+", "+str(now.month)+", "+str(now.day)+"));\n"

src/qr.rs

@@ -1,20 +1,20 @@
 //! # QR code module
 
+use anyhow::{bail, ensure, format_err, Error};
 use once_cell::sync::Lazy;
 use percent_encoding::percent_decode_str;
 use serde::Deserialize;
+use std::collections::BTreeMap;
 
 use crate::chat;
-use crate::config::*;
+use crate::config::Config;
 use crate::constants::Blocked;
-use crate::contact::*;
+use crate::contact::{addr_normalize, may_be_valid_addr, Contact, Origin};
 use crate::context::Context;
-use crate::error::{bail, ensure, format_err, Error};
 use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::message::Message;
-use crate::param::*;
-use crate::peerstate::*;
+use crate::peerstate::Peerstate;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
@@ -93,16 +93,18 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         }
     };
 
-    // replace & with \n to match expected param format
-    let fragment = fragment.replace('&', "\n");
+    let param: BTreeMap<&str, &str> = fragment
+        .split('&')
+        .filter_map(|s| {
+            if let [key, value] = s.splitn(2, '=').collect::<Vec<_>>()[..] {
+                Some((key, value))
+            } else {
+                None
+            }
+        })
+        .collect();
 
-    // Then parse the parameters
-    let param: Params = match fragment.parse() {
-        Ok(params) => params,
-        Err(err) => return err.into(),
-    };
-
-    let addr = if let Some(addr) = param.get(Param::Forwarded) {
+    let addr = if let Some(addr) = param.get("a") {
         match normalize_address(addr) {
             Ok(addr) => Some(addr),
             Err(err) => return err.into(),
@@ -112,7 +114,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     };
 
     // what is up with that param name?
-    let name = if let Some(encoded_name) = param.get(Param::SetLongitude) {
+    let name = if let Some(encoded_name) = param.get("n") {
         let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
         match percent_decode_str(&encoded_name).decode_utf8() {
             Ok(name) => name.to_string(),
@@ -122,12 +124,12 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         "".to_string()
     };
 
-    let invitenumber = param.get(Param::ProfileImage).map(|s| s.to_string());
-    let auth = param.get(Param::Auth).map(|s| s.to_string());
-    let grpid = param.get(Param::GroupId).map(|s| s.to_string());
+    let invitenumber = param.get("i").map(|s| s.to_string());
+    let auth = param.get("s").map(|s| s.to_string());
+    let grpid = param.get("x").map(|s| s.to_string());
 
     let grpname = if grpid.is_some() {
-        if let Some(encoded_name) = param.get(Param::GroupName) {
+        if let Some(encoded_name) = param.get("g") {
             let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
             match percent_decode_str(&encoded_name).decode_utf8() {
                 Ok(name) => Some(name.to_string()),
@@ -488,6 +490,8 @@ mod tests {
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
         assert_eq!(contact.get_name(), "First Last");
+        assert_eq!(contact.get_authname(), "");
+        assert_eq!(contact.get_display_name(), "First Last");
     }
 
     #[async_std::test]
@@ -563,6 +567,7 @@ mod tests {
         assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
         // Test it again with lowercased "openpgp4fpr:" uri scheme
+        let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
             "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
@@ -603,6 +608,21 @@ mod tests {
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
         assert_eq!(contact.get_name(), "Jörn P. P.");
+
+        // Regression test
+        let ctx = TestContext::new().await;
+        let res = check_qr(
+            &ctx.ctx,
+            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
+        ).await;
+
+        println!("{:?}", res);
+        assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
+        assert_ne!(res.get_id(), 0);
+
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
+        assert_eq!(contact.get_addr(), "cli@deltachat.de");
+        assert_eq!(contact.get_name(), "");
     }
 
     #[async_std::test]

src/scheduler.rs

@@ -1,12 +1,16 @@
 use async_std::prelude::*;
-use async_std::sync::{channel, Receiver, Sender};
-use async_std::task;
+use async_std::{
+    channel::{self, Receiver, Sender},
+    task,
+};
 
+use crate::config::Config;
 use crate::context::Context;
 use crate::dc_tools::maybe_add_time_based_warnings;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
-use crate::{config::Config, message::MsgId, smtp::Smtp};
+use crate::message::MsgId;
+use crate::smtp::Smtp;
 
 pub(crate) struct StopToken;
 
@@ -54,7 +58,10 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started.send(()).await;
+        started
+            .send(())
+            .await
+            .expect("inbox loop, missing started receiver");
         let ctx = ctx1;
 
         // track number of continously executed jobs
@@ -103,7 +110,10 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         })
         .race(fut)
         .await;
-    shutdown_sender.send(()).await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("inbox loop, missing shutdown receiver");
 }
 
 async fn fetch(ctx: &Context, connection: &mut Imap) {
@@ -117,7 +127,7 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
             // fetch
             if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
                 connection.trigger_reconnect();
-                warn!(ctx, "{}", err);
+                warn!(ctx, "{:#}", err);
             }
         }
         None => {
@@ -139,6 +149,12 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
             // fetch
             if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
                 connection.trigger_reconnect();
+                warn!(ctx, "{:#}", err);
+            }
+
+            if let Err(err) = connection.scan_folders(&ctx).await {
+                // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
+                // but maybe just one folder can't be selected or something
                 warn!(ctx, "{}", err);
             }
 
@@ -181,7 +197,10 @@ async fn simple_imap_loop(
     let ctx1 = ctx.clone();
 
     let fut = async move {
-        started.send(()).await;
+        started
+            .send(())
+            .await
+            .expect("simple imap loop, missing started receive");
         let ctx = ctx1;
 
         loop {
@@ -196,7 +215,10 @@ async fn simple_imap_loop(
         })
         .race(fut)
         .await;
-    shutdown_sender.send(()).await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("simple imap loop, missing shutdown receiver");
 }
 
 async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnectionHandlers) {
@@ -212,7 +234,10 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started.send(()).await;
+        started
+            .send(())
+            .await
+            .expect("smtp loop, missing started receiver");
         let ctx = ctx1;
 
         let mut interrupt_info = Default::default();
@@ -240,7 +265,10 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
         })
         .race(fut)
         .await;
-    shutdown_sender.send(()).await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("smtp loop, missing shutdown receiver");
 }
 
 impl Scheduler {
@@ -251,23 +279,25 @@ impl Scheduler {
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
         let (inbox, inbox_handlers) = ImapConnectionState::new();
 
-        let (inbox_start_send, inbox_start_recv) = channel(1);
-        let (mvbox_start_send, mvbox_start_recv) = channel(1);
+        let (inbox_start_send, inbox_start_recv) = channel::bounded(1);
+        let (mvbox_start_send, mvbox_start_recv) = channel::bounded(1);
         let mut mvbox_handle = None;
-        let (sentbox_start_send, sentbox_start_recv) = channel(1);
+        let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1);
         let mut sentbox_handle = None;
-        let (smtp_start_send, smtp_start_recv) = channel(1);
+        let (smtp_start_send, smtp_start_recv) = channel::bounded(1);
 
-        let ctx1 = ctx.clone();
-        let inbox_handle = Some(task::spawn(async move {
-            inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
-        }));
+        let inbox_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                inbox_loop(ctx, inbox_start_send, inbox_handlers).await
+            }))
+        };
 
         if ctx.get_config_bool(Config::MvboxWatch).await {
-            let ctx1 = ctx.clone();
+            let ctx = ctx.clone();
             mvbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
-                    ctx1,
+                    ctx,
                     mvbox_start_send,
                     mvbox_handlers,
                     Config::ConfiguredMvboxFolder,
@@ -275,14 +305,17 @@ impl Scheduler {
                 .await
             }));
         } else {
-            mvbox_start_send.send(()).await;
+            mvbox_start_send
+                .send(())
+                .await
+                .expect("mvbox start send, missing receiver");
         }
 
         if ctx.get_config_bool(Config::SentboxWatch).await {
-            let ctx1 = ctx.clone();
+            let ctx = ctx.clone();
             sentbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
-                    ctx1,
+                    ctx,
                     sentbox_start_send,
                     sentbox_handlers,
                     Config::ConfiguredSentboxFolder,
@@ -290,13 +323,18 @@ impl Scheduler {
                 .await
             }));
         } else {
-            sentbox_start_send.send(()).await;
+            sentbox_start_send
+                .send(())
+                .await
+                .expect("sentbox start send, missing receiver");
         }
 
-        let ctx1 = ctx.clone();
-        let smtp_handle = Some(task::spawn(async move {
-            smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
-        }));
+        let smtp_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                smtp_loop(ctx, smtp_start_send, smtp_handlers).await
+            }))
+        };
 
         *self = Scheduler::Running {
             inbox,
@@ -367,17 +405,27 @@ impl Scheduler {
             }
             Scheduler::Running {
                 inbox,
+                inbox_handle,
                 mvbox,
+                mvbox_handle,
                 sentbox,
+                sentbox_handle,
                 smtp,
+                smtp_handle,
                 ..
             } => {
-                inbox
-                    .stop()
-                    .join(mvbox.stop())
-                    .join(sentbox.stop())
-                    .join(smtp.stop())
-                    .await;
+                if inbox_handle.is_some() {
+                    inbox.stop().await;
+                }
+                if mvbox_handle.is_some() {
+                    mvbox.stop().await;
+                }
+                if sentbox_handle.is_some() {
+                    sentbox.stop().await;
+                }
+                if smtp_handle.is_some() {
+                    smtp.stop().await;
+                }
 
                 StopToken
             }
@@ -436,7 +484,10 @@ impl ConnectionState {
     /// Shutdown this connection completely.
     async fn stop(&self) {
         // Trigger shutdown of the run loop.
-        self.stop_sender.send(()).await;
+        self.stop_sender
+            .send(())
+            .await
+            .expect("stop, missing receiver");
         // Wait for a notification that the run loop has been shutdown.
         self.shutdown_receiver.recv().await.ok();
     }
@@ -454,9 +505,9 @@ pub(crate) struct SmtpConnectionState {
 
 impl SmtpConnectionState {
     fn new() -> (Self, SmtpConnectionHandlers) {
-        let (stop_sender, stop_receiver) = channel(1);
-        let (shutdown_sender, shutdown_receiver) = channel(1);
-        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+        let (stop_sender, stop_receiver) = channel::bounded(1);
+        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = SmtpConnectionHandlers {
             connection: Smtp::new(),
@@ -502,9 +553,9 @@ pub(crate) struct ImapConnectionState {
 impl ImapConnectionState {
     /// Construct a new connection.
     fn new() -> (Self, ImapConnectionHandlers) {
-        let (stop_sender, stop_receiver) = channel(1);
-        let (shutdown_sender, shutdown_receiver) = channel(1);
-        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+        let (stop_sender, stop_receiver) = channel::bounded(1);
+        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = ImapConnectionHandlers {
             connection: Imap::new(idle_interrupt_receiver),

src/securejoin/bobstate.rs

@@ -0,0 +1,521 @@
+//! Secure-Join protocol state machine for Bob, the joiner-side.
+//!
+//! This module contains the state machine to run the Secure-Join handshake for Bob and does
+//! not do any user interaction required by the protocol.  Instead the state machine
+//! provides all the information to its driver so it can perform the correct interactions.
+//!
+//! The [`BobState`] is only directly used to initially create it when starting the
+//! protocol.  Afterwards it must be stored in a mutex and the [`BobStateHandle`] should be
+//! used to work with the state.
+
+use anyhow::{Error, Result};
+use async_std::sync::MutexGuard;
+
+use crate::chat::{self, ChatId};
+use crate::constants::{Blocked, Viewtype};
+use crate::contact::{Contact, Origin};
+use crate::context::Context;
+use crate::events::EventType;
+use crate::headerdef::HeaderDef;
+use crate::key::{DcKey, SignedPublicKey};
+use crate::message::Message;
+use crate::mimeparser::{MimeMessage, SystemMessage};
+use crate::param::Param;
+
+use super::qrinvite::QrInvite;
+use super::{
+    encrypted_and_signed, fingerprint_equals_sender, mark_peer_as_verified, JoinError, SendMsgError,
+};
+
+/// The stage of the [`BobState`] securejoin handshake protocol state machine.
+///
+/// This does not concern itself with user interactions, only represents what happened to
+/// the protocol state machine from handling this message.
+#[derive(Clone, Copy, Debug, Display)]
+pub enum BobHandshakeStage {
+    /// Step 2 completed: (vc|vg)-request message sent.
+    ///
+    /// Note that this is only ever returned by [`BobState::start_protocol`] and never by
+    /// [`BobState::handle_message`].
+    RequestSent,
+    /// Step 4 completed: (vc|vg)-request-with-auth message sent.
+    RequestWithAuthSent,
+    /// The protocol completed successfully.
+    Completed,
+    /// The protocol prematurely terminated with given reason.
+    Terminated(&'static str),
+}
+
+/// A handle to work with the [`BobState`] of Bob's securejoin protocol.
+///
+/// This handle can only be created for when an underlying [`BobState`] exists.  It keeps
+/// open a lock which guarantees unique access to the state and this struct must be dropped
+/// to return the lock.
+pub struct BobStateHandle<'a> {
+    guard: MutexGuard<'a, Option<BobState>>,
+    bobstate: BobState,
+    clear_state_on_drop: bool,
+}
+
+impl<'a> BobStateHandle<'a> {
+    /// Creates a new instance, upholding the guarantee that [`BobState`] must exist.
+    pub fn from_guard(mut guard: MutexGuard<'a, Option<BobState>>) -> Option<Self> {
+        match guard.take() {
+            Some(bobstate) => Some(Self {
+                guard,
+                bobstate,
+                clear_state_on_drop: false,
+            }),
+            None => None,
+        }
+    }
+
+    /// Returns the [`ChatId`] of the 1:1 chat with the inviter (Alice).
+    pub fn chat_id(&self) -> ChatId {
+        self.bobstate.chat_id
+    }
+
+    /// Returns a reference to the [`QrInvite`] of the joiner process.
+    pub fn invite(&self) -> &QrInvite {
+        &self.bobstate.invite
+    }
+
+    /// Handles the given message for the securejoin handshake for Bob.
+    ///
+    /// This proxies to [`BobState::handle_message`] and makes sure to clear the state when
+    /// the protocol state is terminal.  It returns `Some` if the message successfully
+    /// advanced the state of the protocol state machine, `None` otherwise.
+    pub async fn handle_message(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Option<BobHandshakeStage> {
+        info!(context, "Handling securejoin message for BobStateHandle");
+        match self.bobstate.handle_message(context, mime_message).await {
+            Ok(Some(stage)) => {
+                if matches!(stage, BobHandshakeStage::Completed | BobHandshakeStage::Terminated(_))
+                {
+                    self.finish_protocol(context).await;
+                }
+                Some(stage)
+            }
+            Ok(None) => None,
+            Err(err) => {
+                warn!(
+                    context,
+                    "Error handling handshake message, aborting handshake: {}", err
+                );
+                self.finish_protocol(context).await;
+                None
+            }
+        }
+    }
+
+    /// Marks the bob handshake as finished.
+    ///
+    /// This will clear the state on [`InnerContext::bob`] once this handle is dropped,
+    /// allowing a new handshake to be started from [`Bob`].
+    ///
+    /// Note that the state is only cleared on Drop since otherwise the invariant that the
+    /// state is always consistent is violated.  However the "ongoing" prococess is released
+    /// here a little bit earlier as this requires access to the Context, which we do not
+    /// have on Drop (Drop can not run asynchronous code).
+    ///
+    /// [`InnerContext::bob`]: crate::context::InnerContext::bob
+    /// [`Bob`]: super::Bob
+    async fn finish_protocol(&mut self, context: &Context) {
+        info!(context, "Finishing securejoin handshake protocol for Bob");
+        self.clear_state_on_drop = true;
+        if let QrInvite::Group { .. } = self.bobstate.invite {
+            context.stop_ongoing().await;
+        }
+    }
+}
+
+impl<'a> Drop for BobStateHandle<'a> {
+    fn drop(&mut self) {
+        if self.clear_state_on_drop {
+            // The Option should already be empty because we take it out in the ctor,
+            // however the typesystem doesn't guarantee this so do it again anyway.
+            self.guard.take();
+        } else {
+            // Make sure to put back the BobState into the Option of the Mutex, it was taken
+            // out by the constructor.
+            self.guard.replace(self.bobstate.clone());
+        }
+    }
+}
+
+/// The securejoin state kept in-memory while Bob is joining.
+///
+/// This is currently stored in [`Bob`] which is stored on the [`Context`], thus Bob can
+/// only run one securejoin joiner protocol at a time.
+///
+/// This purposefully has nothing optional, the state is always fully valid.  See
+/// [`Bob::state`] to get access to this state.
+///
+/// # Conducting the securejoin handshake
+///
+/// The methods on this struct allow you to interact with the state and thus conduct the
+/// securejoin handshake for Bob.  The methods only concern themselves with the protocol
+/// state and explicitly avoid performing any user interactions required by securejoin.
+/// This simplifies the concerns and logic required in both the callers and in the state
+/// management.  The return values can be used to understand what user interactions need to
+/// happen.
+///
+/// [`Bob`]: super::Bob
+/// [`Bob::state`]: super::Bob::state
+#[derive(Debug, Clone)]
+pub struct BobState {
+    /// The QR Invite code.
+    invite: QrInvite,
+    /// The next expected message from Alice.
+    next: SecureJoinStep,
+    /// The [`ChatId`] of the 1:1 chat with Alice, matching [`QrInvite::contact_id`].
+    chat_id: ChatId,
+}
+
+impl BobState {
+    /// Starts the securejoin protocol and creates a new [`BobState`].
+    ///
+    /// # Bob - the joiner's side
+    /// ## Step 2 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    pub async fn start_protocol(
+        context: &Context,
+        invite: QrInvite,
+    ) -> Result<(Self, BobHandshakeStage), JoinError> {
+        let chat_id = chat::create_by_contact_id(context, invite.contact_id())
+            .await
+            .map_err(JoinError::UnknownContact)?;
+        if fingerprint_equals_sender(context, invite.fingerprint(), chat_id).await {
+            // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
+            info!(context, "Taking securejoin protocol shortcut");
+            let state = Self {
+                invite,
+                next: SecureJoinStep::ContactConfirm,
+                chat_id,
+            };
+            state
+                .send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
+                .await?;
+            Ok((state, BobHandshakeStage::RequestWithAuthSent))
+        } else {
+            let state = Self {
+                invite,
+                next: SecureJoinStep::AuthRequired,
+                chat_id,
+            };
+            state
+                .send_handshake_message(context, BobHandshakeMsg::Request)
+                .await?;
+            Ok((state, BobHandshakeStage::RequestSent))
+        }
+    }
+
+    /// Returns the [`QrInvite`] used to create this [`BobState`].
+    pub fn invite(&self) -> &QrInvite {
+        &self.invite
+    }
+
+    /// Handles the given message for the securejoin handshake for Bob.
+    ///
+    /// If the message was not used for this handshake `None` is returned, otherwise the new
+    /// stage is returned.  Once [`BobHandshakeStage::Completed`] or
+    /// [`BobHandshakeStage::Terminated`] are reached this [`BobState`] should be destroyed,
+    /// further calling it will just result in the messages being unused by this handshake.
+    ///
+    /// # Errors
+    ///
+    /// Under normal operation this should never return an error, regardless of what kind of
+    /// message it is called with.  Any errors therefore should be treated as fatal internal
+    /// errors and this entire [`BobState`] should be thrown away as the state machine can
+    /// no longer be considered consistent.
+    async fn handle_message(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Result<Option<BobHandshakeStage>> {
+        let step = match mime_message.get(HeaderDef::SecureJoin) {
+            Some(step) => step,
+            None => {
+                warn!(
+                    context,
+                    "Message has no Secure-Join header: {}",
+                    mime_message.get_rfc724_mid().unwrap_or_default()
+                );
+                return Ok(None);
+            }
+        };
+        if !self.is_msg_expected(context, step.as_str()) {
+            info!(context, "{} message out of sync for BobState", step);
+            return Ok(None);
+        }
+        match step.as_str() {
+            "vg-auth-required" | "vc-auth-required" => {
+                self.step_auth_required(context, mime_message).await
+            }
+            "vg-member-added" | "vc-contact-confirm" => {
+                self.step_contact_confirm(context, mime_message).await
+            }
+            _ => {
+                warn!(context, "Invalid step for BobState: {}", step);
+                Ok(None)
+            }
+        }
+    }
+
+    /// Returns `true` if the message is expected according to the protocol.
+    fn is_msg_expected(&self, context: &Context, step: &str) -> bool {
+        let variant_matches = match self.invite {
+            QrInvite::Contact { .. } => step.starts_with("vc-"),
+            QrInvite::Group { .. } => step.starts_with("vg-"),
+        };
+        let step_matches = self.next.matches(context, step);
+        variant_matches && step_matches
+    }
+
+    /// Handles a *vc-auth-required* or *vg-auth-required* message.
+    ///
+    /// # Bob - the joiner's side
+    /// ## Step 4 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    async fn step_auth_required(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Result<Option<BobHandshakeStage>> {
+        info!(
+            context,
+            "Bob Step 4 - handling vc-auth-require/vg-auth-required message"
+        );
+        if !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint())) {
+            let reason = if mime_message.was_encrypted() {
+                "Valid signature missing"
+            } else {
+                "Required encryption missing"
+            };
+            self.next = SecureJoinStep::Terminated;
+            return Ok(Some(BobHandshakeStage::Terminated(reason)));
+        }
+        if !fingerprint_equals_sender(context, self.invite.fingerprint(), self.chat_id).await {
+            self.next = SecureJoinStep::Terminated;
+            return Ok(Some(BobHandshakeStage::Terminated("Fingerprint mismatch")));
+        }
+        info!(context, "Fingerprint verified.",);
+        self.next = SecureJoinStep::ContactConfirm;
+        self.send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
+            .await?;
+        Ok(Some(BobHandshakeStage::RequestWithAuthSent))
+    }
+
+    /// Handles a *vc-contact-confirm* or *vg-member-added* message.
+    ///
+    /// # Bob - the joiner's side
+    /// ## Step 7 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    ///
+    /// This deviates from the protocol by also sending a confirmation message in response
+    /// to the *vc-contact-confirm* message.  This has no specific value to the protocol and
+    /// is only done out of symmerty with *vg-member-added* handling.
+    async fn step_contact_confirm(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Result<Option<BobHandshakeStage>> {
+        info!(
+            context,
+            "Bob Step 7 - handling vc-contact-confirm/vg-member-added message"
+        );
+        let vg_expect_encrypted = match self.invite {
+            QrInvite::Contact { .. } => {
+                // setup-contact is always encrypted
+                true
+            }
+            QrInvite::Group { ref grpid, .. } => {
+                // This is buggy, is_verified_group will always be
+                // false since the group is created by receive_imf for
+                // the very handshake message we're handling now.  But
+                // only after we have returned.  It does not impact
+                // the security invariants of secure-join however.
+                let (_, is_verified_group, _) = chat::get_chat_id_by_grpid(context, grpid)
+                    .await
+                    .unwrap_or((ChatId::new(0), false, Blocked::Not));
+                // when joining a non-verified group
+                // the vg-member-added message may be unencrypted
+                // when not all group members have keys or prefer encryption.
+                // So only expect encryption if this is a verified group
+                is_verified_group
+            }
+        };
+        if vg_expect_encrypted
+            && !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint()))
+        {
+            self.next = SecureJoinStep::Terminated;
+            return Ok(Some(BobHandshakeStage::Terminated(
+                "Contact confirm message not encrypted",
+            )));
+        }
+        mark_peer_as_verified(context, self.invite.fingerprint()).await?;
+        Contact::scaleup_origin_by_id(context, self.invite.contact_id(), Origin::SecurejoinJoined)
+            .await;
+        emit_event!(context, EventType::ContactsChanged(None));
+
+        if let QrInvite::Group { .. } = self.invite {
+            let member_added = mime_message
+                .get(HeaderDef::ChatGroupMemberAdded)
+                .map(|s| s.as_str())
+                .ok_or_else(|| Error::msg("Missing Chat-Group-Member-Added header"))?;
+            if !context.is_self_addr(member_added).await? {
+                info!(context, "Message belongs to a different handshake (scaled up contact anyway to allow creation of group).");
+                return Ok(None);
+            }
+        }
+
+        self.send_handshake_message(context, BobHandshakeMsg::ContactConfirmReceived)
+            .await
+            .map_err(|_| {
+                warn!(
+                    context,
+                    "Failed to send vc-contact-confirm-received/vg-member-added-received"
+                );
+            })
+            // This is not an error affecting the protocol outcome.
+            .ok();
+
+        self.next = SecureJoinStep::Completed;
+        Ok(Some(BobHandshakeStage::Completed))
+    }
+
+    /// Sends the requested handshake message to Alice.
+    ///
+    /// This takes care of adding the required headers for the step.
+    async fn send_handshake_message(
+        &self,
+        context: &Context,
+        step: BobHandshakeMsg,
+    ) -> Result<(), SendMsgError> {
+        let mut msg = Message {
+            viewtype: Viewtype::Text,
+            text: Some(step.body_text(&self.invite)),
+            hidden: true,
+            ..Default::default()
+        };
+        msg.param.set_cmd(SystemMessage::SecurejoinMessage);
+
+        // Sends the step in Secure-Join header.
+        msg.param
+            .set(Param::Arg, step.securejoin_header(&self.invite));
+
+        match step {
+            BobHandshakeMsg::Request => {
+                // Sends the Secure-Join-Invitenumber header in mimefactory.rs.
+                msg.param.set(Param::Arg2, self.invite.invitenumber());
+                msg.param.set_int(Param::ForcePlaintext, 1);
+            }
+            BobHandshakeMsg::RequestWithAuth => {
+                // Sends the Secure-Join-Auth header in mimefactory.rs.
+                msg.param.set(Param::Arg2, self.invite.authcode());
+                msg.param.set_int(Param::GuaranteeE2ee, 1);
+            }
+            BobHandshakeMsg::ContactConfirmReceived => {
+                msg.param.set_int(Param::GuaranteeE2ee, 1);
+            }
+        };
+
+        // Sends our own fingerprint in the Secure-Join-Fingerprint header.
+        let bob_fp = SignedPublicKey::load_self(context).await?.fingerprint();
+        msg.param.set(Param::Arg3, bob_fp.hex());
+
+        // Sends the grpid in the Secure-Join-Group header.
+        if let QrInvite::Group { ref grpid, .. } = self.invite {
+            msg.param.set(Param::Arg4, grpid);
+        }
+
+        chat::send_msg(context, self.chat_id, &mut msg).await?;
+        Ok(())
+    }
+}
+
+/// Identifies the SecureJoin handshake messages Bob can send.
+enum BobHandshakeMsg {
+    /// vc-request or vg-request
+    Request,
+    /// vc-request-with-auth or vg-request-with-auth
+    RequestWithAuth,
+    /// vc-contact-confirm-received or vg-member-added-received
+    ContactConfirmReceived,
+}
+
+impl BobHandshakeMsg {
+    /// Returns the text to send in the body of the handshake message.
+    ///
+    /// This text has no significance to the protocol, but would be visible if users see
+    /// this email message directly, e.g. when accessing their email without using
+    /// DeltaChat.
+    fn body_text(&self, invite: &QrInvite) -> String {
+        format!("Secure-Join: {}", self.securejoin_header(invite))
+    }
+
+    /// Returns the `Secure-Join` header value.
+    ///
+    /// This identifies the step this message is sending information about.  Most protocol
+    /// steps include additional information into other headers, see
+    /// [`BobState::send_handshake_message`] for these.
+    fn securejoin_header(&self, invite: &QrInvite) -> &'static str {
+        match self {
+            Self::Request => match invite {
+                QrInvite::Contact { .. } => "vc-request",
+                QrInvite::Group { .. } => "vg-request",
+            },
+            Self::RequestWithAuth => match invite {
+                QrInvite::Contact { .. } => "vc-request-with-auth",
+                QrInvite::Group { .. } => "vg-request-with-auth",
+            },
+            Self::ContactConfirmReceived => match invite {
+                QrInvite::Contact { .. } => "vc-contact-confirm-received",
+                QrInvite::Group { .. } => "vg-member-added-received",
+            },
+        }
+    }
+}
+
+/// The next message expected by [`BobState`] in the setup-contact/secure-join protocol.
+#[derive(Debug, Clone, PartialEq)]
+enum SecureJoinStep {
+    /// Expecting the auth-required message.
+    ///
+    /// This corresponds to the `vc-auth-required` or `vg-auth-required` message of step 3d.
+    AuthRequired,
+    /// Expecting the contact-confirm message.
+    ///
+    /// This corresponds to the `vc-contact-confirm` or `vg-member-added` message of step
+    /// 6b.
+    ContactConfirm,
+    /// The protocol terminated because of an error.
+    ///
+    /// The securejoin protocol terminated, this exists to ensure [`BobState`] can detect
+    /// when it earlier signalled that is should be terminated.  It is an error to call with
+    /// this state.
+    Terminated,
+    /// The protocol completed.
+    ///
+    /// This exists to ensure [`BobState`] can detect when it earlier signalled that it is
+    /// complete.  It is an error to call with this state.
+    Completed,
+}
+
+impl SecureJoinStep {
+    /// Compares the legacy string representation of a step to a [`SecureJoinStep`] variant.
+    fn matches(&self, context: &Context, step: &str) -> bool {
+        match self {
+            Self::AuthRequired => step == "vc-auth-required" || step == "vg-auth-required",
+            Self::ContactConfirm => step == "vc-contact-confirm" || step == "vg-member-added",
+            SecureJoinStep::Terminated => {
+                warn!(context, "Terminated state for next securejoin step");
+                false
+            }
+            SecureJoinStep::Completed => {
+                warn!(context, "Complted state for next securejoin step");
+                false
+            }
+        }
+    }
+}

src/securejoin/qrinvite.rs

@@ -0,0 +1,120 @@
+//! Supporting code for the QR-code invite.
+//!
+//! QR-codes are decoded into a more general-purpose [`Lot`] struct normally, this struct is
+//! so general it is not even specific to QR-codes.  This makes working with it rather hard,
+//! so here we have a wrapper type that specifically deals with Secure-Join QR-codes so
+//! that the Secure-Join code can have many more guarantees when dealing with this.
+
+use std::convert::TryFrom;
+
+use anyhow::Result;
+
+use crate::key::{Fingerprint, FingerprintError};
+use crate::lot::{Lot, LotState};
+
+/// Represents the data from a QR-code scan.
+///
+/// There are methods to conveniently access fields present in both variants.
+#[derive(Debug, Clone)]
+pub enum QrInvite {
+    Contact {
+        contact_id: u32,
+        fingerprint: Fingerprint,
+        invitenumber: String,
+        authcode: String,
+    },
+    Group {
+        contact_id: u32,
+        fingerprint: Fingerprint,
+        name: String,
+        grpid: String,
+        invitenumber: String,
+        authcode: String,
+    },
+}
+
+impl QrInvite {
+    /// The contact ID of the inviter.
+    ///
+    /// The actual QR-code contains a URL-encoded email address, but upon scanning this is
+    /// translated to a contact ID.
+    pub fn contact_id(&self) -> u32 {
+        match self {
+            Self::Contact { contact_id, .. } | Self::Group { contact_id, .. } => *contact_id,
+        }
+    }
+
+    /// The fingerprint of the inviter.
+    pub fn fingerprint(&self) -> &Fingerprint {
+        match self {
+            Self::Contact { fingerprint, .. } | Self::Group { fingerprint, .. } => &fingerprint,
+        }
+    }
+
+    /// The `INVITENUMBER` of the setup-contact/secure-join protocol.
+    pub fn invitenumber(&self) -> &str {
+        match self {
+            Self::Contact { invitenumber, .. } | Self::Group { invitenumber, .. } => &invitenumber,
+        }
+    }
+
+    /// The `AUTH` code of the setup-contact/secure-join protocol.
+    pub fn authcode(&self) -> &str {
+        match self {
+            Self::Contact { authcode, .. } | Self::Group { authcode, .. } => &authcode,
+        }
+    }
+}
+
+impl TryFrom<Lot> for QrInvite {
+    type Error = QrError;
+
+    fn try_from(lot: Lot) -> Result<Self, Self::Error> {
+        if lot.state != LotState::QrAskVerifyContact && lot.state != LotState::QrAskVerifyGroup {
+            return Err(QrError::UnsupportedProtocol);
+        }
+        if lot.id == 0 {
+            return Err(QrError::MissingContactId);
+        }
+        let fingerprint = lot.fingerprint.ok_or(QrError::MissingFingerprint)?;
+        let invitenumber = lot.invitenumber.ok_or(QrError::MissingInviteNumber)?;
+        let authcode = lot.auth.ok_or(QrError::MissingAuthCode)?;
+        match lot.state {
+            LotState::QrAskVerifyContact => Ok(QrInvite::Contact {
+                contact_id: lot.id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            }),
+            LotState::QrAskVerifyGroup => Ok(QrInvite::Group {
+                contact_id: lot.id,
+                fingerprint,
+                name: lot.text1.ok_or(QrError::MissingGroupName)?,
+                grpid: lot.text2.ok_or(QrError::MissingGroupId)?,
+                invitenumber,
+                authcode,
+            }),
+            _ => Err(QrError::UnsupportedProtocol),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum QrError {
+    #[error("Unsupported protocol in QR-code")]
+    UnsupportedProtocol,
+    #[error("Failed to read fingerprint")]
+    InvalidFingerprint(#[from] FingerprintError),
+    #[error("Missing fingerprint")]
+    MissingFingerprint,
+    #[error("Missing invitenumber")]
+    MissingInviteNumber,
+    #[error("Missing auth code")]
+    MissingAuthCode,
+    #[error("Missing group name")]
+    MissingGroupName,
+    #[error("Missing group id")]
+    MissingGroupId,
+    #[error("Missing contact id")]
+    MissingContactId,
+}

src/simplify.rs

@@ -17,18 +17,23 @@ pub fn escape_message_footer_marks(text: &str) -> String {
 }
 
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
+/// Returns `(lines, footer_lines)` tuple;
+/// `footer_lines` is set to `Some` if the footer was actually removed from `lines`
+/// (which is equal to the input array otherwise).
 #[allow(clippy::indexing_slicing)]
-fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
+fn remove_message_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<&'a [&'a str]>) {
     let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
         match line {
             // some providers encode `-- ` to `-- =20` which results in `--  `
-            "-- " | "--  " => return &lines[..ix],
+            "-- " | "--  " => return (&lines[..ix], lines.get(ix + 1..)),
             // some providers encode `-- ` to `=2D-` which results in only `--`;
             // use that only when no other footer is found
             // and if the line before is empty and the line after is not empty
             "--" => {
-                if (ix == 0 || lines[ix - 1] == "") && ix != lines.len() - 1 && lines[ix + 1] != ""
+                if (ix == 0 || lines[ix - 1].is_empty())
+                    && ix != lines.len() - 1
+                    && !lines[ix + 1].is_empty()
                 {
                     nearly_standard_footer = Some(ix);
                 }
@@ -37,13 +42,15 @@ fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
         }
     }
     if let Some(ix) = nearly_standard_footer {
-        return &lines[..ix];
+        return (&lines[..ix], lines.get(ix + 1..));
     }
-    lines
+    (lines, None)
 }
 
-/// Remove nonstandard footer and a boolean indicating whether such
-/// footer was removed.
+/// Remove nonstandard footer and a boolean indicating whether such footer was removed.
+/// Returns `(lines, is_footer_removed)` tuple;
+/// `is_footer_removed` is set to `true` if the footer was actually removed from `lines`
+/// (which is equal to the input array otherwise).
 #[allow(clippy::indexing_slicing)]
 fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     for (ix, &line) in lines.iter().enumerate() {
@@ -60,20 +67,29 @@ fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     (lines, false)
 }
 
-fn split_lines(buf: &str) -> Vec<&str> {
+pub(crate) fn split_lines(buf: &str) -> Vec<&str> {
     buf.split('\n').collect()
 }
 
 /// Simplify message text for chat display.
 /// Remove quotes, signatures, trailing empty lines etc.
-pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool, Option<String>) {
+/// Returns `(text, is_forwarded, is_cut, quote, footer)` tuple,
+/// returning the simplified text and some additional information gained from the input.
+pub fn simplify(
+    mut input: String,
+    is_chat_message: bool,
+) -> (String, bool, bool, Option<String>, Option<String>) {
+    let mut is_cut = false;
+
     input.retain(|c| c != '\r');
     let lines = split_lines(&input);
     let (lines, is_forwarded) = skip_forward_header(&lines);
 
     let (lines, mut top_quote) = remove_top_quote(lines);
     let original_lines = &lines;
-    let lines = remove_message_footer(lines);
+    let (lines, footer_lines) = remove_message_footer(lines);
+    let footer = footer_lines.map(|footer_lines| render_message(footer_lines, false));
+    is_cut = is_cut || footer.is_some();
 
     let text = if is_chat_message {
         render_message(lines, false)
@@ -88,10 +104,21 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool, Opti
         if lines.iter().all(|it| it.trim().is_empty()) {
             render_message(original_lines, false)
         } else {
+            is_cut = is_cut || has_nonstandard_footer || bottom_quote.is_some();
             render_message(lines, has_nonstandard_footer || bottom_quote.is_some())
         }
     };
-    (text, is_forwarded, top_quote)
+
+    if !is_chat_message {
+        top_quote = top_quote.map(|quote| {
+            let quote_lines = split_lines(&quote);
+            let (quote_lines, quote_footer_lines) = remove_message_footer(&quote_lines);
+            is_cut = is_cut || quote_footer_lines.is_some();
+
+            render_message(quote_lines, false)
+        });
+    }
+    (text, is_forwarded, is_cut, top_quote, footer)
 }
 
 /// Skips "forwarded message" header.
@@ -218,19 +245,10 @@ fn render_message(lines: &[&str], is_cut_at_end: bool) -> String {
  * Tools
  */
 fn is_empty_line(buf: &str) -> bool {
-    // XXX: can it be simplified to buf.chars().all(|c| c.is_whitespace())?
-    //
-    // Strictly speaking, it is not equivalent (^A is not whitespace, but less than ' '),
-    // but having control sequences in email body?!
-    //
-    // See discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
-    for c in buf.chars() {
-        if c > ' ' {
-            return false;
-        }
-    }
-
-    true
+    buf.chars().all(char::is_whitespace)
+    // for some time, this checked for `char <= ' '`,
+    // see discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
+    // and https://github.com/deltachat/deltachat-core-rust/pull/2104/files#r538973613
 }
 
 fn is_quoted_headline(buf: &str) -> bool {
@@ -255,7 +273,7 @@ mod tests {
         #[test]
         // proptest does not support [[:graphical:][:space:]] regex.
         fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let (output, _is_forwarded, _) = simplify(input, true);
+            let (output, _is_forwarded, _, _, _) = simplify(input, true);
             assert!(output.split('\n').all(|s| s != "-- "));
         }
     }
@@ -263,38 +281,47 @@ mod tests {
     #[test]
     fn test_dont_remove_whole_message() {
         let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
-        let (plain, is_forwarded, _) = simplify(input, false);
+        let (plain, is_forwarded, is_cut, _, _) = simplify(input, false);
         assert_eq!(
             plain,
             "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
         );
         assert!(!is_forwarded);
+        assert!(!is_cut);
     }
 
     #[test]
     fn test_chat_message() {
         let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
-        let (plain, is_forwarded, _) = simplify(input, true);
+        let (plain, is_forwarded, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
         assert!(!is_forwarded);
+        assert!(is_cut);
+        assert_eq!(
+            footer.unwrap(),
+            "Sent with my Delta Chat Messenger: https://delta.chat"
+        );
     }
 
     #[test]
     fn test_simplify_trim() {
         let input = "line1\n\r\r\rline2".to_string();
-        let (plain, is_forwarded, _) = simplify(input, false);
+        let (plain, is_forwarded, is_cut, _, _) = simplify(input, false);
 
         assert_eq!(plain, "line1\nline2");
         assert!(!is_forwarded);
+        assert!(!is_cut);
     }
 
     #[test]
     fn test_simplify_forwarded_message() {
         let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string();
-        let (plain, is_forwarded, _) = simplify(input, false);
+        let (plain, is_forwarded, is_cut, _, footer) = simplify(input, false);
 
         assert_eq!(plain, "Forwarded message");
         assert!(is_forwarded);
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "Signature goes here");
     }
 
     #[test]
@@ -336,41 +363,60 @@ mod tests {
     #[test]
     fn test_remove_message_footer() {
         let input = "text\n--\nno footer".to_string();
-        let (plain, _, _) = simplify(input, true);
+        let (plain, _, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n--\nno footer");
+        assert_eq!(footer, None);
+        assert!(!is_cut);
 
         let input = "text\n\n--\n\nno footer".to_string();
-        let (plain, _, _) = simplify(input, true);
+        let (plain, _, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\n\nno footer");
+        assert_eq!(footer, None);
+        assert!(!is_cut);
 
         let input = "text\n\n-- no footer\n\n".to_string();
-        let (plain, _, _) = simplify(input, true);
+        let (plain, _, _, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n\n-- no footer");
+        assert_eq!(footer, None);
 
         let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
-        let (plain, _, _) = simplify(input, true);
+        let (plain, _, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\nno footer");
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "footer");
 
         let input = "text\n\n--\ntreated as footer when unescaped".to_string();
-        let (plain, _, _) = simplify(input.clone(), true);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), true);
         assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "treated as footer when unescaped");
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _, _) = simplify(escaped, true);
+        let (plain, _, is_cut, _, footer) = simplify(escaped, true);
         assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
+        assert!(!is_cut);
+        assert_eq!(footer, None);
 
         // Nonstandard footer sent by https://siju.es/
         let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
-        let (plain, _, _) = simplify(input.clone(), false);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), false);
         assert_eq!(plain, "Message text here [...]");
-        let (plain, _, _) = simplify(input.clone(), true);
+        assert!(is_cut);
+        assert_eq!(footer, None);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), true);
         assert_eq!(plain, input);
+        assert!(!is_cut);
+        assert_eq!(footer, None);
 
         let input = "--\ntreated as footer when unescaped".to_string();
-        let (plain, _, _) = simplify(input.clone(), true);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), true);
         assert_eq!(plain, ""); // see remove_message_footer() for some explanations
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "treated as footer when unescaped");
 
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _, _) = simplify(escaped, true);
+        let (plain, _, is_cut, _, footer) = simplify(escaped, true);
         assert_eq!(plain, "--\ntreated as footer when unescaped");
+        assert!(!is_cut);
+        assert_eq!(footer, None);
     }
 }

src/smtp/mod.rs

@@ -4,16 +4,16 @@ pub mod send;
 
 use std::time::{Duration, SystemTime};
 
-use async_smtp::smtp::client::net::*;
-use async_smtp::*;
+use async_smtp::smtp::client::net::ClientTlsParameters;
+use async_smtp::{error, smtp, EmailAddress};
 
-use crate::constants::*;
+use crate::constants::DC_LP_AUTH_OAUTH2;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam};
-use crate::oauth2::*;
-use crate::provider::{get_provider_info, Socket};
-use crate::stock::StockMessage;
+use crate::oauth2::dc_get_oauth2_access_token;
+use crate::provider::Socket;
+use crate::stock_str;
 
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
@@ -107,16 +107,16 @@ impl Smtp {
                 &lp.smtp,
                 &lp.addr,
                 lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+                lp.provider.map_or(false, |provider| provider.strict_tls),
             )
             .await;
         if let Err(ref err) = res {
-            let message = context
-                .stock_string_repl_str2(
-                    StockMessage::ServerResponse,
-                    format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
-                    err.to_string(),
-                )
-                .await;
+            let message = stock_str::server_response(
+                context,
+                format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
+                err.to_string(),
+            )
+            .await;
 
             context.emit_event(EventType::ErrorNetwork(message));
         };
@@ -130,6 +130,7 @@ impl Smtp {
         lp: &ServerLoginParam,
         addr: &str,
         oauth2: bool,
+        provider_strict_tls: bool,
     ) -> Result<()> {
         if self.is_connected().await {
             warn!(context, "SMTP already connected.");
@@ -151,9 +152,8 @@ impl Smtp {
         let domain = &lp.server;
         let port = lp.port;
 
-        let provider = get_provider_info(addr);
         let strict_tls = match lp.certificate_checks {
-            CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
+            CertificateChecks::Automatic => provider_strict_tls,
             CertificateChecks::Strict => true,
             CertificateChecks::AcceptInvalidCertificates
             | CertificateChecks::AcceptInvalidCertificates2 => false,

src/smtp/send.rs

@@ -1,13 +1,11 @@
 //! # SMTP message sending
 
 use super::Smtp;
-use async_smtp::*;
+use async_smtp::{EmailAddress, Envelope, SendableEmail, Transport};
 
-use crate::config::Config;
 use crate::constants::DEFAULT_MAX_SMTP_RCPT_TO;
 use crate::context::Context;
 use crate::events::EventType;
-use crate::provider::get_provider_info;
 use itertools::Itertools;
 use std::time::Duration;
 
@@ -38,12 +36,7 @@ impl Smtp {
         let message_len_bytes = message.len();
 
         let mut chunk_size = DEFAULT_MAX_SMTP_RCPT_TO;
-        if let Some(provider) = get_provider_info(
-            &context
-                .get_config(Config::ConfiguredAddr)
-                .await
-                .unwrap_or_default(),
-        ) {
+        if let Some(provider) = context.get_configured_provider().await {
             if let Some(max_smtp_rcpt_to) = provider.max_smtp_rcpt_to {
                 chunk_size = max_smtp_rcpt_to as usize;
             }

src/sql.rs

@@ -7,16 +7,22 @@ use std::collections::HashSet;
 use std::path::Path;
 use std::time::Duration;
 
+use anyhow::format_err;
 use rusqlite::{Connection, Error as SqlError, OpenFlags};
 
-use crate::chat::{update_device_icon, update_saved_messages_icon};
-use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH};
+use crate::chat::{add_device_msg, update_device_icon, update_saved_messages_icon};
+use crate::config::Config;
+use crate::config::Config::DeleteServerAfter;
+use crate::constants::{ShowEmails, Viewtype, DC_CHAT_ID_TRASH};
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_delete_file, time, EmailAddress};
 use crate::ephemeral::start_ephemeral_timers;
-use crate::error::format_err;
-use crate::param::*;
-use crate::peerstate::*;
+use crate::imap;
+use crate::message::Message;
+use crate::param::{Param, Params};
+use crate::peerstate::Peerstate;
+use crate::provider::get_provider_by_domain;
+use crate::stock_str;
 
 #[macro_export]
 macro_rules! paramsv {
@@ -45,7 +51,7 @@ pub enum Error {
     #[error("{0:?}")]
     BlobError(#[from] crate::blob::BlobError),
     #[error("{0}")]
-    Other(#[from] crate::error::Error),
+    Other(#[from] anyhow::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -83,7 +89,7 @@ impl Sql {
         context: &Context,
         dbfile: T,
         readonly: bool,
-    ) -> crate::error::Result<()> {
+    ) -> anyhow::Result<()> {
         let res = open(context, self, &dbfile, readonly).await;
         if let Err(err) = &res {
             match err.downcast_ref::<Error>() {
@@ -220,6 +226,31 @@ impl Sql {
         .await
     }
 
+    /// Check if a column exists in a given table.
+    pub async fn col_exists(
+        &self,
+        table_name: impl AsRef<str>,
+        col_name: impl AsRef<str>,
+    ) -> Result<bool> {
+        let table_name = table_name.as_ref().to_string();
+        let col_name = col_name.as_ref().to_string();
+        self.with_conn(move |conn| {
+            let mut exists = false;
+            // `PRAGMA table_info` returns one row per column,
+            // each row containing 0=cid, 1=name, 2=type, 3=notnull, 4=dflt_value
+            conn.pragma(None, "table_info", &table_name, |row| {
+                let curr_name: String = row.get(1)?;
+                if col_name == curr_name {
+                    exists = true;
+                }
+                Ok(())
+            })?;
+
+            Ok(exists)
+        })
+        .await
+    }
+
     /// Execute a query which is expected to return zero or one row.
     pub async fn query_row_optional<T, F>(
         &self,
@@ -465,6 +496,10 @@ pub fn get_rowid2(
 }
 
 pub async fn housekeeping(context: &Context) {
+    if let Err(err) = crate::ephemeral::delete_expired_messages(context).await {
+        warn!(context, "Failed to delete expired messages: {}", err);
+    }
+
     let mut files_in_use = HashSet::new();
     let mut unreferenced_count = 0;
 
@@ -595,7 +630,13 @@ pub async fn housekeeping(context: &Context) {
         );
     }
 
-    info!(context, "Housekeeping done.",);
+    if let Err(e) = context
+        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
+        .await
+    {
+        warn!(context, "Can't set config: {}", e);
+    }
+    info!(context, "Housekeeping done.");
 }
 
 #[allow(clippy::indexing_slicing)]
@@ -653,7 +694,7 @@ async fn open(
     sql: &Sql,
     dbfile: impl AsRef<Path>,
     readonly: bool,
-) -> crate::error::Result<()> {
+) -> anyhow::Result<()> {
     if sql.is_open().await {
         error!(
             context,
@@ -926,6 +967,7 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
         let mut dbversion = dbversion_before_update;
         let mut recalc_fingerprints = false;
         let mut update_icons = !exists_before_update;
+        let mut disable_server_delete = false;
 
         if dbversion < 1 {
             info!(context, "[migration] v1");
@@ -1382,6 +1424,88 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
             .await?;
             sql.set_raw_config_int(context, "dbversion", 69).await?;
         }
+        if dbversion < 71 {
+            info!(context, "[migration] v71");
+            if let Some(addr) = context.get_config(Config::ConfiguredAddr).await {
+                if let Ok(domain) = addr.parse::<EmailAddress>().map(|email| email.domain) {
+                    context
+                        .set_config(
+                            Config::ConfiguredProvider,
+                            get_provider_by_domain(&domain).map(|provider| provider.id),
+                        )
+                        .await?;
+                } else {
+                    warn!(context, "Can't parse configured address: {:?}", addr);
+                }
+            }
+
+            sql.set_raw_config_int(context, "dbversion", 71).await?;
+        }
+        if dbversion < 72 {
+            info!(context, "[migration] v72");
+            if !sql.col_exists("msgs", "mime_modified").await? {
+                sql.execute(
+                    "ALTER TABLE msgs ADD COLUMN mime_modified INTEGER DEFAULT 0;",
+                    paramsv![],
+                )
+                .await?;
+            }
+            sql.set_raw_config_int(context, "dbversion", 72).await?;
+        }
+        if dbversion < 73 {
+            use Config::*;
+            info!(context, "[migration] v73");
+            sql.execute(
+                "CREATE TABLE imap_sync (folder TEXT PRIMARY KEY, uidvalidity INTEGER DEFAULT 0, uid_next INTEGER DEFAULT 0);",
+                paramsv![],
+            )
+            .await?;
+            for c in &[
+                ConfiguredInboxFolder,
+                ConfiguredSentboxFolder,
+                ConfiguredMvboxFolder,
+            ] {
+                if let Some(folder) = context.get_config(*c).await {
+                    let (uid_validity, last_seen_uid) =
+                        imap::get_config_last_seen_uid(context, &folder).await;
+                    if last_seen_uid > 0 {
+                        imap::set_uid_next(context, &folder, last_seen_uid + 1).await?;
+                        imap::set_uidvalidity(context, &folder, uid_validity).await?;
+                    }
+                }
+            }
+            if exists_before_update {
+                disable_server_delete = true;
+
+                // Don't disable server delete if it was on by default (Nauta):
+                if let Some(provider) = context.get_configured_provider().await {
+                    if let Some(defaults) = &provider.config_defaults {
+                        if defaults.iter().any(|d| d.key == Config::DeleteServerAfter) {
+                            disable_server_delete = false;
+                        }
+                    }
+                }
+            }
+            sql.set_raw_config_int(context, "dbversion", 73).await?;
+        }
+        if dbversion < 74 {
+            info!(context, "[migration] v74");
+            sql.execute(
+                "UPDATE contacts SET name='' WHERE name=authname",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 74).await?;
+        }
+        if dbversion < 75 {
+            info!(context, "[migration] v75");
+            sql.execute(
+                "ALTER TABLE contacts ADD COLUMN status TEXT DEFAULT '';",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 75).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)
@@ -1412,6 +1536,16 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
             update_saved_messages_icon(context).await?;
             update_device_icon(context).await?;
         }
+        if disable_server_delete {
+            // We now always watch all folders and delete messages there if delete_server is enabled.
+            // So, for people who have delete_server enabled, disable it and add a hint to the devicechat:
+            if context.get_config_delete_server_after().await.is_some() {
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text = Some(stock_str::delete_server_turned_off(context).await);
+                add_device_msg(context, None, Some(&mut msg)).await?;
+                context.set_config(DeleteServerAfter, Some("0")).await?;
+            }
+        }
     }
 
     info!(context, "Opened {:?}.", dbfile.as_ref(),);
@@ -1437,6 +1571,7 @@ async fn prune_tombstones(context: &Context) -> Result<()> {
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_maybe_add_file() {
@@ -1463,4 +1598,19 @@ mod test {
         assert!(!is_file_in_use(&files, Some(".txt"), "hello"));
         assert!(is_file_in_use(&files, Some("-suffix"), "world.txt-suffix"));
     }
+
+    #[async_std::test]
+    async fn test_table_exists() {
+        let t = TestContext::new().await;
+        assert!(t.ctx.sql.table_exists("msgs").await.unwrap());
+        assert!(!t.ctx.sql.table_exists("foobar").await.unwrap());
+    }
+
+    #[async_std::test]
+    async fn test_col_exists() {
+        let t = TestContext::new().await;
+        assert!(t.ctx.sql.col_exists("msgs", "mime_modified").await.unwrap());
+        assert!(!t.ctx.sql.col_exists("msgs", "foobar").await.unwrap());
+        assert!(!t.ctx.sql.col_exists("foobar", "foobar").await.unwrap());
+    }
 }

src/stock.rs

@@ -1,695 +0,0 @@
-//! Module to work with translatable stock strings
-
-use std::borrow::Cow;
-
-use strum::EnumProperty;
-use strum_macros::EnumProperty;
-
-use crate::chat;
-use crate::chat::ProtectionStatus;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
-use crate::contact::*;
-use crate::context::Context;
-use crate::error::{bail, Error};
-use crate::message::Message;
-use crate::param::Param;
-use crate::stock::StockMessage::{DeviceMessagesHint, WelcomeMessage};
-use crate::{blob::BlobObject, config::Config};
-
-/// Stock strings
-///
-/// These identify the string to return in [Context.stock_str].  The
-/// numbers must stay in sync with `deltachat.h` `DC_STR_*` constants.
-///
-/// See the `stock_*` methods on [Context] to use these.
-///
-/// [Context]: crate::context::Context
-#[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, EnumProperty)]
-#[repr(u32)]
-pub enum StockMessage {
-    #[strum(props(fallback = "No messages."))]
-    NoMessages = 1,
-
-    #[strum(props(fallback = "Me"))]
-    SelfMsg = 2,
-
-    #[strum(props(fallback = "Draft"))]
-    Draft = 3,
-
-    #[strum(props(fallback = "Voice message"))]
-    VoiceMessage = 7,
-
-    #[strum(props(fallback = "Contact requests"))]
-    DeadDrop = 8,
-
-    #[strum(props(fallback = "Image"))]
-    Image = 9,
-
-    #[strum(props(fallback = "Video"))]
-    Video = 10,
-
-    #[strum(props(fallback = "Audio"))]
-    Audio = 11,
-
-    #[strum(props(fallback = "File"))]
-    File = 12,
-
-    #[strum(props(fallback = "Sent with my Delta Chat Messenger: https://delta.chat"))]
-    StatusLine = 13,
-
-    #[strum(props(fallback = "Hello, I\'ve just created the group \"%1$s\" for us."))]
-    NewGroupDraft = 14,
-
-    #[strum(props(fallback = "Group name changed from \"%1$s\" to \"%2$s\"."))]
-    MsgGrpName = 15,
-
-    #[strum(props(fallback = "Group image changed."))]
-    MsgGrpImgChanged = 16,
-
-    #[strum(props(fallback = "Member %1$s added."))]
-    MsgAddMember = 17,
-
-    #[strum(props(fallback = "Member %1$s removed."))]
-    MsgDelMember = 18,
-
-    #[strum(props(fallback = "Group left."))]
-    MsgGroupLeft = 19,
-
-    #[strum(props(fallback = "GIF"))]
-    Gif = 23,
-
-    #[strum(props(fallback = "Encrypted message"))]
-    EncryptedMsg = 24,
-
-    #[strum(props(fallback = "End-to-end encryption available."))]
-    E2eAvailable = 25,
-
-    #[strum(props(fallback = "Transport-encryption."))]
-    EncrTransp = 27,
-
-    #[strum(props(fallback = "No encryption."))]
-    EncrNone = 28,
-
-    #[strum(props(fallback = "This message was encrypted for another setup."))]
-    CantDecryptMsgBody = 29,
-
-    #[strum(props(fallback = "Fingerprints"))]
-    FingerPrints = 30,
-
-    #[strum(props(fallback = "Return receipt"))]
-    ReadRcpt = 31,
-
-    #[strum(props(fallback = "This is a return receipt for the message \"%1$s\"."))]
-    ReadRcptMailBody = 32,
-
-    #[strum(props(fallback = "Group image deleted."))]
-    MsgGrpImgDeleted = 33,
-
-    #[strum(props(fallback = "End-to-end encryption preferred."))]
-    E2ePreferred = 34,
-
-    #[strum(props(fallback = "%1$s verified."))]
-    ContactVerified = 35,
-
-    #[strum(props(fallback = "Cannot verify %1$s"))]
-    ContactNotVerified = 36,
-
-    #[strum(props(fallback = "Changed setup for %1$s"))]
-    ContactSetupChanged = 37,
-
-    #[strum(props(fallback = "Archived chats"))]
-    ArchivedChats = 40,
-
-    #[strum(props(fallback = "Autocrypt Setup Message"))]
-    AcSetupMsgSubject = 42,
-
-    #[strum(props(
-        fallback = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\nTo decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device."
-    ))]
-    AcSetupMsgBody = 43,
-
-    #[strum(props(
-        fallback = "Cannot login as \"%1$s\". Please check if the email address and the password are correct."
-    ))]
-    CannotLogin = 60,
-
-    #[strum(props(fallback = "Could not connect to %1$s: %2$s"))]
-    ServerResponse = 61,
-
-    #[strum(props(fallback = "%1$s by %2$s."))]
-    MsgActionByUser = 62,
-
-    #[strum(props(fallback = "%1$s by me."))]
-    MsgActionByMe = 63,
-
-    #[strum(props(fallback = "Location streaming enabled."))]
-    MsgLocationEnabled = 64,
-
-    #[strum(props(fallback = "Location streaming disabled."))]
-    MsgLocationDisabled = 65,
-
-    #[strum(props(fallback = "Location"))]
-    Location = 66,
-
-    #[strum(props(fallback = "Sticker"))]
-    Sticker = 67,
-
-    #[strum(props(fallback = "Device messages"))]
-    DeviceMessages = 68,
-
-    #[strum(props(fallback = "Saved messages"))]
-    SavedMessages = 69,
-
-    #[strum(props(
-        fallback = "Messages in this chat are generated locally by your Delta Chat app. \
-                    Its makers use it to inform about app updates and problems during usage."
-    ))]
-    DeviceMessagesHint = 70,
-
-    #[strum(props(fallback = "Welcome to Delta Chat! – \
-                    Delta Chat looks and feels like other popular messenger apps, \
-                    but does not involve centralized control, \
-                    tracking or selling you, friends, colleagues or family out to large organizations.\n\n\
-                    Technically, Delta Chat is an email application with a modern chat interface. \
-                    Email in a new dress if you will 👻\n\n\
-                    Use Delta Chat with anyone out of billions of people: just use their e-mail address. \
-                    Recipients don't need to install Delta Chat, visit websites or sign up anywhere - \
-                    however, of course, if they like, you may point them to 👉 https://get.delta.chat"))]
-    WelcomeMessage = 71,
-
-    #[strum(props(fallback = "Unknown sender for this chat. See 'info' for more details."))]
-    UnknownSenderForChat = 72,
-
-    #[strum(props(fallback = "Message from %1$s"))]
-    SubjectForNewContact = 73,
-
-    #[strum(props(fallback = "Failed to send message to %1$s."))]
-    FailedSendingTo = 74,
-
-    #[strum(props(fallback = "Message deletion timer is disabled."))]
-    MsgEphemeralTimerDisabled = 75,
-
-    // A fallback message for unknown timer values.
-    // "s" stands for "second" SI unit here.
-    #[strum(props(fallback = "Message deletion timer is set to %1$s s."))]
-    MsgEphemeralTimerEnabled = 76,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 minute."))]
-    MsgEphemeralTimerMinute = 77,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 hour."))]
-    MsgEphemeralTimerHour = 78,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 day."))]
-    MsgEphemeralTimerDay = 79,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 week."))]
-    MsgEphemeralTimerWeek = 80,
-
-    #[strum(props(fallback = "Message deletion timer is set to 4 weeks."))]
-    MsgEphemeralTimerFourWeeks = 81,
-
-    #[strum(props(fallback = "Video chat invitation"))]
-    VideochatInvitation = 82,
-
-    #[strum(props(fallback = "You are invited to a video chat, click %1$s to join."))]
-    VideochatInviteMsgBody = 83,
-
-    #[strum(props(fallback = "Error:\n\n“%1$s”"))]
-    ConfigurationFailed = 84,
-
-    #[strum(props(
-        fallback = "⚠️ Date or time of your device seem to be inaccurate (%1$s).\n\n\
-                    Adjust your clock ⏰🔧 to ensure your messages are received correctly."
-    ))]
-    BadTimeMsgBody = 85,
-
-    #[strum(props(fallback = "⚠️ Your Delta Chat version might be outdated.\n\n\
-                    This may cause problems because your chat partners use newer versions - \
-                    and you are missing the latest features 😳\n\
-                    Please check https://get.delta.chat or your app store for updates."))]
-    UpdateReminderMsgBody = 86,
-
-    #[strum(props(
-        fallback = "Could not find your mail server.\n\nPlease check your internet connection."
-    ))]
-    ErrorNoNetwork = 87,
-
-    #[strum(props(fallback = "Chat protection enabled."))]
-    ProtectionEnabled = 88,
-
-    #[strum(props(fallback = "Chat protection disabled."))]
-    ProtectionDisabled = 89,
-
-    // used in summaries, a noun, not a verb (not: "to reply")
-    #[strum(props(fallback = "Reply"))]
-    ReplyNoun = 90,
-
-    #[strum(props(fallback = "You deleted the \"Saved messages\" chat.\n\n\
-                    To use the \"Saved messages\" feature again, create a new chat with yourself."))]
-    SelfDeletedMsgBody = 91,
-}
-
-/*
-"
-*/
-
-impl StockMessage {
-    /// Default untranslated strings for stock messages.
-    ///
-    /// These could be used in logging calls, so no logging here.
-    fn fallback(self) -> &'static str {
-        self.get_str("fallback").unwrap_or_default()
-    }
-}
-
-impl Context {
-    /// Set the stock string for the [StockMessage].
-    ///
-    pub async fn set_stock_translation(
-        &self,
-        id: StockMessage,
-        stockstring: String,
-    ) -> Result<(), Error> {
-        if stockstring.contains("%1") && !id.fallback().contains("%1") {
-            bail!(
-                "translation {} contains invalid %1 placeholder, default is {}",
-                stockstring,
-                id.fallback()
-            );
-        }
-        if stockstring.contains("%2") && !id.fallback().contains("%2") {
-            bail!(
-                "translation {} contains invalid %2 placeholder, default is {}",
-                stockstring,
-                id.fallback()
-            );
-        }
-        self.translated_stockstrings
-            .write()
-            .await
-            .insert(id as usize, stockstring);
-        Ok(())
-    }
-
-    /// Return the stock string for the [StockMessage].
-    ///
-    /// Return a translation (if it was set with set_stock_translation before)
-    /// or a default (English) string.
-    pub async fn stock_str(&self, id: StockMessage) -> Cow<'_, str> {
-        match self
-            .translated_stockstrings
-            .read()
-            .await
-            .get(&(id as usize))
-        {
-            Some(ref x) => Cow::Owned((*x).to_string()),
-            None => Cow::Borrowed(id.fallback()),
-        }
-    }
-
-    /// Return stock string, replacing placeholders with provided string.
-    ///
-    /// This replaces both the *first* `%1$s`, `%1$d` and `%1$@`
-    /// placeholders with the provided string.
-    /// (the `%1$@` variant is used on iOS, the other are used on Android and Desktop)
-    pub async fn stock_string_repl_str(&self, id: StockMessage, insert: impl AsRef<str>) -> String {
-        self.stock_str(id)
-            .await
-            .replacen("%1$s", insert.as_ref(), 1)
-            .replacen("%1$d", insert.as_ref(), 1)
-            .replacen("%1$@", insert.as_ref(), 1)
-    }
-
-    /// Return stock string, replacing placeholders with provided int.
-    ///
-    /// Like [Context::stock_string_repl_str] but substitute the placeholders
-    /// with an integer.
-    pub async fn stock_string_repl_int(&self, id: StockMessage, insert: i32) -> String {
-        self.stock_string_repl_str(id, format!("{}", insert).as_str())
-            .await
-    }
-
-    /// Return stock string, replacing 2 placeholders with provided string.
-    ///
-    /// This replaces both the *first* `%1$s`, `%1$d` and `%1$@`
-    /// placeholders with the string in `insert` and does the same for
-    /// `%2$s`, `%2$d` and `%2$@` for `insert2`.
-    /// (the `%1$@` variant is used on iOS, the other are used on Android and Desktop)
-    pub async fn stock_string_repl_str2(
-        &self,
-        id: StockMessage,
-        insert: impl AsRef<str>,
-        insert2: impl AsRef<str>,
-    ) -> String {
-        self.stock_str(id)
-            .await
-            .replacen("%1$s", insert.as_ref(), 1)
-            .replacen("%1$d", insert.as_ref(), 1)
-            .replacen("%1$@", insert.as_ref(), 1)
-            .replacen("%2$s", insert2.as_ref(), 1)
-            .replacen("%2$d", insert2.as_ref(), 1)
-            .replacen("%2$@", insert2.as_ref(), 1)
-    }
-
-    /// Return some kind of stock message
-    ///
-    /// If the `id` is [StockMessage::MsgAddMember] or
-    /// [StockMessage::MsgDelMember] then `param1` is considered to be the
-    /// contact address and will be replaced by that contact's display
-    /// name.
-    ///
-    /// If `from_id` is not `0`, any trailing dot is removed from the
-    /// first stock string created so far.  If the `from_id` contact is
-    /// the user itself, i.e. `DC_CONTACT_ID_SELF` the string is used
-    /// itself as param to the [StockMessage::MsgActionByMe] stock string
-    /// resulting in a string like "Member Alice added by me." (for
-    /// [StockMessage::MsgAddMember] as `id`).  If the `from_id` contact
-    /// is any other user than the contact's display name is looked up and
-    /// used as the second parameter to [StockMessage::MsgActionByUser] with
-    /// again the original stock string being used as the first parameter,
-    /// resulting in a string like "Member Alice added by Bob.".
-    pub async fn stock_system_msg(
-        &self,
-        id: StockMessage,
-        param1: impl AsRef<str>,
-        param2: impl AsRef<str>,
-        from_id: u32,
-    ) -> String {
-        let insert1 = if id == StockMessage::MsgAddMember || id == StockMessage::MsgDelMember {
-            let contact_id =
-                Contact::lookup_id_by_addr(self, param1.as_ref(), Origin::Unknown).await;
-            if contact_id != 0 {
-                Contact::get_by_id(self, contact_id)
-                    .await
-                    .map(|contact| contact.get_name_n_addr())
-                    .unwrap_or_default()
-            } else {
-                param1.as_ref().to_string()
-            }
-        } else {
-            param1.as_ref().to_string()
-        };
-
-        let action = self
-            .stock_string_repl_str2(id, insert1, param2.as_ref().to_string())
-            .await;
-        let action1 = action.trim_end_matches('.');
-        match from_id {
-            0 => action,
-            DC_CONTACT_ID_SELF => {
-                self.stock_string_repl_str(StockMessage::MsgActionByMe, action1)
-                    .await
-            }
-            _ => {
-                let displayname = Contact::get_by_id(self, from_id)
-                    .await
-                    .map(|contact| contact.get_name_n_addr())
-                    .unwrap_or_default();
-
-                self.stock_string_repl_str2(StockMessage::MsgActionByUser, action1, &displayname)
-                    .await
-            }
-        }
-    }
-
-    /// Returns a stock message saying that protection status has changed.
-    pub async fn stock_protection_msg(&self, protect: ProtectionStatus, from_id: u32) -> String {
-        self.stock_system_msg(
-            match protect {
-                ProtectionStatus::Protected => StockMessage::ProtectionEnabled,
-                ProtectionStatus::Unprotected => StockMessage::ProtectionDisabled,
-            },
-            "",
-            "",
-            from_id,
-        )
-        .await
-    }
-
-    pub(crate) async fn update_device_chats(&self) -> Result<(), Error> {
-        if self.get_config_bool(Config::Bot).await {
-            return Ok(());
-        }
-
-        // create saved-messages chat; we do this only once, if the user has deleted the chat,
-        // he can recreate it manually (make sure we do not re-add it when configure() was called a second time)
-        if !self.sql.get_raw_config_bool(&self, "self-chat-added").await {
-            self.sql
-                .set_raw_config_bool(&self, "self-chat-added", true)
-                .await?;
-            chat::create_by_contact_id(&self, DC_CONTACT_ID_SELF).await?;
-        }
-
-        // add welcome-messages. by the label, this is done only once,
-        // if the user has deleted the message or the chat, it is not added again.
-        let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(self.stock_str(DeviceMessagesHint).await.to_string());
-        chat::add_device_msg(&self, Some("core-about-device-chat"), Some(&mut msg)).await?;
-
-        let image = include_bytes!("../assets/welcome-image.jpg");
-        let blob = BlobObject::create(&self, "welcome-image.jpg".to_string(), image).await?;
-        let mut msg = Message::new(Viewtype::Image);
-        msg.param.set(Param::File, blob.as_name());
-        chat::add_device_msg(&self, Some("core-welcome-image"), Some(&mut msg)).await?;
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(self.stock_str(WelcomeMessage).await.to_string());
-        chat::add_device_msg(&self, Some("core-welcome"), Some(&mut msg)).await?;
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::test_utils::*;
-
-    use crate::constants::DC_CONTACT_ID_SELF;
-
-    use crate::chat::Chat;
-    use crate::chatlist::Chatlist;
-    use num_traits::ToPrimitive;
-
-    #[test]
-    fn test_enum_mapping() {
-        assert_eq!(StockMessage::NoMessages.to_usize().unwrap(), 1);
-        assert_eq!(StockMessage::SelfMsg.to_usize().unwrap(), 2);
-    }
-
-    #[test]
-    fn test_fallback() {
-        assert_eq!(StockMessage::NoMessages.fallback(), "No messages.");
-    }
-
-    #[async_std::test]
-    async fn test_set_stock_translation() {
-        let t = TestContext::new().await;
-        t.ctx
-            .set_stock_translation(StockMessage::NoMessages, "xyz".to_string())
-            .await
-            .unwrap();
-        assert_eq!(t.ctx.stock_str(StockMessage::NoMessages).await, "xyz")
-    }
-
-    #[async_std::test]
-    async fn test_set_stock_translation_wrong_replacements() {
-        let t = TestContext::new().await;
-        assert!(t
-            .ctx
-            .set_stock_translation(StockMessage::NoMessages, "xyz %1$s ".to_string())
-            .await
-            .is_err());
-        assert!(t
-            .ctx
-            .set_stock_translation(StockMessage::NoMessages, "xyz %2$s ".to_string())
-            .await
-            .is_err());
-    }
-
-    #[async_std::test]
-    async fn test_stock_str() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx.stock_str(StockMessage::NoMessages).await,
-            "No messages."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_string_repl_str() {
-        let t = TestContext::new().await;
-        // uses %1$s substitution
-        assert_eq!(
-            t.ctx
-                .stock_string_repl_str(StockMessage::MsgAddMember, "Foo")
-                .await,
-            "Member Foo added."
-        );
-        // We have no string using %1$d to test...
-    }
-
-    #[async_std::test]
-    async fn test_stock_string_repl_int() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_string_repl_int(StockMessage::MsgAddMember, 42)
-                .await,
-            "Member 42 added."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_string_repl_str2() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_string_repl_str2(StockMessage::ServerResponse, "foo", "bar")
-                .await,
-            "Could not connect to foo: bar"
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_simple() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
-                .await,
-            "Location streaming enabled."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_add_member_by_me() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgAddMember,
-                    "alice@example.com",
-                    "",
-                    DC_CONTACT_ID_SELF
-                )
-                .await,
-            "Member alice@example.com added by me."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_add_member_by_me_with_displayname() {
-        let t = TestContext::new().await;
-        Contact::create(&t.ctx, "Alice", "alice@example.com")
-            .await
-            .expect("failed to create contact");
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgAddMember,
-                    "alice@example.com",
-                    "",
-                    DC_CONTACT_ID_SELF
-                )
-                .await,
-            "Member Alice (alice@example.com) added by me."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_add_member_by_other_with_displayname() {
-        let t = TestContext::new().await;
-        let contact_id = {
-            Contact::create(&t.ctx, "Alice", "alice@example.com")
-                .await
-                .expect("Failed to create contact Alice");
-            Contact::create(&t.ctx, "Bob", "bob@example.com")
-                .await
-                .expect("failed to create bob")
-        };
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgAddMember,
-                    "alice@example.com",
-                    "",
-                    contact_id,
-                )
-                .await,
-            "Member Alice (alice@example.com) added by Bob (bob@example.com)."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_grp_name() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgGrpName,
-                    "Some chat",
-                    "Other chat",
-                    DC_CONTACT_ID_SELF
-                )
-                .await,
-            "Group name changed from \"Some chat\" to \"Other chat\" by me."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_grp_name_other() {
-        let t = TestContext::new().await;
-        let id = Contact::create(&t.ctx, "Alice", "alice@example.com")
-            .await
-            .expect("failed to create contact");
-
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(StockMessage::MsgGrpName, "Some chat", "Other chat", id)
-                .await,
-            "Group name changed from \"Some chat\" to \"Other chat\" by Alice (alice@example.com)."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_update_device_chats() {
-        let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.ok();
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 2);
-
-        let chat0 = Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
-            .await
-            .unwrap();
-        let (self_talk_id, device_chat_id) = if chat0.is_self_talk() {
-            (chats.get_chat_id(0), chats.get_chat_id(1))
-        } else {
-            (chats.get_chat_id(1), chats.get_chat_id(0))
-        };
-
-        // delete self-talk first; this adds a message to device-chat about how self-talk can be restored
-        let device_chat_msgs_before = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None)
-            .await
-            .len();
-        self_talk_id.delete(&t.ctx).await.ok();
-        assert_eq!(
-            chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None)
-                .await
-                .len(),
-            device_chat_msgs_before + 1
-        );
-
-        // delete device chat
-        device_chat_id.delete(&t.ctx).await.ok();
-
-        // check, that the chatlist is empty
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 0);
-
-        // a subsequent call to update_device_chats() must not re-add manally deleted messages or chats
-        t.ctx.update_device_chats().await.ok();
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 0);
-    }
-}

src/test_utils.rs

@@ -1,40 +1,70 @@
 //! Utilities to help writing tests.
 //!
-//! This module is only compiled for test runs.
+//! This private module is only compiled for test runs.
 
+use std::collections::BTreeMap;
+use std::fmt;
+use std::ops::Deref;
 use std::str::FromStr;
 use std::time::{Duration, Instant};
 
+use ansi_term::Color;
+use async_std::future::Future;
 use async_std::path::PathBuf;
-use async_std::sync::RwLock;
+use async_std::pin::Pin;
+use async_std::sync::{Arc, RwLock};
+use chat::ChatItem;
+use once_cell::sync::Lazy;
 use tempfile::{tempdir, TempDir};
 
-use crate::chat;
-use crate::chat::{ChatId, ChatItem};
+use crate::chat::{self, Chat, ChatId};
+use crate::chatlist::Chatlist;
 use crate::config::Config;
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF, DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
+use crate::contact::{Contact, Origin};
 use crate::context::Context;
 use crate::dc_receive_imf::dc_receive_imf;
 use crate::dc_tools::EmailAddress;
+use crate::events::{Event, EventType};
 use crate::job::Action;
 use crate::key::{self, DcKey};
-use crate::message::Message;
+use crate::message::{update_msg_state, Message, MessageState, MsgId};
 use crate::mimeparser::MimeMessage;
 use crate::param::{Param, Params};
 
+type EventSink =
+    dyn Fn(Event) -> Pin<Box<dyn Future<Output = ()> + Send + 'static>> + Send + Sync + 'static;
+
+/// Map of [`Context::id`] to names for [`TestContext`]s.
+static CONTEXT_NAMES: Lazy<std::sync::RwLock<BTreeMap<u32, String>>> =
+    Lazy::new(|| std::sync::RwLock::new(BTreeMap::new()));
+
 /// A Context and temporary directory.
 ///
 /// The temporary directory can be used to store the SQLite database,
 /// see e.g. [test_context] which does this.
-#[derive(Debug)]
 pub(crate) struct TestContext {
     pub ctx: Context,
     pub dir: TempDir,
     /// Counter for fake IMAP UIDs in [recv_msg], for private use in that function only.
     recv_idx: RwLock<u32>,
+    /// Functions to call for events received.
+    event_sinks: Arc<RwLock<Vec<Box<EventSink>>>>,
+}
+
+impl fmt::Debug for TestContext {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("TestContext")
+            .field("ctx", &self.ctx)
+            .field("dir", &self.dir)
+            .field("recv_idx", &self.recv_idx)
+            .field("event_sinks", &String::from("Vec<EventSink>"))
+            .finish()
+    }
 }
 
 impl TestContext {
-    /// Create a new [TestContext].
+    /// Creates a new [`TestContext`].
     ///
     /// The [Context] will be created and have an SQLite database named "db.sqlite" in the
     /// [TestContext.dir] directory.  This directory is cleaned up when the [TestContext] is
@@ -42,44 +72,99 @@ impl TestContext {
     ///
     /// [Context]: crate::context::Context
     pub async fn new() -> Self {
+        Self::new_named(None).await
+    }
+
+    /// Creates a new [`TestContext`] with a set name used in event logging.
+    pub async fn with_name(name: impl Into<String>) -> Self {
+        Self::new_named(Some(name.into())).await
+    }
+
+    async fn new_named(name: Option<String>) -> Self {
         use rand::Rng;
 
         let dir = tempdir().unwrap();
         let dbfile = dir.path().join("db.sqlite");
         let id = rand::thread_rng().gen();
+        if let Some(name) = name {
+            let mut context_names = CONTEXT_NAMES.write().unwrap();
+            context_names.insert(id, name);
+        }
         let ctx = Context::new("FakeOS".into(), dbfile.into(), id)
             .await
             .unwrap();
+
+        let events = ctx.get_event_emitter();
+        let event_sinks: Arc<RwLock<Vec<Box<EventSink>>>> = Arc::new(RwLock::new(Vec::new()));
+        let sinks = Arc::clone(&event_sinks);
+        async_std::task::spawn(async move {
+            while let Some(event) = events.recv().await {
+                {
+                    let sinks = sinks.read().await;
+                    for sink in sinks.iter() {
+                        sink(event.clone()).await;
+                    }
+                }
+                receive_event(event);
+            }
+        });
+
         Self {
             ctx,
             dir,
             recv_idx: RwLock::new(0),
+            event_sinks,
         }
     }
 
-    /// Create a new configured [TestContext].
+    /// Creates a new configured [`TestContext`].
     ///
-    /// This is a shortcut which automatically calls [TestContext::configure_alice] after
+    /// This is a shortcut which automatically calls [`TestContext::configure_alice`] after
     /// creating the context.
     pub async fn new_alice() -> Self {
-        let t = Self::new().await;
+        let t = Self::with_name("alice").await;
         t.configure_alice().await;
         t
     }
 
-    /// Create a new configured [TestContext].
+    /// Creates a new configured [`TestContext`].
     ///
     /// This is a shortcut which configures bob@example.net with a fixed key.
     pub async fn new_bob() -> Self {
-        let t = Self::new().await;
+        let t = Self::with_name("bob").await;
         let keypair = bob_keypair();
         t.configure_addr(&keypair.addr.to_string()).await;
-        key::store_self_keypair(&t.ctx, &keypair, key::KeyPairUse::Default)
+        key::store_self_keypair(&t, &keypair, key::KeyPairUse::Default)
             .await
             .expect("Failed to save Bob's key");
         t
     }
 
+    /// Sets a name for this [`TestContext`] if one isn't yet set.
+    ///
+    /// This will show up in events logged in the test output.
+    pub fn set_name(&self, name: impl Into<String>) {
+        let mut context_names = CONTEXT_NAMES.write().unwrap();
+        context_names
+            .entry(self.ctx.get_id())
+            .or_insert_with(|| name.into());
+    }
+
+    /// Add a new callback which will receive events.
+    ///
+    /// The test context runs an async task receiving all events from the [`Context`], which
+    /// are logged to stdout.  This allows you to register additional callbacks which will
+    /// receive all events in case your tests need to watch for a specific event.
+    pub async fn add_event_sink<F, R>(&self, sink: F)
+    where
+        // Aka `F: EventSink` but type aliases are not allowed.
+        F: Fn(Event) -> R + Send + Sync + 'static,
+        R: Future<Output = ()> + Send + 'static,
+    {
+        let mut sinks = self.event_sinks.write().await;
+        sinks.push(Box::new(move |evt| Box::pin(sink(evt))));
+    }
+
     /// Configure with alice@example.com.
     ///
     /// The context will be fake-configured as the alice user, with a pre-generated secret
@@ -107,9 +192,12 @@ impl TestContext {
             .set_config(Config::Configured, Some("1"))
             .await
             .unwrap();
+        if let Some(name) = addr.split('@').next() {
+            self.set_name(name);
+        }
     }
 
-    /// Retrieve a sent message from the jobs table.
+    /// Retrieves a sent message from the jobs table.
     ///
     /// This retrieves and removes a message which has been scheduled to send from the jobs
     /// table.  Messages are returned in the order they have been sent.
@@ -126,7 +214,7 @@ impl TestContext {
                     SELECT id, foreign_id, param
                       FROM jobs
                      WHERE action=?
-                  ORDER BY desired_timestamp;
+                  ORDER BY desired_timestamp DESC;
                 "#,
                     paramsv![Action::SendMsgToSmtp],
                     |row| {
@@ -146,7 +234,7 @@ impl TestContext {
                 panic!("no sent message found in jobs table");
             }
         };
-        let id = ChatId::new(foreign_id as u32);
+        let id = MsgId::new(foreign_id as u32);
         let params = Params::from_str(&raw_params).unwrap();
         let blob_path = params
             .get_blob(Param::File, &self.ctx, false)
@@ -159,14 +247,15 @@ impl TestContext {
             .execute("DELETE FROM jobs WHERE id=?;", paramsv![rowid])
             .await
             .expect("failed to remove job");
+        update_msg_state(&self.ctx, id, MessageState::OutDelivered).await;
         SentMessage {
-            id,
             params,
             blob_path,
+            sender_msg_id: id,
         }
     }
 
-    /// Parse a message.
+    /// Parses a message.
     ///
     /// Parsing a message does not run the entire receive pipeline, but is not without
     /// side-effects either.  E.g. if the message includes autocrypt headers the relevant
@@ -185,15 +274,19 @@ impl TestContext {
     pub async fn recv_msg(&self, msg: &SentMessage) {
         let mut idx = self.recv_idx.write().await;
         *idx += 1;
-        dc_receive_imf(&self.ctx, msg.payload().as_bytes(), "INBOX", *idx, false)
+        let received_msg =
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n"
+                .to_owned()
+                + &msg.payload();
+        dc_receive_imf(&self.ctx, received_msg.as_bytes(), "INBOX", *idx, false)
             .await
             .unwrap();
     }
 
-    /// Get the most recent message of a chat.
+    /// Gets the most recent message of a chat.
     ///
     /// Panics on errors or if the most recent message is a marker.
-    pub async fn get_last_msg(&self, chat_id: ChatId) -> Message {
+    pub async fn get_last_msg_in(&self, chat_id: ChatId) -> Message {
         let msgs = chat::get_chat_msgs(&self.ctx, chat_id, 0, None).await;
         let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() {
             msg_id
@@ -202,6 +295,119 @@ impl TestContext {
         };
         Message::load_from_db(&self.ctx, *msg_id).await.unwrap()
     }
+
+    /// Gets the most recent message over all chats.
+    pub async fn get_last_msg(&self) -> Message {
+        let chats = Chatlist::try_load(&self.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(chats.len() - 1).unwrap();
+        Message::load_from_db(&self.ctx, msg_id).await.unwrap()
+    }
+
+    /// Creates or returns an existing 1:1 [`Chat`] with another account.
+    ///
+    /// This first creates a contact using the configured details on the other account, then
+    /// creates a 1:1 chat with this contact.
+    pub async fn create_chat(&self, other: &TestContext) -> Chat {
+        let (contact_id, _modified) = Contact::add_or_lookup(
+            self,
+            other
+                .ctx
+                .get_config(Config::Displayname)
+                .await
+                .unwrap_or_default(),
+            other.ctx.get_config(Config::ConfiguredAddr).await.unwrap(),
+            Origin::ManuallyCreated,
+        )
+        .await
+        .unwrap();
+
+        let chat_id = chat::create_by_contact_id(self, contact_id).await.unwrap();
+        Chat::load_from_db(self, chat_id).await.unwrap()
+    }
+
+    /// Creates or returns an existing [`Contact`] and 1:1 [`Chat`] with another email.
+    ///
+    /// This first creates a contact from the `name` and `addr` and then creates a 1:1 chat
+    /// with this contact.
+    pub async fn create_chat_with_contact(&self, name: &str, addr: &str) -> Chat {
+        let contact = Contact::create(self, name, addr)
+            .await
+            .expect("failed to create contact");
+        let chat_id = chat::create_by_contact_id(self, contact).await.unwrap();
+        Chat::load_from_db(self, chat_id).await.unwrap()
+    }
+
+    /// Retrieves the "self" chat.
+    pub async fn get_self_chat(&self) -> Chat {
+        let chat_id = chat::create_by_contact_id(self, DC_CONTACT_ID_SELF)
+            .await
+            .unwrap();
+        Chat::load_from_db(self, chat_id).await.unwrap()
+    }
+
+    /// Sends out the text message.
+    ///
+    /// This is not hooked up to any SMTP-IMAP pipeline, so the other account must call
+    /// [`TestContext::recv_msg`] with the returned [`SentMessage`] if it wants to receive
+    /// the message.
+    pub async fn send_text(&self, chat_id: ChatId, txt: &str) -> SentMessage {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some(txt.to_string()));
+        chat::prepare_msg(&self, chat_id, &mut msg).await.unwrap();
+        chat::send_msg(&self, chat_id, &mut msg).await.unwrap();
+        self.pop_sent_msg().await
+    }
+
+    /// Prints out the entire chat to stdout.
+    ///
+    /// You can use this to debug your test by printing the entire chat conversation.
+    // This code is mainly the same as `log_msglist` in `cmdline.rs`, so one day, we could
+    // merge them to a public function in the `deltachat` crate.
+    #[allow(dead_code)]
+    pub async fn print_chat(&self, chat: &Chat) {
+        let msglist = chat::get_chat_msgs(&self, chat.get_id(), 0x1, None).await;
+        let msglist: Vec<MsgId> = msglist
+            .into_iter()
+            .map(|x| match x {
+                ChatItem::Message { msg_id } => msg_id,
+                ChatItem::Marker1 => MsgId::new(DC_MSG_ID_MARKER1),
+                ChatItem::DayMarker { .. } => MsgId::new(DC_MSG_ID_DAYMARKER),
+            })
+            .collect();
+
+        let mut lines_out = 0;
+        for msg_id in msglist {
+            if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
+                println!(
+                "--------------------------------------------------------------------------------"
+            );
+
+                lines_out += 1
+            } else if !msg_id.is_special() {
+                if lines_out == 0 {
+                    println!(
+                    "--------------------------------------------------------------------------------",
+                );
+                    lines_out += 1
+                }
+                let msg = Message::load_from_db(&self, msg_id).await.unwrap();
+                log_msg(self, "", &msg).await;
+            }
+        }
+        if lines_out > 0 {
+            println!(
+                "--------------------------------------------------------------------------------"
+            );
+        }
+    }
+}
+
+impl Deref for TestContext {
+    type Target = Context;
+
+    fn deref(&self) -> &Context {
+        &self.ctx
+    }
 }
 
 /// A raw message as it was scheduled to be sent.
@@ -210,17 +416,12 @@ impl TestContext {
 /// passed through a SMTP-IMAP pipeline.
 #[derive(Debug, Clone)]
 pub struct SentMessage {
-    id: ChatId,
     params: Params,
     blob_path: PathBuf,
+    pub sender_msg_id: MsgId,
 }
 
 impl SentMessage {
-    /// The ChatId the message belonged to.
-    pub fn id(&self) -> ChatId {
-        self.id
-    }
-
     /// A recipient the message was destined for.
     ///
     /// If there are multiple recipients this is just a random one, so is not very useful.
@@ -244,7 +445,7 @@ impl SentMessage {
 /// This saves CPU cycles by avoiding having to generate a key.
 ///
 /// The keypair was created using the crate::key::tests::gen_key test.
-pub(crate) fn alice_keypair() -> key::KeyPair {
+pub fn alice_keypair() -> key::KeyPair {
     let addr = EmailAddress::new("alice@example.com").unwrap();
     let public =
         key::SignedPublicKey::from_base64(include_str!("../test-data/key/alice-public.asc"))
@@ -262,7 +463,7 @@ pub(crate) fn alice_keypair() -> key::KeyPair {
 /// Load a pre-generated keypair for bob@example.net from disk.
 ///
 /// Like [alice_keypair] but a different key and identity.
-pub(crate) fn bob_keypair() -> key::KeyPair {
+pub fn bob_keypair() -> key::KeyPair {
     let addr = EmailAddress::new("bob@example.net").unwrap();
     let public =
         key::SignedPublicKey::from_base64(include_str!("../test-data/key/bob-public.asc")).unwrap();
@@ -274,3 +475,147 @@ pub(crate) fn bob_keypair() -> key::KeyPair {
         secret,
     }
 }
+
+/// Gets a specific message from a chat and asserts that the chat has a specific length.
+///
+/// Panics if the length of the chat is not `asserted_msgs_count` or if the chat item at `index` is not a Message.
+#[allow(clippy::indexing_slicing)]
+pub(crate) async fn get_chat_msg(
+    t: &TestContext,
+    chat_id: ChatId,
+    index: usize,
+    asserted_msgs_count: usize,
+) -> Message {
+    let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None).await;
+    assert_eq!(msgs.len(), asserted_msgs_count);
+    let msg_id = if let ChatItem::Message { msg_id } = msgs[index] {
+        msg_id
+    } else {
+        panic!("Wrong item type");
+    };
+    Message::load_from_db(&t.ctx, msg_id).await.unwrap()
+}
+
+/// Pretty-print an event to stdout
+///
+/// Done during tests this is captured by `cargo test` and associated with the test itself.
+fn receive_event(event: Event) {
+    let green = Color::Green.normal();
+    let yellow = Color::Yellow.normal();
+    let red = Color::Red.normal();
+
+    let msg = match event.typ {
+        EventType::Info(msg) => format!("INFO: {}", msg),
+        EventType::SmtpConnected(msg) => format!("[SMTP_CONNECTED] {}", msg),
+        EventType::ImapConnected(msg) => format!("[IMAP_CONNECTED] {}", msg),
+        EventType::SmtpMessageSent(msg) => format!("[SMTP_MESSAGE_SENT] {}", msg),
+        EventType::Warning(msg) => format!("WARN: {}", yellow.paint(msg)),
+        EventType::Error(msg) => format!("ERROR: {}", red.paint(msg)),
+        EventType::ErrorNetwork(msg) => format!("{}", red.paint(format!("[NETWORK] msg={}", msg))),
+        EventType::ErrorSelfNotInGroup(msg) => {
+            format!("{}", red.paint(format!("[SELF_NOT_IN_GROUP] {}", msg)))
+        }
+        EventType::MsgsChanged { chat_id, msg_id } => format!(
+            "{}",
+            green.paint(format!(
+                "Received MSGS_CHANGED(chat_id={}, msg_id={})",
+                chat_id, msg_id,
+            ))
+        ),
+        EventType::ContactsChanged(_) => format!("{}", green.paint("Received CONTACTS_CHANGED()")),
+        EventType::LocationChanged(contact) => format!(
+            "{}",
+            green.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
+        ),
+        EventType::ConfigureProgress { progress, comment } => {
+            if let Some(comment) = comment {
+                format!(
+                    "{}",
+                    green.paint(format!(
+                        "Received CONFIGURE_PROGRESS({} ‰, {})",
+                        progress, comment
+                    ))
+                )
+            } else {
+                format!(
+                    "{}",
+                    green.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+                )
+            }
+        }
+        EventType::ImexProgress(progress) => format!(
+            "{}",
+            green.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
+        ),
+        EventType::ImexFileWritten(file) => format!(
+            "{}",
+            green.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
+        ),
+        EventType::ChatModified(chat) => format!(
+            "{}",
+            green.paint(format!("Received CHAT_MODIFIED({})", chat))
+        ),
+        _ => format!("Received {:?}", event),
+    };
+    let context_names = CONTEXT_NAMES.read().unwrap();
+    match context_names.get(&event.id) {
+        Some(ref name) => println!("{} {}", name, msg),
+        None => println!("{} {}", event.id, msg),
+    }
+}
+
+/// Logs and individual message to stdout.
+///
+/// This includes a bunch of the message meta-data as well.
+async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
+    let contact = Contact::get_by_id(context, msg.get_from_id())
+        .await
+        .expect("invalid contact");
+
+    let contact_name = contact.get_name();
+    let contact_id = contact.get_id();
+
+    let statestr = match msg.get_state() {
+        MessageState::OutPending => " o",
+        MessageState::OutDelivered => " √",
+        MessageState::OutMdnRcvd => " √√",
+        MessageState::OutFailed => " !!",
+        _ => "",
+    };
+    let msgtext = msg.get_text();
+    println!(
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}",
+        prefix.as_ref(),
+        msg.get_id(),
+        if msg.get_showpadlock() { "🔒" } else { "" },
+        if msg.has_location() { "📍" } else { "" },
+        &contact_name,
+        contact_id,
+        msgtext.unwrap_or_default(),
+        if msg.get_from_id() == 1u32 {
+            ""
+        } else if msg.get_state() == MessageState::InSeen {
+            "[SEEN]"
+        } else if msg.get_state() == MessageState::InNoticed {
+            "[NOTICED]"
+        } else {
+            "[FRESH]"
+        },
+        if msg.is_info() { "[INFO]" } else { "" },
+        if msg.get_viewtype() == Viewtype::VideochatInvitation {
+            format!(
+                "[VIDEOCHAT-INVITATION: {}, type={}]",
+                msg.get_videochat_url().unwrap_or_default(),
+                msg.get_videochat_type().unwrap_or_default()
+            )
+        } else {
+            "".to_string()
+        },
+        if msg.is_forwarded() {
+            "[FORWARDED]"
+        } else {
+            ""
+        },
+        statestr,
+    );
+}

src/token.rs

@@ -4,11 +4,11 @@
 //!
 //! Tokens are used in countermitm verification protocols.
 
-use deltachat_derive::*;
+use deltachat_derive::{FromSql, ToSql};
 
 use crate::chat::ChatId;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_create_id, time};
 
 /// Token namespace
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
@@ -26,38 +26,71 @@ impl Default for Namespace {
 }
 
 /// Creates a new token and saves it into the database.
+///
 /// Returns created token.
-pub async fn save(context: &Context, namespace: Namespace, foreign_id: ChatId) -> String {
-    // foreign_id may be 0
+pub async fn save(context: &Context, namespace: Namespace, chat: Option<ChatId>) -> String {
     let token = dc_create_id();
-    context
-        .sql
-        .execute(
-            "INSERT INTO tokens (namespc, foreign_id, token, timestamp) VALUES (?, ?, ?, ?);",
-            paramsv![namespace, foreign_id, token, time()],
-        )
-        .await
-        .ok();
+    match chat {
+        Some(chat_id) => context
+            .sql
+            .execute(
+                "INSERT INTO tokens (namespc, foreign_id, token, timestamp) VALUES (?, ?, ?, ?);",
+                paramsv![namespace, chat_id, token, time()],
+            )
+            .await
+            .ok(),
+        None => context
+            .sql
+            .execute(
+                "INSERT INTO tokens (namespc, token, timestamp) VALUES (?, ?, ?);",
+                paramsv![namespace, token, time()],
+            )
+            .await
+            .ok(),
+    };
     token
 }
 
-pub async fn lookup(context: &Context, namespace: Namespace, foreign_id: ChatId) -> Option<String> {
-    context
-        .sql
-        .query_get_value::<String>(
-            context,
-            "SELECT token FROM tokens WHERE namespc=? AND foreign_id=?;",
-            paramsv![namespace, foreign_id],
-        )
-        .await
+pub async fn lookup(
+    context: &Context,
+    namespace: Namespace,
+    chat: Option<ChatId>,
+) -> Option<String> {
+    match chat {
+        Some(chat_id) => {
+            context
+                .sql
+                .query_get_value::<String>(
+                    context,
+                    "SELECT token FROM tokens WHERE namespc=? AND foreign_id=?;",
+                    paramsv![namespace, chat_id],
+                )
+                .await
+        }
+        // foreign_id is declared as `INTEGER DEFAULT 0` in the schema.
+        None => {
+            context
+                .sql
+                .query_get_value::<String>(
+                    context,
+                    "SELECT token FROM tokens WHERE namespc=? AND foreign_id=0;",
+                    paramsv![namespace],
+                )
+                .await
+        }
+    }
 }
 
-pub async fn lookup_or_new(context: &Context, namespace: Namespace, foreign_id: ChatId) -> String {
-    if let Some(token) = lookup(context, namespace, foreign_id).await {
+pub async fn lookup_or_new(
+    context: &Context,
+    namespace: Namespace,
+    chat: Option<ChatId>,
+) -> String {
+    if let Some(token) = lookup(context, namespace, chat).await {
         return token;
     }
 
-    save(context, namespace, foreign_id).await
+    save(context, namespace, chat).await
 }
 
 pub async fn exists(context: &Context, namespace: Namespace, token: &str) -> bool {

standards.md

@@ -6,6 +6,8 @@ Tasks                            | Standards
 ---------------------------------|---------------------------------------------
 Transport                        | IMAP v4 ([RFC 3501](https://tools.ietf.org/html/rfc3501)), SMTP ([RFC 5321](https://tools.ietf.org/html/rfc5321)) and Internet Message Format (IMF, [RFC 5322](https://tools.ietf.org/html/rfc5322))
 Embedded media                   | MIME Document Series ([RFC 2045](https://tools.ietf.org/html/rfc2045), [RFC 2046](https://tools.ietf.org/html/rfc2046)), Content-Disposition Header ([RFC 2183](https://tools.ietf.org/html/rfc2183)), Multipart/Related ([RFC 2387](https://tools.ietf.org/html/rfc2387))
+Text and Quote encoding          | Fixed, Flowed ([RFC 3676](https://tools.ietf.org/html/rfc3676))
+Filename encoding                | Encoded Words ([RFC 2047](https://tools.ietf.org/html/rfc2047)), Encoded Word Extensions ([RFC 2231](https://tools.ietf.org/html/rfc2231))
 Identify server folders          | IMAP LIST Extension ([RFC 6154](https://tools.ietf.org/html/rfc6154))
 Push                             | IMAP IDLE ([RFC 2177](https://tools.ietf.org/html/rfc2177))
 Authorization                    | OAuth2 ([RFC 6749](https://tools.ietf.org/html/rfc6749))
@@ -13,6 +15,7 @@ End-to-end encryption            | [Autocrypt Level 1](https://autocrypt.org/lev
 Configuration assistance         | [Autoconfigure](https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover](https://technet.microsoft.com/library/bb124251(v=exchg.150).aspx)
 Messenger functions              | [Chat-over-Email](https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#chat-over-email-specification)
 Detect mailing list              | List-Id ([RFC 2919](https://tools.ietf.org/html/rfc2919)) and Precedence ([RFC 3834](https://tools.ietf.org/html/rfc3834))
+User and chat colors             | [XEP-0392](https://xmpp.org/extensions/xep-0392.html): Consistent Color Generation
 Send and receive system messages | Multipart/Report Media Type ([RFC 6522](https://tools.ietf.org/html/rfc6522))
 Return receipts                  | Message Disposition Notification (MDN, [RFC 8098](https://tools.ietf.org/html/rfc8098), [RFC 3503](https://tools.ietf.org/html/rfc3503)) using the Chat-Disposition-Notification-To header
 Locations                        | KML ([Open Geospatial Consortium](http://www.opengeospatial.org/standards/kml/), [Google Dev](https://developers.google.com/kml/))

test-data/message/AutocryptSetupMessage.eml

@@ -0,0 +1,77 @@
+Return-Path: <alice@example.com>
+Delivered-To: alice@example.com
+Received: from hq5.merlinux.eu
+	by hq5.merlinux.eu with LMTP
+	id gNKpOrrTvF+tVAAAPzvFDg
+	(envelope-from <alice@example.com>)
+	for <alice@example.com>; Tue, 24 Nov 2020 10:34:50 +0100
+Subject: Autocrypt Setup Message
+DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=testrun.org;
+	s=testrun; t=1606210490;
+	bh=MXqLqHFK1xC48pxx2TS1GUdxKSi4tdejRRSV4EAN5Tc=;
+	h=Subject:Date:To:From:From;
+	b=DRajftyu+Ycfhaxy0jXAIKCihQRMI0rxbo9+EBu6y5jhtZx13emW3odgZnvyhU6uD
+	 IKfMXaqlmc/2HNV1/mloJVIRsIp5ORncSPX9tLykNApJVyPHg3NKdMo3Ib4NGIJ1Qo
+	 binmLtL5qqL3bYCL68WUgieH1rcgCaf9cwck9GvwZ79pexGuWz4ItgtNWqYfapG8Zc
+	 9eD5maiTMNkV7UwgtOzhbBd39uKgKCoGdLAq63hoJF6dhdBBRVRyRMusAooGUZMgwm
+	 QVuTZ76z9G8w3rDgZuHmoiICWsLsar4CDl4zAgicE6bHwtw3a7YuMiHoCtceq0RjQP
+	 BHVaXT7B75BoA==
+MIME-Version: 1.0
+Date: Tue, 24 Nov 2020 09:34:48 +0000
+Chat-Version: 1.0
+Autocrypt-Setup-Message: v1
+Message-ID: <Mr._G2UTiTkgfk.HWf5RnFC2xy@testrun.org>
+To: <alice@example.com>
+From:  <alice@example.com>
+Content-Type: multipart/mixed; boundary="dKhu3bbmBniQsT8W8w58YRCCiBK2YY"
+
+
+--dKhu3bbmBniQsT8W8w58YRCCiBK2YY
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+This is the Autocrypt Setup Message used to transfer your end-to-end setup 
+between clients.
+
+To decrypt and use your setup, open the message in an Autocrypt-compliant 
+client and enter the setup code presented on the generating device.
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat
+
+--dKhu3bbmBniQsT8W8w58YRCCiBK2YY
+Content-Type: application/autocrypt-setup
+Content-Disposition: attachment; filename="autocrypt-setup-message.html"
+Content-Transfer-Encoding: base64
+
+PCFET0NUWVBFIGh0bWw+DQo8aHRtbD4NCiAgPGhlYWQ+DQogICAgPHRpdGxlPkF1dG9jcnlwdCBTZX
+R1cCBNZXNzYWdlPC90aXRsZT4NCiAgPC9oZWFkPg0KICA8Ym9keT4NCiAgICA8aDE+QXV0b2NyeXB0
+IFNldHVwIE1lc3NhZ2U8L2gxPg0KICAgIDxwPlRoaXMgaXMgdGhlIEF1dG9jcnlwdCBTZXR1cCBNZX
+NzYWdlIHVzZWQgdG8gdHJhbnNmZXIgeW91ciBlbmQtdG8tZW5kIHNldHVwIGJldHdlZW4gY2xpZW50
+cy48YnI+PGJyPlRvIGRlY3J5cHQgYW5kIHVzZSB5b3VyIHNldHVwLCBvcGVuIHRoZSBtZXNzYWdlIG
+luIGFuIEF1dG9jcnlwdC1jb21wbGlhbnQgY2xpZW50IGFuZCBlbnRlciB0aGUgc2V0dXAgY29kZSBw
+cmVzZW50ZWQgb24gdGhlIGdlbmVyYXRpbmcgZGV2aWNlLjwvcD4NCiAgICA8cHJlPg0KLS0tLS1CRU
+dJTiBQR1AgTUVTU0FHRS0tLS0tDQpQYXNzcGhyYXNlLUZvcm1hdDogbnVtZXJpYzl4NA0KUGFzc3Bo
+cmFzZS1CZWdpbjogNjIKCnd4NEVCd01JWEUzNCs4RGhtSC9nRDNNY21JTjhCSUorbmhpbDMrOFE3bF
+hTd21JQnhDSnhBU2VhQUJlTGdFOTIKTi9WaER5MHlrUHFBQkp0S0xvSG9pQmxTQWZJajFRemdPeVlV
+Wjl3czRtSng5OVREUE1lSnNmNHJaemJhUHZFSApQcEIrTTgyTjVhUitvV0dTcWRtUUZNQUplNWNtWX
+hwM3p4eE5aTEc2cXVnRzUzOFNxNUV1TzBDSGduaXlFeEwyCkJya2hFOWVFVE1oSkNRQ3dCZDc5alhN
+U2Mwcm5xYjFHbS9Kd21jbXFqVFNHMlBLTWNNcFlaV1QwQkNMaDE2TmwKTkNNbmRQWGt2cTlHd1crNX
+pEMHc4cElyOERNRHk1SWVBcG83amNZR1U5UWNUR1lMWmltR2QxK1RYYlgvdGxqRQplMnNZd0hZeU5D
+R1N5bHVsYi9XYnNkS2FrYXVodHJ6cUVHOXNYSkJkMnF5ZjNJajRULzdCd1pJVk42OXF1T21sCnlmMm
+9PTmtYY1pCcFBJUE9ZQzdhMnJ5aFh0Q0NhbWhIVEw0czdzclg2NzJXMTVXS3VqNGVBK25URlNocFBC
+cXoKb05EY3QzbG95V0hNSUluSzRha1VJeTFZak42TDFSbGwwRVhudlVQS0lkT0FpY0swbFBPaDVUZU
+t6ZFMvTklyMQpQc2x6c2RyWTRZd0diMWNTdk95OXJQRFpaS3Y4d0dzbFczcFpFOCs3NnJWckllbkNY
+dTdvOUZ6OFhQcVlxTGRrCkpCZGRHUGZnY0l6Um5nZjZqb0lmT0RsU2NiajR0VlgyK3htVVN5RlVhSD
+RQcDFzZDgwVjhDN2xhREJ2WTc0TlAKQW9ydEVhL2xGbzQzcHNOdlhrc0JUUEVRNHFoTVZneVdQWW9V
+ZGV2aUFZOGVDMmJjT0dMSFVURk5zaHZCaDFGRgozVGpIZEVRVk5zZVlqaWtZRWtkUU9Mb3B5VWdqbj
+lSTUJnV2xIZTNKL1VRcmtFUkNYWi9BSVRXeGdYdmE0NHBPCkkzUHllcnF2T1lpVlJLam9JSTVIZGU4
+UFdkTnZwb2J5ZCsrTHlqN3Jxd0kyNFRwbVRwYWtIZ1RJNEJvYWtLSUcKWm1JWDhsQm4xMnQ5dlcvcD
+lrbDluYWluS3Z1VFBoTk4xZmkrTE1YYTRDK1hqRXVPUnQwMFMzc01MdVo3RnBPaQprcXdGWk12RUtw
+bHA3dmRLSnJNbmVzZ2dKLzBLeWc1RTJ4dVd2VFdkZUFBOE1saEJqSGlsK3JVK0dSZzdaTmxsCkxUej
+RKeGpWUVl5TGpFbkhqdGU4bUVnZlNIZEE3ZDErVnV1RTZSZjlYMzRPeXhkL3NocllJSU8xY3FVdnQw
+V3MKNGIwQURIN0lkbjkveTdDRjVrbWFONkMyQURBRkhFRzNIRWFZaDVNNmIwVzVJSW55WkhUQ0QxdC
+tmUFdQYndxUQo0TzFRMEROZ01QT1FCRVJ0ODNXR3g5YW5GQU9YCj05dTUrCi0tLS0tRU5EIFBHUCBN
+RVNTQUdFLS0tLS0KDQo8L3ByZT4NCiAgPC9ib2R5Pg0KPC9odG1sPg0K
+
+--dKhu3bbmBniQsT8W8w58YRCCiBK2YY--
+

test-data/message/apple_cid_jpg.eml

@@ -0,0 +1,53 @@
+From: =?utf-8?Q?Bj=C3=B6rn_Petersen?= <somewhere-apple@me.com>
+Content-Type: multipart/alternative;
+	boundary="Apple-Mail=_19251BCB-E12B-423A-9553-5A68560C2AFD"
+Mime-Version: 1.0 (Mac OS X Mail 13.4 \(3608.120.23.2.4\))
+Subject: a jpeg
+Message-Id: <BC47DA72-6C78-443A-8EBF-2CD199ABAD09@me.com>
+Date: Sat, 9 Jan 2021 00:36:11 +0100
+To: somewhere-nonapple@testrun.org
+X-Mailer: Apple Mail (2.3608.120.23.2.4)
+
+
+--Apple-Mail=_19251BCB-E12B-423A-9553-5A68560C2AFD
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain;
+	charset=us-ascii
+
+
+a jpeg
+--Apple-Mail=_19251BCB-E12B-423A-9553-5A68560C2AFD
+Content-Type: multipart/related;
+	type="text/html";
+	boundary="Apple-Mail=_4C3710FD-D75D-47FB-8D41-983220390856"
+
+
+--Apple-Mail=_4C3710FD-D75D-47FB-8D41-983220390856
+Content-Transfer-Encoding: 7bit
+Content-Type: text/html;
+	charset=us-ascii
+
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=us-ascii"><base></head><body style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class=""><base class=""><div class="Apple-Mail-URLShareUserContentTopClass"><br class=""></div><div class="Apple-Mail-URLShareWrapperClass"><blockquote type="cite" style="border-left-style: none; color: inherit; padding: inherit; margin: inherit;" class="">a jpeg
+	<img apple-inline="yes" id="118F6150-5EF5-4DE8-917F-1851EC94FB7C" src="cid:8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box" class=""></blockquote></div></body></html>
+--Apple-Mail=_4C3710FD-D75D-47FB-8D41-983220390856
+Content-Transfer-Encoding: base64
+Content-Disposition: inline;
+	filename=small.jpg
+Content-Type: image/jpeg;
+	x-unix-mode=0666;
+	name="small.jpg"
+Content-Id: <8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box>
+
+/9j/4AAQSkZJRgABAQEBLAEsAAD/2wBDABELDA8MChEPDg8TEhEUGSobGRcXGTMkJh4qPDU/Pjs1
+OjlDS2BRQ0daSDk6U3FUWmNma2xrQFB2fnRofWBpa2f/2wBDARITExkWGTEbGzFnRTpFZ2dnZ2dn
+Z2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2f/wgARCAAIAAgDAREA
+AhEBAxEB/8QAFAABAAAAAAAAAAAAAAAAAAAABP/EABUBAQEAAAAAAAAAAAAAAAAAAAMF/9oADAMB
+AAIQAxAAAAF8s//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAQUCf//EABQRAQAAAAAAAAAA
+AAAAAAAAAAD/2gAIAQMBAT8Bf//EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQIBAT8Bf//EABQQ
+AQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEABj8Cf//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEA
+AT8hf//aAAwDAQACAAMAAAAQ3//EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQMBAT8Qf//EABQR
+AQAAAAAAAAAAAAAAAAAAAAD/2gAIAQIBAT8Qf//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEA
+AT8Qf//Z
+--Apple-Mail=_4C3710FD-D75D-47FB-8D41-983220390856--
+
+--Apple-Mail=_19251BCB-E12B-423A-9553-5A68560C2AFD--

test-data/message/attach_filename_apostrophed.eml

@@ -0,0 +1,24 @@
+Subject: Test apostrophed filenames
+Message-ID: 12345@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+X-Mailer: Kopano 8.7.16
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+apostrophed filenames as of
+https://tools.ietf.org/html/rfc2231
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment;
+ filename*=utf-8''Ma%C3%9Fnahmen%20Okt.%202021.html
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--

test-data/message/attach_filename_apostrophed_cont.eml

@@ -0,0 +1,34 @@
+Subject: Test apostrophed filenames
+Message-ID: 12345@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+X-Mailer: Kopano 8.7.16
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+apostrophed filenames as of
+https://tools.ietf.org/html/rfc2231,
+span over several header lines.
+
+note, that, in contrast to encoded-words,
+the character-set is not repeated.
+
+as a side-effect,
+this tests unquoted header attributes in filename*1*
+and lower-case-urlencoded utf-8
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment;
+ filename*0*="utf-8''Ma%C3%9Fna";
+ filename*1*=hm;
+ filename*2*="en%20M%c3%a4rz%202022.html";
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--

test-data/message/attach_filename_apostrophed_cp1252.eml

@@ -0,0 +1,23 @@
+Subject: Test apostrophed filenames
+Message-ID: 12345@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+X-Mailer: Kopano 8.7.16
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+testing cp1252 aka ANSI aka Windows-1252
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment;
+ filename*=Cp1252''Auftragsbest%E4tigung.pdf;
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--

test-data/message/attach_filename_apostrophed_invalid.eml

@@ -0,0 +1,30 @@
+Subject: Test apostrophed filenames
+Message-ID: 12345@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+this mail uses filename* in an invalid way,
+not specifying chatset or language parts.
+
+see https://tools.ietf.org/html/rfc2231, section 4.:
+"Note also that the single quote delimiters MUST be present
+even when one of the field values is omitted."
+
+however, for now, Delta Chat is generous in this case,
+just assuming encoded words
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment;
+ filename*="=?utf-8?Q?somed=C3=A4=C3=BCta.html.zip?="
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--

test-data/message/attach_filename_apostrophed_windows1251.eml

@@ -0,0 +1,29 @@
+Subject: Test apostrophed filenames
+Message-ID: 12345@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+X-Mailer: Kopano 8.7.16
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+apostrophed filenames as of
+https://tools.ietf.org/html/rfc2231,
+testing non-utf-8 charset
+
+examples:
+%EF%F0%EE%E3%F0%E0%EC%EC%E8%F0%EE%E2%E0%ED%E8%E5 = программирование = programming
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment;
+ filename*0*=windows-1251''%EF%F0%EE%E3%F0%E0%EC%EC%E8%F0%EE%E2%E0;
+ filename*1*=%ED%E8%E5.HTM
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--

test-data/message/attach_filename_combined.eml

@@ -0,0 +1,31 @@
+Subject: Test combined filenames
+Message-ID: 12345@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+X-Mailer: Kopano 8.7.16
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="NwkNRwaJw1M5n2xcr2ODfAqvTjcj9Z"
+
+
+--NwkNRwaJw1M5n2xcr2ODfAqvTjcj9Z
+Content-Type: text/plain; charset=utf-8
+
+yip, filename may be given twice,
+seen this way in Kopano.
+
+normally, both filenames will be the same,
+for the test, we use different filenames,
+we define that `filename` has a higher precedence.
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat
+
+--NwkNRwaJw1M5n2xcr2ODfAqvTjcj9Z
+Content-Type: text/html
+Content-Disposition: attachment; filename="=?utf-8?Q?Ma=C3=9Fnahmen_Okt=2E_2020=2Ehtml?=";
+ filename*=utf-8''Ma%C3%9Fnahmen%20Okt.%202020.ignored.html
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--NwkNRwaJw1M5n2xcr2ODfAqvTjcj9Z--

test-data/message/attach_filename_encoded_words.eml

@@ -0,0 +1,23 @@
+Subject: Test encoded-words filenames
+Message-ID: 123456@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+Chat-Version: 1.0
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="mwkNRwabw1M5n2xcr2ODfAqvTjcj9Z"
+
+
+--mwkNRwabw1M5n2xcr2ODfAqvTjcj9Z
+Content-Type: text/plain; charset=utf-8
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat
+
+--mwkNRwabw1M5n2xcr2ODfAqvTjcj9Z
+Content-Type: text/html
+Content-Disposition: attachment; filename="=?utf-8?Q?Ma=C3=9Fnahmen_Okt=2E_2020=2Ehtml?=";
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--mwkNRwabw1M5n2xcr2ODfAqvTjcj9Z--

test-data/message/attach_filename_encoded_words_bad_delimiter.eml

@@ -0,0 +1,29 @@
+Subject: Test encoded-words filenames
+Message-ID: 123456@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+Chat-Version: 1.0
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+there MUST be a space between encoded words and plain text,
+if there is none, decoding should return the original string
+https://tools.ietf.org/html/rfc2047 5.1:
+
+"Ordinary ASCII text and 'encoded-word's may appear together in the
+same header field.  However, an 'encoded-word' that appears in a
+header field defined as '*text' MUST be separated from any adjacent
+'encoded-word' or 'text' by 'linear-white-space'."
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment; filename="=?utf-8?q?foo?=.bar";
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--

test-data/message/attach_filename_encoded_words_binary.eml

@@ -0,0 +1,26 @@
+Subject: Test binary-encoded-words filenames
+Message-ID: 123456@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+Chat-Version: 1.0
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+test binary word-encoded filename,
+filename is " § 165 Abs" - note the leading space.
+
+as a side-effect, this also tests that the encoding-name
+also works in UPPERCASE.
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment; filename="=?UTF-8?B?IMKnIDE2NSBBYnM=?=";
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--

test-data/message/attach_filename_encoded_words_cont.eml

@@ -0,0 +1,25 @@
+Subject: Test encoded-words filenames, spanning different lines
+Message-ID: Mr.XA6y3og8-az.WGbH9_dNcQx@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+X-Mailer: Delta Chat Core 1.0.0-beta.12/DcFFI
+Chat-Version: 1.0
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="maaNRwaJw1M5n2xcr2ODfAqvTjcj9Z"
+
+
+--maaNRwaJw1M5n2xcr2ODfAqvTjcj9Z
+Content-Type: text/plain; charset=utf-8
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat
+
+--maaNRwaJw1M5n2xcr2ODfAqvTjcj9Z
+Content-Type: text/html
+Content-Disposition: attachment; filename*0="=?utf-8?Q?Ma=C3=9Fn'ah'men_?=";
+  filename*1="=?utf-8?Q?Okt=2E_2020=2Ehtml?=";
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--maaNRwaJw1M5n2xcr2ODfAqvTjcj9Z--

test-data/message/attach_filename_encoded_words_windows1251.eml

@@ -0,0 +1,31 @@
+Subject: Test encoded-words
+Message-ID: 123456@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+Chat-Version: 1.0
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+testing encoded-words filenames with windows-1251 (cyrillic) encoding.
+
+as a side-effect, this also tests that encoded words work together with
+plain text as long as they're separated by spaces, see
+https://tools.ietf.org/html/rfc2047 5.1:
+
+"Ordinary ASCII text and 'encoded-word's may appear together in the
+same header field.  However, an 'encoded-word' that appears in a
+header field defined as '*text' MUST be separated from any adjacent
+'encoded-word' or 'text' by 'linear-white-space'."
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment; filename="file =?Windows-1251?B?1/LuIO3u4u7j7g==?= 2020.pdf";
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--