add missing members receiving non-dc messages

* Save subject for sent-out messages

* Test that subject is saved (outgoing)

* Correctly set subject when forwarding messages, add test for this

.circleci/config.yml

@@ -7,9 +7,6 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
-  python:
-    docker:
-      - image: 3.7.7-stretch
             
 
 restore-workspace: &restore-workspace
@@ -156,8 +153,6 @@ jobs:
       - checkout
       - attach_workspace:
           at: workspace
-      - run: pyenv versions
-      - run: pyenv global 3.5.2
       - run: ls -laR workspace
       - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 
@@ -192,8 +187,8 @@ workflows:
           filters:
             branches:
               only: master
-            tags:
-              only: /.*/
+            #tags:
+            #  only: /.*/
 
       - upload_docs_wheels:
           requires:

.gitattributes

@@ -12,3 +12,6 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
+*.py diff=python
+*.rs diff=rust
+*.md diff=markdown

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.45.0
+          toolchain: 1.50.0
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: 1.45.0
+            toolchain: 1.50.0
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
@@ -40,6 +40,28 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --workspace --tests --examples
 
+  docs:
+    name: Rust doc comments
+    runs-on: ubuntu-latest
+    env:
+      RUSTDOCFLAGS: -Dwarnings
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - name: Install rust stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          components: rust-docs
+          override: true
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v1
+      - name: Rustdoc
+        uses: actions-rs/cargo@v1
+        with:
+          command: doc
+          args: --document-private-items --no-deps
 
   build_and_test:
     name: Build and test
@@ -50,7 +72,7 @@ jobs:
         # macOS disabled due to random failures related to caching
         #os: [ubuntu-latest, windows-latest, macOS-latest]
         os: [ubuntu-latest, windows-latest]
-        rust: [1.45.0]
+        rust: [1.50.0]
         experimental: [false]
 #        include:
 #          - os: ubuntu-latest

CHANGELOG.md

@@ -1,5 +1,177 @@
 # Changelog 
 
+## UNRELEASED
+
+- breaking change: You have to call dc_stop_io()/dc_start_io() before/after EXPORT_BACKUP:
+  fix race condition and db corruption when a message was received during backup #2253
+
+- save subject for messages:
+  new api `dc_msg_get_subject()`,
+  when quoting, use the subject of the quoted message as the new subject, instead of the
+  last subject in the chat
+
+- new apis to get full or html message,
+  `dc_msg_has_html()` and `dc_get_msg_html()` #2125 #2151
+
+- new chat type and apis for the new mailing list support,
+  `DC_CHAT_TYPE_MAILINGLIST`, `dc_msg_get_real_chat_id()`,
+  `dc_msg_get_override_sender_name()` #1964 #2181 #2185 #2195 #2211 #2210 #2240
+  #2243
+
+- new api `dc_decide_on_contact_request()`,
+  deprecated `dc_create_chat_by_msg_id()` and `dc_marknoticed_contact()` #1964
+
+- new flag `DC_GCM_INFO_ONLY` for api `dc_get_chat_msgs()` #2132
+
+- new api `dc_get_chat_encrinfo()` #2186
+
+- new api `dc_contact_get_status()`, returning the recent footer #2218
+
+- improve contact name update rules,
+  add api `dc_contact_get_auth_name()` #2206 #2212 #2225
+
+- new api for bots: `dc_msg_set_html()` #2153
+
+- new api for bots: `dc_msg_set_override_sender_name()` #2231
+
+- api removed: `dc_is_io_running()` #2139
+
+- api removed: `dc_contact_get_first_name()` #2165 #2171
+
+- implement Consistent Color Generation (XEP-0392),
+  that results in contact colors be be changed #2228 #2229 #2239
+
+- fetch recent existing messages
+  and create corresponding chats after configure #2106
+
+- improve e-mail compatibility
+  by scanning all folders from time to time #2067 #2152 #2158 #2184 #2215 #2224
+
+- better support videochat-services not supporting random rooms #2191
+
+- export backups as .tar files #2023
+
+- scale avatars based on media_quality, fix avatar rotation #2063
+
+- compare ephemeral timer to parent message to deal with reordering better #2100
+
+- better ephemeral system messages #2183
+
+- read quotes out of html messages #2104
+
+- prepend subject to messages with attachments, if needed #2111
+
+- run housekeeping at least once a day #2114
+
+- resolve MX domain only once per OAuth2 provider #2122
+
+- configure provider based on MX record #2123 #2134
+
+- make transient bad destination address error permanent
+  after n tries #2126 #2202
+
+- enable strict TLS for known providers by default #2121
+
+- improve and harden secure join #2154 #2161
+
+- update `dc_get_info()` to return more information #2156
+
+- prefer In-Reply-To/References
+  over group-id stored in Message-ID #2164 #2172 #2173
+
+- apply gossiped encryption preference to new peerstates #2174
+
+- fix: do not return quoted messages from the trash chat #2221
+
+- fix: allow emojis for location markers #2177
+
+- fix encoding of Chat-Group-Name-Changed messages that could even lead to
+  messages not being delivered #2141
+
+- fix error when no temporary directory is available #1929
+
+- fix marking read receipts as seen #2117
+
+- fix read-notification for mixed-case addresses #2103
+
+- fix decoding of attachment filenames #2080 #2094 #2102
+
+- fix downloading ranges of message #2061
+
+- fix parsing quoted encoded words in From: header #2193 #2204
+
+- fix ci #2217 #2226
+
+- try harder on backup opening #2148
+
+- switch to rust 1.50, update toolchains, deps #2150 #2155 #2165 #2107 #2262 #2271
+
+- improve python bindings #2113 #2115 #2133 #2214
+
+- improve documentation #2143 #2160 #2175 #2146
+
+- refactorings #2110 #2136 #2135 #2168 #2178 #2189 #2190 #2198 #2197 #2201 #2196
+  #2200 #2230
+
+
+## 1.50.0
+
+- do not fetch emails in between inbox_watch disabled and enabled again #2087
+
+- fix: do not fetch from INBOX if inbox_watch is disabled #2085
+
+- fix: do not use STARTTLS when PLAIN connection is requested
+  and do not allow downgrade if STARTTLS is not available #2071
+
+
+## 1.49.0
+
+- add timestamps to image and video filenames #2068
+
+- forbid quoting messages from another context #2069
+
+- fix: preserve quotes in messages with attachments #2070
+
+
+## 1.48.0
+
+- `fetch_existing` renamed to `fetch_existing_msgs` and disabled by default
+  #2035 #2042
+
+- skip fetch existing messages/contacts if config-option `bot` set #2017
+
+- always log why a message is sorted to trash #2045
+
+- display a quote if top posting is detected #2047
+
+- add ephemeral task cancellation to `dc_stop_io()`;
+  before, there was no way to quickly terminate pending ephemeral tasks #2051
+
+- when saved-messages chat is deleted,
+  a device-message about recreation is added #2050
+
+- use `max_smtp_rcpt_to` from provider-db,
+  sending messages to many recipients in configurable chunks #2056
+
+- fix handling of empty autoconfigure files #2027
+
+- fix adding saved messages to wrong chats on multi-device #2034 #2039
+
+- fix hang on android4.4 and other systems
+  by adding a workaround to executer-blocking-handling bug #2040
+
+- fix secret key export/import roundtrip #2048
+
+- fix mistakenly unarchived chats #2057
+
+- fix outdated-reminder test that fails only 7 days a year,
+  including halloween :) #2059
+
+- improve python bindings #2021 #2036 #2038
+
+- update provider-database #2037
+
+
 ## 1.47.0
 
 - breaking change: `dc_update_device_chats()` removed;
@@ -27,6 +199,7 @@
 
 - configure now collects recent contacts and fetches last messages
   unless disabled by `fetch_existing` config-option #1913 #2003
+  EDIT: `fetch_existing` renamed to `fetch_existing_msgs` in 1.48.0 #2042
 
 - emit `DC_EVENT_CHAT_MODIFIED` on contact rename
   and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
@@ -40,6 +213,8 @@
 
 - mark all failed messages as failed when receiving an NDN #1993
 
+- check some easy cases for bad system clock and outdated app #1901
+
 - fix import temporary directory usage #1929
 
 - fix forcing encryption for reset peers #1998

CMakeLists.txt

@@ -0,0 +1,28 @@
+cmake_minimum_required(VERSION 3.18)
+project (deltachat)
+
+find_program(CARGO cargo)
+
+add_custom_command(
+	OUTPUT
+	"target/release/libdeltachat.a"
+	"target/release/libdeltachat.so"
+	"target/release/pkgconfig/deltachat.pc"
+	COMMAND PREFIX=${CMAKE_INSTALL_PREFIX} ${CARGO} build --package deltachat_ffi --release
+	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
+add_custom_target(
+	lib_deltachat
+	ALL
+	DEPENDS
+	"target/release/libdeltachat.a"
+	"target/release/libdeltachat.so"
+	"target/release/pkgconfig/deltachat.pc"
+)
+
+include(GNUInstallDirs)
+install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,10 +1,13 @@
 [package]
 name = "deltachat"
-version = "1.47.0"
+version = "1.51.0-alpha.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
+[profile.dev]
+debug = 0
+
 [profile.release]
 lto = true
 
@@ -14,6 +17,7 @@ deltachat_derive = { path = "./deltachat_derive" }
 libc = "0.2.51"
 pgp = { version = "0.7.0", default-features = false } 
 hex = "0.4.0"
+sha-1 = "0.9.3"
 sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
@@ -25,7 +29,7 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.4.0"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.4", features = ["unstable"] }
+async-std = { version = "~1.8.0", features = ["unstable"] }
 base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -61,6 +65,7 @@ url = "2.1.1"
 async-std-resolver = "0.19.5"
 async-tar = "0.3.0"
 uuid = { version = "0.8", features = ["serde", "v4"] }
+rust-hsluv = "0.1.4"
 
 pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
@@ -78,6 +83,7 @@ proptest = "0.10"
 async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
 futures-lite = "1.7.0"
 criterion = "0.3"
+ansi_term = "0.12.0"
 
 [workspace]
 members = [

README.md

@@ -123,7 +123,7 @@ Language bindings are available for:
 - [C](https://c.delta.chat)
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
-- [Go](https://github.com/hugot/go-deltachat/)
+- [Go](https://github.com/deltachat/go-deltachat/)
 - [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 

ci_scripts/ci_upload.sh

@@ -10,9 +10,13 @@ set -xe
 PYDOCDIR=${1:?directory with python docs}
 WHEELHOUSEDIR=${2:?directory with pre-built wheels}
 DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+SSHTARGET=ci@b1.delta.chat
 
+    
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
+
 
 # python docs to py.delta.chat
 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
@@ -35,23 +39,36 @@ echo upload wheels
 echo -----------------------
 
 # Bundle external shared libraries into the wheels
-pushd $WHEELHOUSEDIR
 
-pip3 install -U pip setuptools
-pip3 install devpi-client
-devpi use https://m.devpi.net
-devpi login dc --password $DEVPI_LOGIN
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ci_scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  $WHEELHOUSEDIR \
+  $SSHTARGET:$BUILDDIR 
 
-N_BRANCH=${BRANCH//[\/]}
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
 
-devpi use dc/$N_BRANCH || {
-    devpi index -c $N_BRANCH 
-    devpi use dc/$N_BRANCH
-}
-devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*
+    # we rely on the "venv" virtualenv on the remote account to exist 
+    source venv/bin/activate
+    cd $BUILDDIR
 
-popd
+    devpi use https://m.devpi.net
+    devpi login dc --password $DEVPI_LOGIN
 
-# remove devpi non-master dc indices if thy are too old
-python ci_scripts/cleanup_devpi_indices.py
+    N_BRANCH=${BRANCH//[\/]}
+
+    devpi use dc/\$N_BRANCH || {
+        devpi index -c \$N_BRANCH 
+        devpi use dc/\$N_BRANCH
+    }
+    devpi index \$N_BRANCH bases=/root/pypi
+    devpi upload wheelhouse/deltachat*
+
+    # remove devpi non-master dc indices if thy are too old
+    # this script was copied above 
+    python cleanup_devpi_indices.py
+_HERE

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.50.0-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share
+rm -rf /root/.rustup/toolchains/1.50.0-x86_64-unknown-linux-gnu/share

ci_scripts/remote_python_packaging.sh

@@ -1,7 +1,7 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}

ci_scripts/remote_tests_python.sh

@@ -1,7 +1,7 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}

ci_scripts/remote_tests_rust.sh

@@ -1,7 +1,7 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.47.0"
+version = "1.51.0-alpha.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/DoxygenLayout.xml

@@ -0,0 +1,38 @@
+<doxygenlayout version="1.0">
+  <!-- Generated by doxygen 1.8.20 -->
+  <!-- Navigation index tabs for HTML output -->
+  <navindex>
+    <tab type="mainpage" visible="yes" title=""/>
+    <tab type="classes" visible="yes" title="">
+      <tab type="classlist" visible="no" title="" intro=""/>
+      <tab type="classindex" visible="no" title=""/>
+      <tab type="hierarchy" visible="no" title="" intro=""/>
+      <tab type="classmembers" visible="no" title="" intro=""/>
+    </tab>
+    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
+    <tab type="pages" visible="yes" title="" intro=""/>
+    <tab type="namespaces" visible="yes" title="">
+      <tab type="namespacelist" visible="yes" title="" intro=""/>
+      <tab type="namespacemembers" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="interfaces" visible="yes" title="">
+      <tab type="interfacelist" visible="yes" title="" intro=""/>
+      <tab type="interfaceindex" visible="$ALPHABETICAL_INDEX" title=""/>
+      <tab type="interfacehierarchy" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="structs" visible="yes" title="">
+      <tab type="structlist" visible="yes" title="" intro=""/>
+      <tab type="structindex" visible="$ALPHABETICAL_INDEX" title=""/>
+    </tab>
+    <tab type="exceptions" visible="yes" title="">
+      <tab type="exceptionlist" visible="yes" title="" intro=""/>
+      <tab type="exceptionindex" visible="$ALPHABETICAL_INDEX" title=""/>
+      <tab type="exceptionhierarchy" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="files" visible="yes" title="">
+      <tab type="filelist" visible="yes" title="" intro=""/>
+      <tab type="globals" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="examples" visible="yes" title="" intro=""/>
+  </navindex>
+</doxygenlayout>

deltachat-ffi/src/lib.rs

@@ -24,7 +24,6 @@ use std::time::{Duration, SystemTime};
 use async_std::task::{block_on, spawn};
 use num_traits::{FromPrimitive, ToPrimitive};
 
-use deltachat::accounts::Accounts;
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, Origin};
@@ -32,8 +31,9 @@ use deltachat::context::Context;
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
-use deltachat::stock::StockMessage;
+use deltachat::stock_str::StockMessage;
 use deltachat::*;
+use deltachat::{accounts::Accounts, log::LogExt};
 
 mod dc_array;
 
@@ -296,16 +296,6 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     block_on(ctx.start_io())
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_is_io_running(context: *mut dc_context_t) -> libc::c_int {
-    if context.is_null() {
-        return 0;
-    }
-    let ctx = &*context;
-
-    block_on(ctx.is_io_running()) as libc::c_int
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_id(context: *mut dc_context_t) -> libc::c_int {
     if context.is_null() {
@@ -935,6 +925,8 @@ pub unsafe extern "C" fn dc_get_fresh_msgs(
         let arr = dc_array_t::from(
             ctx.get_fresh_msgs()
                 .await
+                .log_err(ctx, "Failed to get fresh messages")
+                .unwrap_or_default()
                 .iter()
                 .map(|msg_id| msg_id.to_u32())
                 .collect::<Vec<u32>>(),
@@ -1146,10 +1138,15 @@ pub unsafe extern "C" fn dc_search_msgs(
         return ptr::null_mut();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
     block_on(async move {
         let arr = dc_array_t::from(
-            ctx.search_msgs(ChatId::new(chat_id), to_string_lossy(query))
+            ctx.search_msgs(chat_id, to_string_lossy(query))
                 .await
                 .iter()
                 .map(|msg_id| msg_id.to_u32())
@@ -1332,6 +1329,29 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_chat_encrinfo(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_chat_encrinfo()");
+        return "".strdup();
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        ChatId::new(chat_id)
+            .get_encryption_info(&ctx)
+            .await
+            .map(|s| s.strdup())
+            .unwrap_or_else(|e| {
+                error!(&ctx, "{}", e);
+                ptr::null_mut()
+            })
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_chat_ephemeral_timer(
     context: *mut dc_context_t,
@@ -1389,6 +1409,20 @@ pub unsafe extern "C" fn dc_get_msg_info(
     block_on(message::get_msg_info(&ctx, MsgId::new(msg_id))).strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_msg_html(
+    context: *mut dc_context_t,
+    msg_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_msg_html()");
+        return ptr::null_mut();
+    }
+    let ctx = &*context;
+
+    block_on(MsgId::new(msg_id).get_html(&ctx)).strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_mime_headers(
     context: *mut dc_context_t,
@@ -1535,6 +1569,9 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
         to_string_lossy(addr),
         Origin::IncomingReplyTo,
     ))
+    .ok()
+    .flatten()
+    .unwrap_or_default()
 }
 
 #[no_mangle]
@@ -1551,10 +1588,9 @@ pub unsafe extern "C" fn dc_create_contact(
     let name = to_string_lossy(name);
 
     block_on(async move {
-        match Contact::create(&ctx, name, to_string_lossy(addr)).await {
-            Ok(id) => id,
-            Err(_) => 0,
-        }
+        Contact::create(&ctx, name, to_string_lossy(addr))
+            .await
+            .unwrap_or(0)
     })
 }
 
@@ -1606,7 +1642,13 @@ pub unsafe extern "C" fn dc_get_blocked_cnt(context: *mut dc_context_t) -> libc:
     }
     let ctx = &*context;
 
-    block_on(Contact::get_blocked_cnt(&ctx)) as libc::c_int
+    block_on(async move {
+        Contact::get_all_blocked(&ctx)
+            .await
+            .log_err(&ctx, "Can't get blocked count")
+            .unwrap_or_default()
+            .len() as libc::c_int
+    })
 }
 
 #[no_mangle]
@@ -1621,7 +1663,10 @@ pub unsafe extern "C" fn dc_get_blocked_contacts(
 
     block_on(async move {
         Box::into_raw(Box::new(dc_array_t::from(
-            Contact::get_all_blocked(&ctx).await,
+            Contact::get_all_blocked(&ctx)
+                .await
+                .log_err(&ctx, "Can't get blocked contacts")
+                .unwrap_or_default(),
         )))
     })
 }
@@ -1727,13 +1772,15 @@ pub unsafe extern "C" fn dc_imex(
 
     let ctx = &*context;
 
-    let param1 = to_opt_string_lossy(param1);
-
-    spawn(async move {
-        imex::imex(&ctx, what, param1)
-            .await
-            .log_err(ctx, "IMEX failed")
-    });
+    if let Some(param1) = to_opt_string_lossy(param1) {
+        spawn(async move {
+            imex::imex(&ctx, what, &param1)
+                .await
+                .log_err(ctx, "IMEX failed")
+        });
+    } else {
+        eprintln!("dc_imex called without a valid directory");
+    }
 }
 
 #[no_mangle]
@@ -1844,9 +1891,14 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
         return "".strdup();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
     block_on(async move {
-        securejoin::dc_get_securejoin_qr(&ctx, ChatId::new(chat_id))
+        securejoin::dc_get_securejoin_qr(&ctx, chat_id)
             .await
             .unwrap_or_else(|| "".to_string())
             .strdup()
@@ -1902,11 +1954,13 @@ pub unsafe extern "C" fn dc_is_sending_locations_to_chat(
         return 0;
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
-    block_on(location::is_sending_locations_to_chat(
-        &ctx,
-        ChatId::new(chat_id),
-    )) as libc::c_int
+    block_on(location::is_sending_locations_to_chat(&ctx, chat_id)) as libc::c_int
 }
 
 #[no_mangle]
@@ -1938,11 +1992,21 @@ pub unsafe extern "C" fn dc_get_locations(
         return ptr::null_mut();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
+    let contact_id = if contact_id == 0 {
+        None
+    } else {
+        Some(contact_id)
+    };
 
     block_on(async move {
         let res = location::get_range(
             &ctx,
-            ChatId::new(chat_id),
+            chat_id,
             contact_id,
             timestamp_begin as i64,
             timestamp_end as i64,
@@ -2560,6 +2624,16 @@ pub unsafe extern "C" fn dc_msg_get_chat_id(msg: *mut dc_msg_t) -> u32 {
     ffi_msg.message.get_chat_id().to_u32()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_real_chat_id(msg: *mut dc_msg_t) -> u32 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_real_chat_id()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_real_chat_id().to_u32()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_viewtype(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2624,6 +2698,16 @@ pub unsafe extern "C" fn dc_msg_get_text(msg: *mut dc_msg_t) -> *mut libc::c_cha
     ffi_msg.message.get_text().unwrap_or_default().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_subject(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_subject()");
+        return "".strdup();
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_subject().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -2722,7 +2806,7 @@ pub unsafe extern "C" fn dc_msg_get_ephemeral_timer(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_ephemeral_timer()
+    ffi_msg.message.get_ephemeral_timer().to_u32()
 }
 
 #[no_mangle]
@@ -2779,6 +2863,17 @@ pub unsafe extern "C" fn dc_msg_get_summarytext(
     .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_override_sender_name(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_override_sender_name()");
+        return "".strdup();
+    }
+    let ffi_msg = &mut *msg;
+
+    ffi_msg.message.get_override_sender_name().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_has_deviating_timestamp(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2859,6 +2954,16 @@ pub unsafe extern "C" fn dc_msg_is_setupmessage(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.is_setupmessage().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_has_html(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_has_html()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.has_html().into()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -2874,6 +2979,32 @@ pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut li
         .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_decide_on_contact_request(
+    context: *mut dc_context_t,
+    msg_id: u32,
+    decision: libc::c_int,
+) -> u32 {
+    if context.is_null() || msg_id <= constants::DC_MSG_ID_LAST_SPECIAL as u32 {
+        eprintln!("ignoring careless call to dc_decide_on_contact_request()");
+    }
+    let ctx = &*context;
+
+    match from_prim(decision) {
+        None => {
+            warn!(ctx, "{} is not a valid decision, ignoring", decision);
+            0
+        }
+        Some(d) => block_on(message::decide_on_contact_request(
+            ctx,
+            MsgId::new(msg_id),
+            d,
+        ))
+        .unwrap_or_default()
+        .to_u32(),
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_videochat_type(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2908,6 +3039,31 @@ pub unsafe extern "C" fn dc_msg_set_text(msg: *mut dc_msg_t, text: *const libc::
     ffi_msg.message.set_text(to_opt_string_lossy(text))
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_html(msg: *mut dc_msg_t, html: *const libc::c_char) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_html()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg.message.set_html(to_opt_string_lossy(html))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_override_sender_name(
+    msg: *mut dc_msg_t,
+    name: *const libc::c_char,
+) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_override_sender_name()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg
+        .message
+        .set_override_sender_name(to_opt_string_lossy(name))
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_set_file(
     msg: *mut dc_msg_t,
@@ -3006,6 +3162,11 @@ pub unsafe extern "C" fn dc_msg_set_quote(msg: *mut dc_msg_t, quote: *const dc_m
     let ffi_msg = &mut *msg;
     let ffi_quote = &*quote;
 
+    if ffi_msg.context != ffi_quote.context {
+        eprintln!("ignoring attempt to quote message from a different context");
+        return;
+    }
+
     block_on(async move {
         ffi_msg
             .message
@@ -3107,6 +3268,16 @@ pub unsafe extern "C" fn dc_contact_get_name(contact: *mut dc_contact_t) -> *mut
     ffi_contact.contact.get_name().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_auth_name(contact: *mut dc_contact_t) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_auth_name()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.get_authname().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_display_name(
     contact: *mut dc_contact_t,
@@ -3131,18 +3302,6 @@ pub unsafe extern "C" fn dc_contact_get_name_n_addr(
     ffi_contact.contact.get_name_n_addr().strdup()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_contact_get_first_name(
-    contact: *mut dc_contact_t,
-) -> *mut libc::c_char {
-    if contact.is_null() {
-        eprintln!("ignoring careless call to dc_contact_get_first_name()");
-        return "".strdup();
-    }
-    let ffi_contact = &*contact;
-    ffi_contact.contact.get_first_name().strdup()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_profile_image(
     contact: *mut dc_contact_t,
@@ -3174,6 +3333,16 @@ pub unsafe extern "C" fn dc_contact_get_color(contact: *mut dc_contact_t) -> u32
     ffi_contact.contact.get_color()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_status(contact: *mut dc_contact_t) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_status()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.get_status().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_is_blocked(contact: *mut dc_contact_t) -> libc::c_int {
     if contact.is_null() {
@@ -3282,15 +3451,11 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
 }
 
 trait ResultExt<T, E> {
+    /// Like `log_err()`, but:
+    /// - returns the default value instead of an Err value.
+    /// - emits an error instead of a warning for an [Err] result. This means
+    /// that the error will be shown to the user in a small pop-up.
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T;
-
-    /// Log a warning to a [ContextWrapper] for an [Err] result.
-    ///
-    /// Does nothing for an [Ok].
-    ///
-    /// You can do this as soon as the wrapper exists, it does not
-    /// have to be open (which is required for the `warn!()` macro).
-    fn log_err(self, wrapper: &Context, message: &str) -> Result<T, E>;
 }
 
 impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
@@ -3303,14 +3468,6 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
             }
         }
     }
-
-    fn log_err(self, ctx: &Context, message: &str) -> Result<T, E> {
-        self.map_err(|err| {
-            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-            warn!(ctx, "{}: {:#}", message, err);
-            err
-        })
-    }
 }
 
 trait ResultNullableExt<T> {
@@ -3351,7 +3508,7 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
         return ptr::null();
     }
     let addr = to_string_lossy(addr);
-    match provider::get_provider_info(addr.as_str()) {
+    match block_on(provider::get_provider_info(addr.as_str())) {
         Some(provider) => provider,
         None => ptr::null_mut(),
     }

draft/group-sync.rst

@@ -5,69 +5,95 @@ Problem: missing eventual group consistency
 If group members are concurrently adding new members,
 the new members will miss each other's additions, example:
 
-- Alice and Bob are in a two-member group
+1. Alice and Bob are in a two-member group
 
-- Alice adds Carol, concurrently Bob adds Doris
+2. Then Alice adds Carol, while concurrently Bob adds Doris
 
-- Carol will see a three-member group (Alice, Bob, Carol),
-  Doris will see a different three-member group (Alice, Bob, Doris),
-  and only Alice and Bob will have all four members.
+Right now, the group has inconsistent memberships:
 
-Note that for verified groups any mitigation mechanism likely
-needs to make all clients to know who originally added a member.
+- Alice and Carol see a (Alice, Carol, Bob) group
+
+- Bob and Doris see a (Bob, Doris, Alice)
+
+This then leads to "sender is unknown" messages in the chat,
+for example when Alice receives a message from Doris,
+or when Bob receives a message from Carol.
+
+There are also other sources for group membership inconsistency:
+
+- leaving/deleting/adding in larger groups, while being offline,
+  increases chances for inconsistent group membership
+
+- dropped group-membership messages
+
+- group-membership messages landing in "Spam"
 
 
-solution: memorize+attach (possible encrypted) chat-meta mime messages
-----------------------------------------------------------------------
+Note that all these problems (can) also happen with verified groups,
+then raising "false alarms" which could lure people to ignore such issues.
 
-For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
+IOW, it's clear we need to do something about it to improve overall
+reliability in group-settings.
 
 
-- All Chat-Group-Member-Added/Removed messages are recorded in their
-  full raw (signed and encrypted) mime-format in the DB
 
-- If an incoming member-add/member-delete messages has a member list
-  which is, apart from the added/removed member, not consistent
-  with our own view, broadcast a "Chat-Group-Member-Correction" message to
-  all members, attaching the original added/removed mime-message for all mismatching
-  contacts.  If we have no relevant add/del information, don't send a
-  correction message out.
+Solution: replay group modification messages on inconsistencies
+------------------------------------------------------------------
 
-- Upong receiving added/removed attachments we don't do the
-  check_consistency+correction message dance.
-  This avoids recursion problems and hard-to-reason-about chatter.
+For brevity let's abbreviate "group membership modification" as **GMM**.
 
-Notes:
+Delta chat has explicit GMM messages, typically encrypted to the group members
+as seen by the device that sends the GMM. The `Spec <https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members>`_ details the Mime headers and format.
 
-- mechanism works for both encrypted and unencrypted add/del messages
+If we detect membership inconsistencies we can resend relevant GMM messages
+to the respective chat.  The receiving devices can process those GMM messages
+as if it would be an incoming message. If for example they have already seen
+the Message-ID of the GMM message, they will ignore the message. It's
+probably useful to record GMM message in their original MIME-format and
+not invent a new recording format. Few notes on three aspects:
 
-- we already have a "mime_headers" column in the DB for each incoming message.
-  We could extend it to also include the payload and store mime unconditionally
-  for member-added/removed messages.
+- **group-membership-tracking**: All valid GMM messages are persisted in
+  their full raw (signed/encrypted?) MIME-format in the DB. Note that GMM messages
+  already are in the msgs table, and there is a mime_header column which we could
+  extend to contain the raw Mime GMM message.
 
-- multiple member-added/removed messages can be attached in a single
-  correction message
+- **consistency_checking**: If an incoming GMM has a member list which is
+  not consistent with our own view, broadcast a "Group-Member-Correction"
+  message to all members containing a multipart list of GMMs.
 
-- it is minimal on the number of overall messages to reach group consistency
-  (best-case: no extra messages, the ABCD case above: max two extra messages)
+- **correcting_memberships**: Upon receiving a Group-Member-Correction
+  message we pass the contained GMMs to the "incoming mail pipeline"
+  (without **consistency_checking** them, to avoid recursion issues)
 
-- somewhat backward compatible: older clients will probably ignore
-  messages which are signed by someone who is not the outer From-address.
 
-- the correction-protocol also helps with dropped messages.  If a member
-  did not see a member-added/removed message, the next member add/removed
-  message in the group will likely heal group consistency for this member.
+Alice/Carol and Bob/Doris getting on the same page
+++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Recall that Alice/Carol and Bob/Doris had a differening view of
+group membership. With the proposed solution, when Bob receives
+Alice's "Carol added" message, he will notice that Alice (and thus
+also carol) did not know about Doris.  Bob's device sends a
+"Chat-Group-Member-Correction" message containing his own GMM
+when adding Doris. Therefore, the group's membership is healed
+for everyone in a single broadcast message.
+
+Alice might also send a Group-member-Correction message,
+so there is a second chance that the group gets to know all GMMs.
+
+Note, for example, that if for some reason Bobs and Carols provider
+drop GMM messages between them (spam) that Alice and Doris can heal
+it by resending GMM messages whenever they detect them to be out of sync.
 
-- we can quite easily extend the mechanism to also provide the group-avatar or
-  other meta-information.
 
 Discussions of variants
 ++++++++++++++++++++++++
 
-- instead of acting on MemberAdded/Removed message we could send
-  corrections for any received message that addresses inconsistent group members but
-  a) this would delay group-membership healing
+- instead of acting on GMM messages we could send corrections
+  for any received message that addresses inconsistent group members but
+  a) this could delay group-membership healing
   b) could lead to a lot of members sending corrections
+  c) means we might rely on "To-Addresses" which we also like to strike
+     at least for protected chats.
 
 - instead of broadcasting correction messages we could only send it to
   the sender of the inconsistent member-added/removed message.
@@ -83,44 +109,3 @@ Discussions of variants
   while both being in an offline or bad-connection situation).
 
 
-solution2: repeat member-added/removed messages
----------------------------------------------------
-
-Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
-but keep sending out the old headers until the new protocol is sufficiently deployed.
-
-The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
-which controls repetition of the signed "add/del e-mail address" payload.
-
-Example::
-
-    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
-        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
-        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
-        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
-        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
-        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
-        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
-        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
-        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
-        =P6GG
-
-TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
-Receivers will apply the add/del change to the group-membership,
-decrease the TTL by 1, and if TTL>0 re-sent the header.
-
-The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
-This allows to propagate, in a cryptographically secured way,
-who added a member. This is particularly important for allowing
-to show in verified groups who added a member (planned).
-
-Disadvantage to solution 1:
-
-- requires to specify encoding and precise rules for what/how is signed.
-
-- causes O(N^2) extra messages
-
-- Not easily extendable for other things (without introducing a new
-  header / encoding)
-
-

examples/repl/cmdline.rs

@@ -2,25 +2,29 @@ extern crate dirs;
 
 use std::str::FromStr;
 
-use anyhow::{bail, ensure};
+use anyhow::{bail, ensure, Error};
 use async_std::path::Path;
-use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility, ProtectionStatus};
+use deltachat::chat::{
+    self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
+};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
-use deltachat::error::Error;
 use deltachat::imex::*;
 use deltachat::location;
+use deltachat::log::LogExt;
 use deltachat::lot::LotState;
-use deltachat::message::{self, Message, MessageState, MsgId};
+use deltachat::message::{self, ContactRequestDecision, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
 use deltachat::EventType;
 use deltachat::{config, provider};
+use std::fs;
+use std::time::{Duration, SystemTime};
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
@@ -171,8 +175,11 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let contact = Contact::get_by_id(context, msg.get_from_id())
         .await
         .expect("invalid contact");
-
-    let contact_name = contact.get_name();
+    let contact_name = if let Some(name) = msg.get_override_sender_name() {
+        format!("~{}", name)
+    } else {
+        contact.get_display_name().to_string()
+    };
     let contact_id = contact.get_id();
 
     let statestr = match msg.get_state() {
@@ -185,7 +192,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -193,6 +200,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
+        if msg.has_html() { "[HAS-HTML]️" } else { "" },
         if msg.get_from_id() == 1 as libc::c_uint {
             ""
         } else if msg.get_state() == MessageState::InSeen {
@@ -251,15 +259,11 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error>
 }
 
 async fn log_contactlist(context: &Context, contacts: &[u32]) {
-    let mut contacts = contacts.to_vec();
-    if !contacts.contains(&1) {
-        contacts.push(1);
-    }
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
-            let name = contact.get_name();
+        if let Ok(contact) = Contact::get_by_id(context, *contact_id).await {
+            let name = contact.get_display_name();
             let addr = contact.get_addr();
             let verified_state = contact.is_verified(context).await;
             let verified_str = if VerifiedStatus::Unverified != verified_state {
@@ -288,14 +292,14 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
             let peerstate = Peerstate::from_addr(context, &addr)
                 .await
                 .expect("peerstate error");
-            if peerstate.is_some() && contact_id != 1 as libc::c_uint {
+            if peerstate.is_some() && *contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
                     peerstate.as_ref().unwrap().prefer_encrypt
                 );
             }
 
-            println!("Contact#{}: {}{}", contact_id, line, line2);
+            println!("Contact#{}: {}{}", *contact_id, line, line2);
         }
     }
 }
@@ -355,7 +359,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listarchived\n\
                  chat [<chat-id>|0]\n\
                  createchat <contact-id>\n\
-                 createchatbymsg <msg-id>\n\
                  creategroup <name>\n\
                  createprotected <name>\n\
                  addmember <contact-id>\n\
@@ -368,9 +371,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  dellocations\n\
                  getlocations [<contact-id>]\n\
                  send <text>\n\
-                 send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
+                 sendhtml <file for html-part> [<text for plain-part>]\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -379,12 +382,19 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  unarchive <chat-id>\n\
                  pin <chat-id>\n\
                  unpin <chat-id>\n\
+                 mute <chat-id> [<seconds>]\n\
+                 unmute <chat-id>\n\
                  protect <chat-id>\n\
                  unprotect <chat-id>\n\
                  delchat <chat-id>\n\
+                 ===========================Contact requests==\n\
+                 decidestartchat <msg-id>\n\
+                 decideblock <msg-id>\n\
+                 decidenotnow <msg-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
+                 html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
@@ -396,10 +406,14 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  contactinfo <contact-id>\n\
                  delcontact <contact-id>\n\
                  cleanupcontacts\n\
+                 block <contact-id>\n\
+                 unblock <contact-id>\n\
+                 listblocked\n\
                  ======================================Misc.==\n\
                  getqr [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
+                 setqr <qr-content>\n\
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
@@ -443,20 +457,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportBackup, Some(&dir)).await?;
+            imex(&context, ImexMode::ExportBackup, &dir).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
+            imex(&context, ImexMode::ImportBackup, arg1).await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?;
+            imex(&context, ImexMode::ExportSelfKeys, &dir).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -500,7 +514,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             context.maybe_network().await;
         }
         "housekeeping" => {
-            sql::housekeeping(&context).await;
+            sql::housekeeping(&context).await.ok_or_log(&context);
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
@@ -525,11 +539,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}{}",
+                        "{}#{}: {} [{} fresh] {}{}{}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
                         chat.get_id().get_fresh_msg_cnt(&context).await,
+                        if chat.is_muted() { "🔇" } else { "" },
                         match chat.visibility {
                             ChatVisibility::Normal => "",
                             ChatVisibility::Archived => "📦",
@@ -570,7 +585,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, ChatId::new(0)).await {
+            if location::is_sending_locations_to_chat(&context, None).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
@@ -606,15 +621,18 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
                 let contact = Contact::get_by_id(&context, members[0]).await?;
                 contact.get_addr().to_string()
+            } else if sel_chat.get_type() == Chattype::Mailinglist && !members.is_empty() {
+                "mailinglist".to_string()
             } else {
                 format!("{} member(s)", members.len())
             };
             println!(
-                "{}#{}: {} [{}]{}{} {}",
+                "{}#{}: {} [{}]{}{}{} {}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
                 sel_chat.get_name(),
                 subtitle,
+                if sel_chat.is_muted() { "🔇" } else { "" },
                 if sel_chat.is_sending_locations() {
                     "📍"
                 } else {
@@ -651,13 +669,34 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!("Single#{} created successfully.", chat_id,);
         }
-        "createchatbymsg" => {
+        "decidestartchat" | "createchatbymsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
             let msg_id = MsgId::new(arg1.parse()?);
-            let chat_id = chat::create_by_msg_id(&context, msg_id).await?;
-            let chat = Chat::load_from_db(&context, chat_id).await?;
-
-            println!("{}#{} created successfully.", chat_prefix(&chat), chat_id,);
+            match message::decide_on_contact_request(
+                &context,
+                msg_id,
+                ContactRequestDecision::StartChat,
+            )
+            .await
+            {
+                Some(chat_id) => {
+                    let chat = Chat::load_from_db(&context, chat_id).await?;
+                    println!("{}#{} created successfully.", chat_prefix(&chat), chat_id);
+                }
+                None => println!("Cannot crate chat."),
+            }
+        }
+        "decidenotnow" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::NotNow)
+                .await;
+        }
+        "decideblock" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::Block)
+                .await;
         }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
@@ -732,7 +771,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 contacts.len(),
                 location::is_sending_locations_to_chat(
                     &context,
-                    sel_chat.as_ref().unwrap().get_id()
+                    Some(sel_chat.as_ref().unwrap().get_id())
                 )
                 .await,
             );
@@ -740,10 +779,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "getlocations" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
-            let contact_id = arg1.parse().unwrap_or_default();
+            let contact_id: Option<u32> = arg1.parse().ok();
             let locations = location::get_range(
                 &context,
-                sel_chat.as_ref().unwrap().get_id(),
+                Some(sel_chat.as_ref().unwrap().get_id()),
                 contact_id,
                 0,
                 0,
@@ -831,6 +870,22 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "sendhtml" => {
+            ensure!(sel_chat.is_some(), "No chat selected.");
+            ensure!(!arg1.is_empty(), "No html-file given.");
+            let path: &Path = arg1.as_ref();
+            let html = &*fs::read(&path)?;
+            let html = String::from_utf8_lossy(html);
+
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_html(Some(html.to_string()));
+            msg.set_text(Some(if arg2.is_empty() {
+                path.file_name().unwrap().to_string_lossy().to_string()
+            } else {
+                arg2.to_string()
+            }));
+            chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
+        }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -839,9 +894,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
             let chat = if let Some(ref sel_chat) = sel_chat {
-                sel_chat.get_id()
+                Some(sel_chat.get_id())
             } else {
-                ChatId::new(0)
+                None
             };
 
             let msglist = context.search_msgs(chat, arg1).await;
@@ -917,6 +972,24 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 )
                 .await?;
         }
+        "mute" | "unmute" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            let duration = match arg0 {
+                "mute" => {
+                    if arg2.is_empty() {
+                        MuteDuration::Forever
+                    } else {
+                        SystemTime::now()
+                            .checked_add(Duration::from_secs(arg2.parse()?))
+                            .map_or(MuteDuration::Forever, MuteDuration::Until)
+                    }
+                }
+                "unmute" => MuteDuration::NotMuted,
+                _ => unreachable!("arg0={:?}", arg0),
+            };
+            chat::set_muted(&context, chat_id, duration).await?;
+        }
         "protect" | "unprotect" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = ChatId::new(arg1.parse()?);
@@ -942,8 +1015,18 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let res = message::get_msg_info(&context, id).await;
             println!("{}", res);
         }
+        "html" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            let file = dirs::home_dir()
+                .unwrap_or_default()
+                .join(format!("msg-{}.html", id.to_u32()));
+            let html = id.get_html(&context).await.unwrap_or_default();
+            fs::write(&file, html)?;
+            println!("HTML written to: {:#?}", file);
+        }
         "listfresh" => {
-            let msglist = context.get_fresh_msgs().await;
+            let msglist = context.get_fresh_msgs().await?;
 
             log_msglist(&context, &msglist).await?;
             print!("{} fresh messages.", msglist.len());
@@ -975,9 +1058,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let contacts = Contact::get_all(
                 &context,
                 if arg0 == "listverified" {
-                    0x1 | 0x2
+                    DC_GCL_VERIFIED_ONLY | DC_GCL_ADD_SELF
                 } else {
-                    0x2
+                    DC_GCL_ADD_SELF
                 },
                 Some(arg1),
             )
@@ -1035,6 +1118,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             Contact::delete(&context, arg1.parse()?).await?;
         }
+        "block" => {
+            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
+            let contact_id = arg1.parse()?;
+            Contact::block(&context, contact_id).await;
+        }
+        "unblock" => {
+            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
+            let contact_id = arg1.parse()?;
+            Contact::unblock(&context, contact_id).await;
+        }
+        "listblocked" => {
+            let contacts = Contact::get_all_blocked(&context).await?;
+            log_contactlist(&context, &contacts).await;
+            println!("{} blocked contacts.", contacts.len());
+        }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
             let res = check_qr(&context, arg1).await;
@@ -1055,7 +1153,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            match provider::get_provider_info(arg1) {
+            match provider::get_provider_info(arg1).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);

examples/repl/main.rs

@@ -168,12 +168,14 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 27] = [
+const CHAT_COMMANDS: [&str; 34] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
-    "createchatbymsg",
+    "decidestartchat",
+    "decideblock",
+    "decidenotnow",
     "creategroup",
     "createverified",
     "addmember",
@@ -188,6 +190,7 @@ const CHAT_COMMANDS: [&str; 27] = [
     "send",
     "sendimage",
     "sendfile",
+    "sendhtml",
     "videochat",
     "draft",
     "listmedia",
@@ -195,25 +198,30 @@ const CHAT_COMMANDS: [&str; 27] = [
     "unarchive",
     "pin",
     "unpin",
+    "mute",
+    "unmute",
+    "protect",
+    "unprotect",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&str; 6] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
-    "star",
-    "unstar",
     "delmsg",
 ];
-const CONTACT_COMMANDS: [&str; 6] = [
+const CONTACT_COMMANDS: [&str; 9] = [
     "listcontacts",
     "listverified",
     "addcontact",
     "contactinfo",
     "delcontact",
     "cleanupcontacts",
+    "block",
+    "unblock",
+    "listblocked",
 ];
 const MISC_COMMANDS: [&str; 10] = [
     "getqr",
@@ -300,7 +308,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::sync::channel(100);
+    let (reader_s, reader_r) = async_std::channel::bounded(100);
     let input_loop = async_std::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
@@ -323,7 +331,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line));
+                    async_std::task::block_on(reader_s.send(line)).unwrap();
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
@@ -400,9 +408,8 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            if let Some(mut qr) =
-                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
-            {
+            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")

python/README.rst

@@ -11,10 +11,10 @@ Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
-yourself <sourceinstall>`_.
+without any "build-from-source" steps.
+Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -19,8 +19,7 @@ if __name__ == "__main__":
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
 
     if target == 'release':
-        extra = " -C lto=on -C embed-bitcode=yes"
-        os.environ["RUSTFLAGS"] = os.environ.get("RUSTFLAGS", "") + extra
+        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
 
     print("running:", " ".join(cmd))

python/src/deltachat/_build.py

@@ -145,9 +145,12 @@ def extract_defines(flags):
                 | DC_STR
                 | DC_CONTACT_ID
                 | DC_GCL
+                | DC_GCM
+                | DC_SOCKET
                 | DC_CHAT
                 | DC_PROVIDER
                 | DC_KEY_GEN
+                | DC_IMEX
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains

python/src/deltachat/account.py

@@ -214,6 +214,19 @@ class Account(object):
         :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
+        (name, addr) = self.get_contact_addr_and_name(obj, name)
+        name = as_dc_charpointer(name)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        return Contact(self, contact_id)
+
+    def get_contact(self, obj):
+        if isinstance(obj, Contact):
+            return obj
+        (_, addr) = self.get_contact_addr_and_name(obj)
+        return self.get_contact_by_addr(addr)
+
+    def get_contact_addr_and_name(self, obj, name=None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -229,13 +242,7 @@ class Account(object):
 
         if name is None and displayname:
             name = displayname
-        return self._create_contact(addr, name)
-
-    def _create_contact(self, addr, name):
-        addr = as_dc_charpointer(addr)
-        name = as_dc_charpointer(name)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        return Contact(self, contact_id)
+        return (name, addr)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -405,23 +412,23 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        return self._export(path, imex_cmd=1)
+        return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
 
-        Note that the account does not have to be started.
+        Note that the account has to be stopped; call stop_io() if necessary.
         """
-        export_files = self._export(path, 11)
+        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            self.imex(path, imex_cmd)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -431,7 +438,7 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        self._import(path, imex_cmd=2)
+        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
 
     def import_all(self, path):
         """import delta chat state from the specified backup `path` (a file).
@@ -439,21 +446,22 @@ class Account(object):
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=12)
+        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
 
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            self.imex(path, imex_cmd)
             imex_tracker.wait_finish()
 
+    def imex(self, path, imex_cmd):
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
-        if not self.is_started():
-            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -572,12 +580,34 @@ class Account(object):
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
-    def configure(self):
+    def maybe_network(self):
+        """This function should be called when there is a hint
+        that the network is available again,
+        e.g. as a response to system event reporting network availability.
+        The library will try to send pending messages out immediately.
+
+        Moreover, to have a reliable state
+        when the app comes to foreground with network available,
+        it may be reasonable to call the function also at that moment.
+
+        It is okay to call the function unconditionally when there is
+        network available, however, calling the function
+        _without_ having network may interfere with the backoff algorithm
+        and will led to let the jobs fail faster, with fewer retries
+        and may avoid messages being sent out.
+
+        Finally, if the context was created by the dc_accounts_t account manager
+        (currently not implemented in the Python bindings),
+        use dc_accounts_maybe_network() instead of this function
+        """
+        lib.dc_maybe_network(self._dc_context)
+
+    def configure(self, reconfigure=False):
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
         """
-        assert not self.is_configured()
+        assert self.is_configured() == reconfigure
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
         configtracker = ConfigureTracker(self)
@@ -585,9 +615,6 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def is_started(self):
-        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
-
     def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
@@ -597,11 +624,8 @@ class Account(object):
         self.log("stop_ongoing")
         self.stop_ongoing()
 
-        if bool(lib.dc_is_io_running(self._dc_context)):
-            self.log("dc_stop_io (stop core IO scheduler)")
-            lib.dc_stop_io(self._dc_context)
-        else:
-            self.log("stop_scheduler called on non-running context")
+        self.log("dc_stop_io (stop core IO scheduler)")
+        lib.dc_stop_io(self._dc_context)
 
     def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove

python/src/deltachat/chat.py

@@ -167,6 +167,13 @@ class Chat(object):
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
+    def get_encryption_info(self):
+        """Return encryption info for this chat.
+
+        :returns: a string with encryption preferences of all chat members"""
+        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
+        return from_dc_charpointer(res)
+
     def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
@@ -371,7 +378,7 @@ class Chat(object):
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.create_contact(obj)
+        contact = self.account.get_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
@@ -495,18 +502,23 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
+                timestamp=datetime.utcfromtimestamp(
+                    lib.dc_array_get_timestamp(dc_array, i)
+                ),
+                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+            )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp):
+    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
         self.timestamp = timestamp
+        self.marker = marker
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__

python/src/deltachat/contact.py

@@ -77,6 +77,14 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
+    @property
+    def status(self):
+        """Get contact status.
+
+        :returns: contact status, empty string if it doesn't exist.
+        """
+        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
+
     def create_chat(self):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 

python/src/deltachat/direct_imap.py

@@ -9,7 +9,9 @@ import ssl
 import pathlib
 from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
+import imaplib
 import deltachat
+from deltachat import const
 
 
 SEEN = b'\\Seen'
@@ -24,13 +26,29 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    if not hasattr(account, "direct_imap"):
-        imap = DirectImap(account)
-        if imap.select_config_folder("mvbox"):
-            imap.delete(ALL, expunge=True)
-        assert imap.select_config_folder("inbox")
-        imap.delete(ALL, expunge=True)
-        setattr(account, "direct_imap", imap)
+    try:
+
+        if not hasattr(account, "direct_imap"):
+            imap = DirectImap(account)
+
+            for folder in imap.list_folders():
+                if folder.lower() == "inbox" or folder.lower() == "deltachat":
+                    assert imap.select_folder(folder)
+                    imap.delete(ALL, expunge=True)
+                else:
+                    imap.conn.delete_folder(folder)
+                    # We just deleted the folder, so we have to make DC forget about it, too
+                    if account.get_config("configured_sentbox_folder") == folder:
+                        account.set_config("configured_sentbox_folder", None)
+                    if account.get_config("configured_spam_folder") == folder:
+                        account.set_config("configured_spam_folder", None)
+
+            setattr(account, "direct_imap", imap)
+
+    except Exception as e:
+        # Uncaught exceptions here would lead to a timeout without any note written to the log
+        account.log("=============================== CAN'T RESET ACCOUNT: ===============================")
+        account.log("===================", e, "===================")
 
 
 @deltachat.global_hookimpl
@@ -50,18 +68,31 @@ class DirectImap:
         self.connect()
 
     def connect(self):
-        ssl_context = ssl.create_default_context()
-
-        # don't check if certificate hostname doesn't match target hostname
-        ssl_context.check_hostname = False
-
-        # don't check if the certificate is trusted by a certificate authority
-        ssl_context.verify_mode = ssl.CERT_NONE
-
         host = self.account.get_config("configured_mail_server")
+        port = int(self.account.get_config("configured_mail_port"))
+        security = int(self.account.get_config("configured_mail_security"))
+
         user = self.account.get_config("addr")
         pw = self.account.get_config("mail_pw")
-        self.conn = IMAPClient(host, ssl_context=ssl_context)
+
+        if security == const.DC_SOCKET_PLAIN:
+            ssl_context = None
+        else:
+            ssl_context = ssl.create_default_context()
+
+            # don't check if certificate hostname doesn't match target hostname
+            ssl_context.check_hostname = False
+
+            # don't check if the certificate is trusted by a certificate authority
+            ssl_context.verify_mode = ssl.CERT_NONE
+
+        if security == const.DC_SOCKET_STARTTLS:
+            self.conn = IMAPClient(host, port, ssl=False)
+            self.conn.starttls(ssl_context)
+        elif security == const.DC_SOCKET_PLAIN:
+            self.conn = IMAPClient(host, port, ssl=False)
+        elif security == const.DC_SOCKET_SSL:
+            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
@@ -76,6 +107,12 @@ class DirectImap:
         except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
+    def create_folder(self, foldername):
+        try:
+            self.conn.create_folder(foldername)
+        except imaplib.IMAP4.error as e:
+            print("Can't create", foldername, "probably it already exists:", str(e))
+
     def select_folder(self, foldername):
         assert not self._idling
         return self.conn.select_folder(foldername)
@@ -226,3 +263,9 @@ class DirectImap:
             res = self.conn.idle_done()
             self._idling = False
             return res
+
+    def append(self, folder, msg):
+        if msg.startswith("\n"):
+            msg = msg[1:]
+        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
+        self.conn.append(folder, msg)

python/src/deltachat/events.py

@@ -103,6 +103,14 @@ class FFIEventTracker:
             if rex.search(ev.data2):
                 return ev
 
+    def get_info_regex_groups(self, regex, check_error=True):
+        rex = re.compile(regex)
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
+            m = rex.match(ev.data2)
+            if m is not None:
+                return m.groups()
+
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))

python/src/deltachat/message.py

@@ -212,6 +212,11 @@ class Message(object):
                 return email.message_from_bytes(s)
             return email.message_from_string(s)
 
+    @property
+    def error(self):
+        """Error message"""
+        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
+
     @property
     def chat(self):
         """chat this message was posted in.

python/src/deltachat/testplugin.py

@@ -312,24 +312,31 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io([ac1])
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
             ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io([ac1, ac2])
             return ac1, ac2
 
         def get_many_online_accounts(self, num, move=True):
             accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io(accounts)
             for acc in accounts:
                 acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
+            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
+            direct_imap object of an online account. This simulates the user setting
+            up a new device without importing a backup.
+
+            `pre_generated_key` only means that a key from python/tests/data/key is
+            used in order to speed things up.
+            """
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
@@ -349,23 +356,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac._configtracker = ac.configure()
             return ac
 
-        def wait_configure_and_start_io(self):
+        def wait_configure_and_start_io(self, accounts=None):
+            if accounts is None:
+                accounts = self._accounts[:]
             started_accounts = []
-            for acc in self._accounts:
-                if hasattr(acc, "_configtracker"):
-                    acc._configtracker.wait_finish()
-                    acc._evtracker.consume_events()
-                    acc.get_device_chat().mark_noticed()
-                    del acc._configtracker
-                acc.set_config("bcc_self", "0")
-                if acc.is_configured() and not acc.is_started():
-                    acc.start_io()
-                    started_accounts.append(acc)
-                print("{}: {} account was successfully setup".format(
-                    acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in accounts:
+                if acc not in started_accounts:
+                    self.wait_configure(acc)
+                    acc.set_config("bcc_self", "0")
+                    if acc.is_configured():
+                        acc.start_io()
+                        started_accounts.append(acc)
+                    print("{}: {} account was started".format(
+                        acc.get_config("displayname"), acc.get_config("addr")))
             for acc in started_accounts:
                 acc._evtracker.wait_all_initial_fetches()
 
+        def wait_configure(self, acc):
+            if hasattr(acc, "_configtracker"):
+                acc._configtracker.wait_finish()
+                acc._evtracker.consume_events()
+                acc.get_device_chat().mark_noticed()
+                del acc._configtracker
+
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
@@ -503,6 +516,9 @@ def lp():
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
 
+        def indent(self, msg):
+            print("  " + msg)
+
     return Printer()
 
 

python/src/deltachat/tracker.py

@@ -20,6 +20,16 @@ class ImexTracker:
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
             self._imex_events.put(ffi_event.data2)
 
+    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, int) and ev >= target_progress:
+                assert ev <= progress_upper_limit, \
+                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
+                return ev
+            if ev == 0:
+                return None
+
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
         files_written = []

python/tests/test_account.py

@@ -6,7 +6,10 @@ import queue
 import time
 from deltachat import const, Account
 from deltachat.message import Message
+from deltachat.tracker import ImexTracker
 from deltachat.hookspec import account_hookimpl
+from deltachat.capi import ffi, lib
+from deltachat.cutil import iter_array
 from datetime import datetime, timedelta
 
 
@@ -288,6 +291,28 @@ class TestOfflineChat:
         qr = chat.get_join_qr()
         assert ac2.check_qr(qr).is_ask_verifygroup
 
+    def test_removing_blocked_user_from_group(self, ac1, lp):
+        """
+        Test that blocked contact is not unblocked when removed from a group.
+        See https://github.com/deltachat/deltachat-core-rust/issues/2030
+        """
+        lp.sec("Create a group chat with a contact")
+        contact = ac1.create_contact("some1@example.org")
+        group = ac1.create_group_chat("title", contacts=[contact])
+        group.send_text("First group message")
+
+        lp.sec("ac1 blocks contact")
+        contact.block()
+        assert contact.is_blocked()
+
+        lp.sec("ac1 removes contact from their group")
+        group.remove_contact(contact)
+        assert contact.is_blocked()
+
+        lp.sec("ac1 adding blocked contact unblocks it")
+        group.add_contact(contact)
+        assert not contact.is_blocked()
+
     def test_get_set_profile_image_simple(self, ac1, data):
         chat = ac1.create_group_chat(name="title1")
         p = data.get_path("d.png")
@@ -410,11 +435,11 @@ class TestOfflineChat:
         email = "hello <hello@example.org>"
         contact1 = ac1.create_contact(email)
         assert contact1.addr == "hello@example.org"
-        assert contact1.display_name == "hello"
+        assert contact1.name == "hello"
         contact1 = ac1.create_contact(email, name="world")
-        assert contact1.display_name == "world"
+        assert contact1.name == "world"
         contact2 = ac1.create_contact("display1 <x@example.org>", "real")
-        assert contact2.display_name == "real"
+        assert contact2.name == "real"
 
     def test_create_chat_simple(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
@@ -451,6 +476,7 @@ class TestOfflineChat:
         contact = msg.get_sender_contact()
         assert contact == ac1.get_self_contact()
         assert not backupdir.listdir()
+        ac1.stop_io()
         path = ac1.export_all(backupdir.strpath)
         assert os.path.exists(path)
         ac2 = acfactory.get_unconfigured_account()
@@ -465,10 +491,6 @@ class TestOfflineChat:
         assert messages[0].text == "msg1"
         assert os.path.exists(messages[1].filename)
 
-    def test_ac_setup_message_fails(self, ac1):
-        with pytest.raises(RuntimeError):
-            ac1.initiate_key_transfer()
-
     def test_set_get_draft(self, chat1):
         msg = Message.new_empty(chat1.account, "text")
         msg1 = chat1.prepare_message(msg)
@@ -576,6 +598,28 @@ class TestOfflineChat:
         assert in_list[1][1] == chat
         assert in_list[1][2] == contacts[3]
 
+    def test_audit_log_view_without_daymarker(self, ac1, lp):
+        lp.sec("ac1: test audit log (show only system messages)")
+        chat = ac1.create_group_chat(name="audit log sample data")
+        # promote chat
+        chat.send_text("hello")
+        assert chat.is_promoted()
+
+        lp.sec("create test data")
+        chat.add_contact(ac1.create_contact("some-1@example.org"))
+        chat.set_name("audit log test group")
+        chat.send_text("a message in between")
+
+        lp.sec("check message count of all messages")
+        assert len(chat.get_messages()) == 4
+
+        lp.sec("check message count of only system messages (without daymarkers)")
+        dc_array = ffi.gc(
+            lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_INFO_ONLY, 0),
+            lib.dc_array_unref
+        )
+        assert len(list(iter_array(dc_array, lambda x: x))) == 2
+
 
 def test_basic_imap_api(acfactory, tmpdir):
     ac1, ac2 = acfactory.get_two_online_accounts()
@@ -609,7 +653,7 @@ class TestOnlineAccount:
             config={"key_gen_type": str(const.DC_KEY_GEN_ED25519)}
         )
         # rsa key gen can be slow especially on CI, adjust timeout
-        ac1._evtracker.set_timeout(120)
+        ac1._evtracker.set_timeout(240)
         acfactory.wait_configure_and_start_io()
         chat = acfactory.get_accepted_chat(ac1, ac2)
 
@@ -646,7 +690,7 @@ class TestOnlineAccount:
         except Exception:
             pass
 
-    def test_export_import_self_keys(self, acfactory, tmpdir):
+    def test_export_import_self_keys(self, acfactory, tmpdir, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         dir = tmpdir.mkdir("exportdir")
@@ -654,8 +698,17 @@ class TestOnlineAccount:
         assert len(export_files) == 2
         for x in export_files:
             assert x.startswith(dir.strpath)
+        key_id, = ac1._evtracker.get_info_regex_groups(r".*xporting.*KeyId\((.*)\).*")
         ac1._evtracker.consume_events()
+
+        lp.sec("exported keys (private and public)")
+        for name in os.listdir(dir.strpath):
+            lp.indent(dir.strpath + os.sep + name)
+        lp.sec("importing into existing account")
         ac2.import_self_keys(dir.strpath)
+        key_id2, = ac2._evtracker.get_info_regex_groups(
+            r".*stored.*KeyId\((.*)\).*", check_error=False)
+        assert key_id2 == key_id
 
     def test_one_account_send_bcc_setting(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
@@ -938,6 +991,54 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
+    @pytest.mark.parametrize("mvbox_move", [True, False])
+    def test_markseen_message_and_mdn(self, acfactory, mvbox_move):
+        # Please only change this test if you are very sure that it will still catch the issues it catches now.
+        # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
+        ac1 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        acfactory.wait_configure_and_start_io()
+
+        acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
+        msg = ac2._evtracker.wait_next_incoming_message()
+
+        folder = "mvbox" if mvbox_move else "inbox"
+        ac1.direct_imap.select_config_folder(folder)
+        ac2.direct_imap.select_config_folder(folder)
+        ac1.direct_imap.idle_start()
+        ac2.direct_imap.idle_start()
+
+        ac2.mark_seen_messages([msg])
+
+        ac1.direct_imap.idle_wait_for_seen()  # Check that the mdn is marked as seen
+        ac2.direct_imap.idle_wait_for_seen()  # Check that the original message is marked as seen
+        ac1.direct_imap.idle_done()
+        ac2.direct_imap.idle_done()
+
+    def test_reply_privately(self, acfactory):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        group1 = ac1.create_group_chat("group")
+        group1.add_contact(ac2)
+        group1.send_text("hello")
+
+        msg2 = ac2._evtracker.wait_next_messages_changed()
+        group2 = msg2.create_chat()
+        assert group2.get_name() == group1.get_name()
+
+        msg_reply = Message.new_empty(ac2, "text")
+        msg_reply.set_text("message reply")
+        msg_reply.quote = msg2
+
+        private_chat1 = ac1.create_chat(ac2)
+        private_chat2 = ac2.create_chat(ac1)
+        private_chat2.send_msg(msg_reply)
+
+        msg_reply1 = ac1._evtracker.wait_next_incoming_message()
+        assert msg_reply1.quoted_text == "hello"
+        assert not msg_reply1.chat.is_group()
+        assert msg_reply1.chat.id == private_chat1.id
+
     def test_mdn_asymetric(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts(move=True)
 
@@ -1017,6 +1118,64 @@ class TestOnlineAccount:
         assert not msg.is_encrypted()
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    def test_gossip_encryption_preference(self, acfactory, lp):
+        """Test that encryption preference of group members is gossiped to new members.
+        This is a Delta Chat extension to Autocrypt 1.1.0, which Autocrypt-Gossip headers
+        SHOULD NOT contain encryption preference.
+        """
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+
+        lp.sec("ac1 learns that ac2 prefers encryption")
+        ac1.create_chat(ac2)
+        msg = ac2.create_chat(ac1).send_text("first message")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "first message"
+        assert not msg.is_encrypted()
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config('addr'))
+        assert msg.chat.get_encryption_info() == res
+        lp.sec("ac2 learns that ac3 prefers encryption")
+        ac2.create_chat(ac3)
+        msg = ac3.create_chat(ac2).send_text("I prefer encryption")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "I prefer encryption"
+        assert not msg.is_encrypted()
+
+        lp.sec("ac3 does not know that ac1 prefers encryption")
+        ac1.create_chat(ac3)
+        chat = ac3.create_chat(ac1)
+        res = "{} No encryption.".format(ac1.get_config('addr'))
+        assert chat.get_encryption_info() == res
+        msg = chat.send_text("not encrypted")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "not encrypted"
+        assert not msg.is_encrypted()
+
+        lp.sec("ac1 creates a group chat with ac2")
+        group_chat = ac1.create_group_chat("hello")
+        group_chat.add_contact(ac2)
+        encryption_info = group_chat.get_encryption_info()
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config("addr"))
+        assert encryption_info == res
+        msg = group_chat.send_text("hi")
+
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+        assert msg.text == "hi"
+
+        lp.sec("ac2 adds ac3 to the group")
+        msg.chat.add_contact(ac3)
+        assert msg.is_encrypted()
+
+        lp.sec("ac3 learns that ac1 prefers encryption")
+        msg = ac3._evtracker.wait_next_incoming_message()
+        encryption_info = msg.chat.get_encryption_info().splitlines()
+        res = "{} End-to-end encryption preferred.".format(ac1.get_config("addr"))
+        assert res in encryption_info
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config("addr"))
+        assert res in encryption_info
+        msg = chat.send_text("encrypted")
+        assert msg.is_encrypted()
+
     def test_send_first_message_as_long_unicode_with_cr(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac2.set_config("save_mime_headers", "1")
@@ -1056,6 +1215,52 @@ class TestOnlineAccount:
         assert not device_chat.can_send()
         assert device_chat.get_draft() is None
 
+    def test_dont_show_emails_in_draft_folder(self, acfactory):
+        """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them.
+        So: If it's outgoing AND there is no Received header AND it's not in the sentbox, then ignore the email."""
+        ac1 = acfactory.get_online_configuring_account()
+        ac1.set_config("show_emails", "2")
+        ac1.create_contact("alice@example.com").create_chat()
+
+        acfactory.wait_configure(ac1)
+        ac1.direct_imap.create_folder("Drafts")
+        ac1.direct_imap.create_folder("Sent")
+
+        acfactory.wait_configure_and_start_io()
+        # Wait until each folder was selected once and we are IDLEing again:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.stop_io()
+
+        ac1.direct_imap.append("Drafts", """
+            From: ac1 <{}>
+            Subject: subj
+            To: alice@example.com
+            Message-ID: <aepiors@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            message in Drafts
+        """.format(ac1.get_config("configured_addr")))
+        ac1.direct_imap.append("Sent", """
+            From: ac1 <{}>
+            Subject: subj
+            To: alice@example.com
+            Message-ID: <hsabaeni@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            message in Sent
+        """.format(ac1.get_config("configured_addr")))
+
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+        ac1.start_io()
+
+        msg = ac1._evtracker.wait_next_messages_changed()
+
+        # Wait until each folder was scanned, this is necessary for this test to test what it should test:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+
+        assert msg.text == "subj – message in Sent"
+        assert len(msg.chat.get_messages()) == 1
+
     def test_prefer_encrypt(self, acfactory, lp):
         """Test quorum rule for encryption preference in 1:1 and group chat."""
         ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
@@ -1106,8 +1311,8 @@ class TestOnlineAccount:
         # Majority prefers encryption now
         assert msg5.is_encrypted()
 
-    def test_reply_encrypted(self, acfactory, lp):
-        """Test that replies to encrypted messages are encrypted."""
+    def test_quote_encrypted(self, acfactory, lp):
+        """Test that replies to encrypted messages with quotes are encrypted."""
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         lp.sec("ac1: create chat with ac2")
@@ -1156,6 +1361,39 @@ class TestOnlineAccount:
             assert msg_in.quoted_text == quoted_msg.text
             assert msg_in.is_encrypted() == quoted_msg.is_encrypted()
 
+    def test_quote_attachment(self, tmpdir, acfactory, lp):
+        """Test that replies with an attachment and a quote are received correctly."""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1 creates chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+
+        lp.sec("ac1 sends text message to ac2")
+        chat1.send_text("hi")
+
+        lp.sec("ac2 receives contact request from ac1")
+        received_message = ac2._evtracker.wait_next_messages_changed()
+        assert received_message.text == "hi"
+
+        basename = "attachment.txt"
+        p = os.path.join(tmpdir.strpath, basename)
+        with open(p, "w") as f:
+            f.write("data to send")
+
+        lp.sec("ac2 sends a reply to ac1")
+        chat2 = received_message.create_chat()
+        reply = Message.new_empty(ac2, "file")
+        reply.set_text("message reply")
+        reply.set_file(p)
+        reply.quote = received_message
+        chat2.send_msg(reply)
+
+        lp.sec("ac1 receives a reply from ac2")
+        received_reply = ac1._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "message reply"
+        assert received_reply.quoted_text == received_message.text
+        assert open(received_reply.filename).read() == "data to send"
+
     def test_saved_mime_on_received_message(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -1249,19 +1487,52 @@ class TestOnlineAccount:
         m = message_queue.get()
         assert m == msg_in
 
-    def test_import_export_online_all(self, acfactory, tmpdir, lp):
+    def test_import_export_online_all(self, acfactory, tmpdir, data, lp):
         ac1 = acfactory.get_one_online_account()
 
         lp.sec("create some chat content")
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact1.create_chat().send_text("msg1")
+        chat1 = ac1.create_contact("some1@example.org", name="some1").create_chat()
+        chat1.send_text("msg1")
         assert len(ac1.get_contacts(query="some1")) == 1
+
+        original_image_path = data.get_path("d.png")
+        chat1.send_image(original_image_path)
+
+        def assert_account_is_proper(ac):
+            contacts = ac.get_contacts(query="some1")
+            assert len(contacts) == 1
+            contact2 = contacts[0]
+            assert contact2.addr == "some1@example.org"
+            chat2 = contact2.create_chat()
+            messages = chat2.get_messages()
+            assert len(messages) == 2
+            assert messages[0].text == "msg1"
+            assert messages[1].filemime == "image/png"
+            assert os.stat(messages[1].filename).st_size == os.stat(original_image_path).st_size
+            ac.set_config("displayname", "new displayname")
+            assert ac.get_config("displayname") == "new displayname"
+
+        assert_account_is_proper(ac1)
+
         backupdir = tmpdir.mkdir("backup")
 
         lp.sec("export all to {}".format(backupdir))
-        path = ac1.export_all(backupdir.strpath)
-        assert os.path.exists(path)
-        t = time.time()
+        with ac1.temp_plugin(ImexTracker()) as imex_tracker:
+
+            ac1.stop_io()
+            ac1.imex(backupdir.strpath, const.DC_IMEX_EXPORT_BACKUP)
+
+            # check progress events for export
+            assert imex_tracker.wait_progress(1, progress_upper_limit=249)
+            assert imex_tracker.wait_progress(250, progress_upper_limit=499)
+            assert imex_tracker.wait_progress(500, progress_upper_limit=749)
+            assert imex_tracker.wait_progress(750, progress_upper_limit=999)
+
+            paths = imex_tracker.wait_finish()
+            assert len(paths) == 1
+            path = paths[0]
+            assert os.path.exists(path)
+            ac1.start_io()
 
         lp.sec("get fresh empty account")
         ac2 = acfactory.get_unconfigured_account()
@@ -1271,22 +1542,20 @@ class TestOnlineAccount:
         assert path2 == path
 
         lp.sec("import backup and check it's proper")
-        ac2.import_all(path)
-        contacts = ac2.get_contacts(query="some1")
-        assert len(contacts) == 1
-        contact2 = contacts[0]
-        assert contact2.addr == "some1@example.org"
-        chat2 = contact2.create_chat()
-        messages = chat2.get_messages()
-        assert len(messages) == 1
-        assert messages[0].text == "msg1"
+        with ac2.temp_plugin(ImexTracker()) as imex_tracker:
+            ac2.import_all(path)
+
+            # check progress events for import
+            assert imex_tracker.wait_progress(1, progress_upper_limit=249)
+            assert imex_tracker.wait_progress(500, progress_upper_limit=749)
+            assert imex_tracker.wait_progress(750, progress_upper_limit=999)
+            assert imex_tracker.wait_progress(1000)
+
+        assert_account_is_proper(ac1)
+        assert_account_is_proper(ac2)
 
-        # wait until a second passed since last backup
-        # because get_latest_backupfile() shall return the latest backup
-        # from a UI it's unlikely anyone manages to export two
-        # backups in one second.
-        time.sleep(max(0, 1 - (time.time() - t)))
         lp.sec("Second-time export all to {}".format(backupdir))
+        ac1.stop_io()
         path2 = ac1.export_all(backupdir.strpath)
         assert os.path.exists(path2)
         assert path2 != path
@@ -1521,7 +1790,7 @@ class TestOnlineAccount:
 
         lp.sec("ac1 blocks ac2")
         contact = ac1.create_contact(ac2)
-        contact.set_blocked()
+        contact.block()
         assert contact.is_blocked()
         ev = ac1._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
         assert ev.data1 == contact.id
@@ -1620,7 +1889,7 @@ class TestOnlineAccount:
 
         ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5)
         ac1._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED")
-        chat1.send_text("hello")
+        chat1.send_text("🍞")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
         lp.sec("ac2: wait for incoming location message")
@@ -1634,6 +1903,7 @@ class TestOnlineAccount:
         assert locations[0].longitude == 3.0
         assert locations[0].accuracy == 0.5
         assert locations[0].timestamp > now
+        assert locations[0].marker == "🍞"
 
         contact = ac2.create_contact(ac1)
         locations2 = chat2.get_locations(contact=contact)
@@ -1685,6 +1955,7 @@ class TestOnlineAccount:
         # Error message should be assigned to the chat with ac1.
         lp.sec("ac4: checking that message is assigned to the sender chat")
         error_msg = ac4._evtracker.wait_next_incoming_message()
+        assert error_msg.error  # There is an error decrypting the message
         assert error_msg.chat == chat41
 
         lp.sec("ac2: sending a reply to the chat")
@@ -1695,6 +1966,7 @@ class TestOnlineAccount:
 
         lp.sec("ac4: checking that reply is assigned to ac2 chat")
         error_reply = ac4._evtracker.wait_next_incoming_message()
+        assert error_reply.error  # There is an error decrypting the message
         assert error_reply.chat == chat42
 
         # Test that ac4 replies to error messages don't appear in the
@@ -1706,11 +1978,13 @@ class TestOnlineAccount:
         chat42.send_text("I can't decrypt your message, ac2!")
 
         msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.error is None
         assert msg.text == "I can't decrypt your message, ac1!"
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac1.create_chat(ac3)
 
         msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.error is None
         assert msg.text == "I can't decrypt your message, ac2!"
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac2.create_chat(ac4)
@@ -1871,7 +2145,9 @@ class TestOnlineAccount:
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac1.set_config("displayname", "Account 1")
 
-        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        # Similar to acfactory.get_accepted_chat, but without setting the contact name.
+        ac2.create_contact(ac1.get_config("addr")).create_chat()
+        chat12 = ac1.create_contact(ac2.get_config("addr")).create_chat()
         contact = None
 
         def update_name():
@@ -1902,12 +2178,24 @@ class TestOnlineAccount:
         # so it should not be changed.
         ac1.set_config("displayname", "Renamed again")
         updated_name = update_name()
-        if updated_name == "Renamed again":
-            # Known bug, mark as XFAIL
-            pytest.xfail("Contact was renamed after explicit rename")
-        else:
-            # No renames should happen after explicit rename
-            assert updated_name == "Renamed"
+        assert updated_name == "Renamed"
+
+    def test_status(self, acfactory):
+        """Test that status is transferred over the network."""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        ac1.set_config("selfstatus", "New status")
+        chat12.send_text("hi")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hi"
+        assert msg.get_sender_contact().status == "New status"
+
+        ac1.set_config("selfstatus", "")
+        chat12.send_text("hello")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+        assert msg.get_sender_contact().status == ""
 
     def test_group_quote(self, acfactory, lp):
         """Test quoting in a group with a new member who have not seen the quoted message."""
@@ -1948,42 +2236,162 @@ class TestOnlineAccount:
         assert received_reply.quoted_text == "hello"
         assert received_reply.quote.id == out_msg.id
 
+    @pytest.mark.parametrize("folder,move,expected_destination,", [
+        ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
+        ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
+        ("Spam", False, "INBOX")  # ...emails are moved from the spam folder to the Inbox
+    ])
+    # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
+    # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
+    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination):
+        """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
+        variant = folder + "-" + str(move) + "-" + expected_destination
+        lp.sec("Testing variant " + variant)
+        ac1 = acfactory.get_online_configuring_account(move=move)
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure(ac1)
+        ac1.direct_imap.create_folder(folder)
+
+        acfactory.wait_configure_and_start_io()
+        # Wait until each folder was selected once and we are IDLEing:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.stop_io()
+
+        # Send a message to ac1 and move it to the mvbox:
+        ac1.direct_imap.select_config_folder("inbox")
+        ac1.direct_imap.idle_start()
+        acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
+
+        lp.sec("Everything prepared, now see if DeltaChat finds the message (" + variant + ")")
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+        ac1.start_io()
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        # Wait until the message was moved (if at all) and we are IDLEing again:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.direct_imap.select_folder(expected_destination)
+        assert len(ac1.direct_imap.get_all_messages()) == 1
+        if folder != expected_destination:
+            ac1.direct_imap.select_folder(folder)
+            assert len(ac1.direct_imap.get_all_messages()) == 0
+
     @pytest.mark.parametrize("mvbox_move", [False, True])
-    def test_add_all_recipients_as_contacts(self, acfactory, lp, mvbox_move):
+    def test_fetch_existing(self, acfactory, lp, mvbox_move):
         """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
         This way, we can already offer them some email addresses they can write to.
 
-        Also test that existing emails are fetched during onboarding.
+        Also, the newest existing emails from each folder are fetched during onboarding.
+
+        Additionally tests that bcc_self messages moved to the mvbox/sentbox are marked as read."""
+
+        def assert_folders_configured(ac):
+            """There was a bug that scan_folders() set the configured folders to None under some circumstances.
+            So, check that they are still configured:"""
+            assert ac.get_config("configured_sentbox_folder") == "Sent"
+            if mvbox_move:
+                assert ac.get_config("configured_mvbox_folder")
 
-        Lastly, tests that bcc_self messages moved to the mvbox are marked as read."""
         ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
+        ac1.set_config("sentbox_move", "1")
         ac2 = acfactory.get_online_configuring_account()
 
+        acfactory.wait_configure(ac1)
+
+        ac1.direct_imap.create_folder("Sent")
+        ac1.set_config("sentbox_watch", "1")
+
+        # We need to reconfigure to find the new "Sent" folder.
+        # `scan_folders()`, which runs automatically shortly after `start_io()` is invoked,
+        # would also find the "Sent" folder, but it would be too late:
+        # The sentbox thread, started by `start_io()`, would have seen that there is no
+        # ConfiguredSentboxFolder and do nothing.
+        ac1._configtracker = ac1.configure(reconfigure=True)
         acfactory.wait_configure_and_start_io()
+        assert_folders_configured(ac1)
 
-        chat = acfactory.get_accepted_chat(ac1, ac2)
-
-        lp.sec("send out message with bcc to ourselves")
         if mvbox_move:
             ac1.direct_imap.select_config_folder("mvbox")
+        else:
+            ac1.direct_imap.select_config_folder("sentbox")
         ac1.direct_imap.idle_start()
+
+        lp.sec("send out message with bcc to ourselves")
         ac1.set_config("bcc_self", "1")
+        chat = acfactory.get_accepted_chat(ac1, ac2)
         chat.send_text("message text")
+        assert_folders_configured(ac1)
 
         # now wait until the bcc_self message arrives
         # Also test that bcc_self messages moved to the mvbox are marked as read.
         assert ac1.direct_imap.idle_wait_for_seen()
+        assert_folders_configured(ac1)
 
         ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone.set_config("fetch_existing_msgs", "1")
         ac1_clone._configtracker.wait_finish()
         ac1_clone.start_io()
+        assert_folders_configured(ac1_clone)
 
         ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
         ac2_addr = ac2.get_config("addr")
         assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
+        assert_folders_configured(ac1_clone)
 
         msg = ac1_clone._evtracker.wait_next_messages_changed()
         assert msg.text == "message text"
+        assert_folders_configured(ac1)
+        assert_folders_configured(ac1_clone)
+
+    def test_fetch_existing_msgs_group_and_single(self, acfactory, lp):
+        """There was a bug concerning fetch-existing-msgs:
+
+        A sent a message to you, adding you to a group. This created a contact request.
+        You wrote a message to A, creating a chat.
+        ...but the group stayed blocked.
+        So, after fetch-existing-msgs you have one contact request and one chat with the same person.
+
+        See https://github.com/deltachat/deltachat-core-rust/issues/2097"""
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure_and_start_io()
+
+        lp.sec("receive a message")
+        ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
+        ac1._evtracker.wait_next_messages_changed()
+
+        lp.sec("send out message with bcc to ourselves")
+        ac1.direct_imap.idle_start()
+        ac1.set_config("bcc_self", "1")
+        ac1.create_chat(ac2).send_text("outgoing, encrypted direct message, creating a chat")
+
+        # now wait until the bcc_self message arrives
+        assert ac1.direct_imap.idle_wait_for_seen()
+
+        lp.sec("Clone online account and let it fetch the existing messages")
+        ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone.set_config("fetch_existing_msgs", "1")
+        ac1_clone._configtracker.wait_finish()
+
+        ac1_clone.start_io()
+        ac1_clone._evtracker.wait_all_initial_fetches()
+        chats = ac1_clone.get_chats()
+        assert len(chats) == 4  # two newly created chats + self-chat + device-chat
+        group_chat = [c for c in chats if c.get_name() == "group name"][0]
+        assert group_chat.is_group()
+        private_chat = [c for c in chats if c.get_name() == "ac2"][0]
+        assert not private_chat.is_group()
+
+        group_messages = group_chat.get_messages()
+        assert len(group_messages) == 1
+        assert group_messages[0].text == "incoming, unencrypted group message"
+        private_messages = private_chat.get_messages()
+        # We can't decrypt the message in this chat, so the chat is empty:
+        assert len(private_messages) == 0
 
 
 class TestGroupStressTests:

python/tox.ini

@@ -47,7 +47,9 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx
+    # With Python 3.7 and Sphinx 3.5.0, it throws an exception.
+    # Pin the version to the working one.
+    sphinx==3.4.3
     breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html

rust-toolchain

@@ -1 +1 @@
-1.45.0
+1.50.0

set_core_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys
@@ -63,13 +63,14 @@ def main():
     ffi_toml = read_toml_version("deltachat-ffi/Cargo.toml")
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
-    for line in open("CHANGELOG.md"):
-        ## 1.25.0
-        if line.startswith("## "):
-            if line[2:].strip().startswith(newversion):
-                break
-    else:
-        raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
+    if "alpha" not in newversion:
+        for line in open("CHANGELOG.md"):
+            ## 1.25.0
+            if line.startswith("## "):
+                if line[2:].strip().startswith(newversion):
+                    break
+        else:
+            raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
 
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)

src/accounts.rs

@@ -6,11 +6,10 @@ use async_std::prelude::*;
 use async_std::sync::{Arc, RwLock};
 use uuid::Uuid;
 
-use anyhow::{ensure, Context as _};
+use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
-use crate::error::Result;
 use crate::events::Event;
 
 /// Account manager, that can handle multiple accounts in a single place.
@@ -188,7 +187,7 @@ impl Accounts {
         let id = self.add_account().await?;
         let ctx = self.get_account(id).await.expect("just added");
 
-        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, Some(file)).await {
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, &file).await {
             Ok(_) => Ok(id),
             Err(err) => {
                 // remove temp account
@@ -347,7 +346,6 @@ impl Config {
 
             inner.accounts.push(AccountConfig {
                 id,
-                name: String::new(),
                 dir: target_dir.into(),
                 uuid,
             });
@@ -414,8 +412,6 @@ impl Config {
 pub struct AccountConfig {
     /// Unique id.
     pub id: u32,
-    /// Display name
-    pub name: String,
     /// Root directory for all data for this account.
     pub dir: std::path::PathBuf,
     pub uuid: Uuid,

src/aheader.rs

@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
-use crate::contact::*;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};

src/blob.rs

@@ -7,14 +7,17 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::Error;
 use image::GenericImageView;
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::{
+    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
+    WORSE_IMAGE_SIZE,
+};
 use crate::context::Context;
-use crate::error::Error;
 use crate::events::EventType;
 use crate::message;
 
@@ -55,7 +58,7 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
-        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext).await?;
+        let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
             .map_err(|err| BlobError::WriteFailure {
@@ -63,6 +66,12 @@ impl<'a> BlobObject<'a> {
                 blobname: name.clone(),
                 cause: err.into(),
             })?;
+
+        // workaround a bug in async-std
+        // (the executor does not handle blocking operation in Drop correctly,
+        // see https://github.com/async-rs/async-std/issues/900 )
+        let _ = file.flush().await;
+
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
@@ -151,6 +160,10 @@ impl<'a> BlobObject<'a> {
                 cause: err,
             });
         }
+
+        // workaround, see create() for details
+        let _ = dst_file.flush().await;
+
         let blob = BlobObject {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
@@ -367,27 +380,18 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
         let blob_abs = self.to_abs_path();
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
-        })?;
 
-        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
-            return Ok(());
-        }
+        let img_wh =
+            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+                .unwrap_or_default()
+            {
+                MediaQuality::Balanced => BALANCED_AVATAR_SIZE,
+                MediaQuality::Worse => WORSE_AVATAR_SIZE,
+            };
 
-        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
-        Ok(())
+        self.recode_to_size(context, blob_abs, img_wh).await
     }
 
     pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
@@ -398,39 +402,54 @@ impl<'a> BlobObject<'a> {
             return Ok(());
         }
 
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+        let img_wh =
+            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+                .unwrap_or_default()
+            {
+                MediaQuality::Balanced => BALANCED_IMAGE_SIZE,
+                MediaQuality::Worse => WORSE_IMAGE_SIZE,
+            };
+
+        self.recode_to_size(context, blob_abs, img_wh).await
+    }
+
+    async fn recode_to_size(
+        &self,
+        context: &Context,
+        blob_abs: PathBuf,
+        img_wh: u32,
+    ) -> Result<(), BlobError> {
+        let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
+        let orientation = self.get_exif_orientation(context);
 
-        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
-            .unwrap_or_default()
-            == MediaQuality::Balanced
-        {
-            BALANCED_IMAGE_SIZE
-        } else {
-            WORSE_IMAGE_SIZE
-        };
+        let do_scale = img.width() > img_wh || img.height() > img_wh;
+        let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
 
-        if img.width() <= img_wh && img.height() <= img_wh {
-            return Ok(());
+        if do_scale || do_rotate {
+            if do_scale {
+                img = img.thumbnail(img_wh, img_wh);
+            }
+
+            if do_rotate {
+                img = match orientation {
+                    Ok(90) => img.rotate90(),
+                    Ok(180) => img.rotate180(),
+                    Ok(270) => img.rotate270(),
+                    _ => img,
+                }
+            }
+
+            img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+                blobdir: context.get_blobdir().to_path_buf(),
+                blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+                cause: err.into(),
+            })?;
         }
 
-        let mut img = img.thumbnail(img_wh, img_wh);
-        match self.get_exif_orientation(context) {
-            Ok(90) => img = img.rotate90(),
-            Ok(180) => img = img.rotate180(),
-            Ok(270) => img = img.rotate270(),
-            _ => {}
-        }
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
         Ok(())
     }
 
@@ -501,69 +520,57 @@ pub enum BlobError {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_create() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
-        let fname = t.ctx.get_blobdir().join("foo");
+        let blob = BlobObject::create(&t, "foo", b"hello").await.unwrap();
+        let fname = t.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
         assert_eq!(data, b"hello");
         assert_eq!(blob.as_name(), "$BLOBDIR/foo");
-        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
+        assert_eq!(blob.to_abs_path(), t.get_blobdir().join("foo"));
     }
 
     #[async_std::test]
     async fn test_lowercase_ext() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.TXT", b"hello").await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_file_name() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_rel_path() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
     #[async_std::test]
     async fn test_suffix() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.suffix(), Some("txt"));
-        let blob = BlobObject::create(&t.ctx, "bar", b"world").await.unwrap();
+        let blob = BlobObject::create(&t, "bar", b"world").await.unwrap();
         assert_eq!(blob.suffix(), None);
     }
 
     #[async_std::test]
     async fn test_create_dup() {
         let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
-        let foo_path = t.ctx.get_blobdir().join("foo.txt");
+        BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let foo_path = t.get_blobdir().join("foo.txt");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.txt", b"world")
-            .await
-            .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
+        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -579,15 +586,15 @@ mod tests {
     #[async_std::test]
     async fn test_double_ext_preserved() {
         let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
+        BlobObject::create(&t, "foo.tar.gz", b"hello")
             .await
             .unwrap();
-        let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
+        let foo_path = t.get_blobdir().join("foo.tar.gz");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"world")
+        BlobObject::create(&t, "foo.tar.gz", b"world")
             .await
             .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -605,7 +612,7 @@ mod tests {
     async fn test_create_long_names() {
         let t = TestContext::new().await;
         let s = "1".repeat(150);
-        let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
+        let blob = BlobObject::create(&t, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
         assert!(blobname.len() < 128);
     }
@@ -615,14 +622,14 @@ mod tests {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
+        let blob = BlobObject::create_and_copy(&t, &src).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).await.is_err());
-        let whoops = t.ctx.get_blobdir().join("whoops");
+        assert!(BlobObject::create_and_copy(&t, &whoops).await.is_err());
+        let whoops = t.get_blobdir().join("whoops");
         assert!(!whoops.exists().await);
     }
 
@@ -632,14 +639,14 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
-        let src_int = t.ctx.get_blobdir().join("internal");
+        let src_int = t.get_blobdir().join("internal");
         fs::write(&src_int, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_int).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_int).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
@@ -649,7 +656,7 @@ mod tests {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"

src/chatlist.rs

@@ -1,15 +1,20 @@
 //! # Chat list module
 
+use anyhow::{bail, ensure, Result};
+
 use crate::chat;
-use crate::chat::*;
-use crate::constants::*;
-use crate::contact::*;
-use crate::context::*;
+use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
+use crate::constants::{
+    Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_DEADDROP,
+    DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT,
+    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+};
+use crate::contact::Contact;
+use crate::context::Context;
 use crate::ephemeral::delete_expired_messages;
-use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 /// An object representing a single chatlist in memory.
 ///
@@ -58,9 +63,8 @@ impl Chatlist {
     ///   messages from addresses that have no relationship to the configured account.
     ///   The last of these messages is represented by DC_CHAT_ID_DEADDROP and you can retrieve details
     ///   about it with chatlist.get_msg_id(). Typically, the UI asks the user "Do you want to chat with NAME?"
-    ///   and offers the options "Yes" (call dc_create_chat_by_msg_id()), "Never" (call dc_block_contact())
-    ///   or "Not now".
-    ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
+    ///   and offers the options "Start chat", "Block" and "Not now";
+    ///   The decision should be passed to dc_decide_on_contact_request().
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
     ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
@@ -359,28 +363,29 @@ impl Chatlist {
             return ret;
         };
 
-        let mut lastcontact = None;
-
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
-            if lastmsg.from_id != DC_CONTACT_ID_SELF && chat.typ == Chattype::Group {
-                lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
-            }
-
-            Some(lastmsg)
-        } else {
-            None
-        };
+        let (lastmsg, lastcontact) =
+            if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
+                if lastmsg.from_id == DC_CONTACT_ID_SELF {
+                    (Some(lastmsg), None)
+                } else {
+                    match chat.typ {
+                        Chattype::Group | Chattype::Mailinglist => {
+                            let lastcontact =
+                                Contact::load_from_db(context, lastmsg.from_id).await.ok();
+                            (Some(lastmsg), lastcontact)
+                        }
+                        Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
+                    }
+                }
+            } else {
+                (None, None)
+            };
 
         if chat.id.is_archived_link() {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.text2 = Some(
-                context
-                    .stock_str(StockMessage::NoMessages)
-                    .await
-                    .to_string(),
-            );
+            ret.text2 = Some(stock_str::no_messages(context).await);
         } else {
             ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
                 .await;
@@ -433,23 +438,26 @@ async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::chat::{create_group_chat, ProtectionStatus};
+    use crate::constants::Viewtype;
+    use crate::stock_str::StockMessage;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_try_load() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "b chat")
+        let chat_id2 = create_group_chat(&t, ProtectionStatus::Unprotected, "b chat")
             .await
             .unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "c chat")
+        let chat_id3 = create_group_chat(&t, ProtectionStatus::Unprotected, "c chat")
             .await
             .unwrap();
 
         // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
         assert_eq!(chats.get_chat_id(0), chat_id3);
         assert_eq!(chats.get_chat_id(1), chat_id2);
@@ -458,26 +466,24 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t.ctx, Some(&mut msg)).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        chat_id2.set_draft(&t, Some(&mut msg)).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None)
-            .await
-            .unwrap();
+        let chats = Chatlist::try_load(&t, 0, Some("b"), None).await.unwrap();
         assert_eq!(chats.len(), 1);
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
         chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .set_visibility(&t, ChatVisibility::Archived)
             .await
             .ok();
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
@@ -486,23 +492,23 @@ mod tests {
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
         let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
-        create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
+        t.update_device_chats().await.unwrap();
+        create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_FOR_FORWARDING, None, None)
             .await
             .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+        assert!(Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -511,31 +517,29 @@ mod tests {
     #[async_std::test]
     async fn test_search_special_chat_names() {
         let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
+        t.update_device_chats().await.unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
-        t.ctx
-            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+        t.set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
 
-        t.ctx
-            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+        t.set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
@@ -544,16 +548,16 @@ mod tests {
     #[async_std::test]
     async fn test_get_summary_unwrap() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t.ctx, Some(&mut msg)).await;
+        chat_id1.set_draft(&t, Some(&mut msg)).await;
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        let summary = chats.get_summary(&t.ctx, 0, None).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let summary = chats.get_summary(&t, 0, None).await;
         assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/color.rs

@@ -0,0 +1,62 @@
+//! Implementation of Consistent Color Generation
+//!
+//! Consistent Color Generation is defined in XEP-0392.
+//!
+//! Color Vision Deficiency correction is not implemented as Delta Chat does not offer
+//! corresponding settings.
+use hsluv::hsluv_to_rgb;
+use sha1::{Digest, Sha1};
+
+/// Converts an identifier to Hue angle.
+fn str_to_angle(s: impl AsRef<str>) -> f64 {
+    let bytes = s.as_ref().as_bytes();
+    let result = Sha1::digest(bytes);
+    let checksum: u16 = result.get(0).map_or(0, |&x| u16::from(x))
+        + 256 * result.get(1).map_or(0, |&x| u16::from(x));
+    f64::from(checksum) / 65536.0 * 360.0
+}
+
+/// Converts RGB tuple to a 24-bit number.
+///
+/// Returns a 24-bit number with 8 least significant bits corresponding to the blue color and 8
+/// most significant bits corresponding to the red color.
+fn rgb_to_u32((r, g, b): (f64, f64, f64)) -> u32 {
+    let r = ((r * 256.0) as u32).min(255);
+    let g = ((g * 256.0) as u32).min(255);
+    let b = ((b * 256.0) as u32).min(255);
+    65536 * r + 256 * g + b
+}
+
+/// Converts an identifier to RGB color.
+///
+/// Saturation is set to maximum (100.0) to make colors distinguishable, and lightness is set to
+/// half (50.0) to make colors suitable both for light and dark theme.
+pub(crate) fn str_to_color(s: impl AsRef<str>) -> u32 {
+    rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_str_to_angle() {
+        // Test against test vectors from
+        // https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd
+        assert!((str_to_angle("Romeo") - 327.255249).abs() < 1e-6);
+        assert!((str_to_angle("juliet@capulet.lit") - 209.410400).abs() < 1e-6);
+        assert!((str_to_angle("😺") - 331.199341).abs() < 1e-6);
+        assert!((str_to_angle("council") - 359.994507).abs() < 1e-6);
+        assert!((str_to_angle("Board") - 171.430664).abs() < 1e-6);
+    }
+
+    #[test]
+    fn test_rgb_to_u32() {
+        assert_eq!(rgb_to_u32((0.0, 0.0, 0.0)), 0);
+        assert_eq!(rgb_to_u32((1.0, 1.0, 1.0)), 0xffffff);
+        assert_eq!(rgb_to_u32((0.0, 0.0, 1.0)), 0x0000ff);
+        assert_eq!(rgb_to_u32((0.0, 1.0, 0.0)), 0x00ff00);
+        assert_eq!(rgb_to_u32((1.0, 0.0, 0.0)), 0xff0000);
+        assert_eq!(rgb_to_u32((1.0, 0.5, 0.0)), 0xff8000);
+    }
+}

src/config.rs

@@ -7,12 +7,13 @@ use crate::blob::BlobObject;
 use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
 use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
-use crate::stock::StockMessage;
+use crate::provider::{get_provider_by_id, Provider};
+use crate::stock_str;
 
 /// The available configuration keys.
 #[derive(
@@ -35,9 +36,6 @@ pub enum Config {
     SmtpCertificateChecks,
     ServerFlags,
 
-    #[strum(props(default = "INBOX"))]
-    ImapFolder,
-
     Displayname,
     Selfstatus,
     Selfavatar,
@@ -63,14 +61,20 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MvboxMove,
 
+    #[strum(props(default = "0"))]
+    SentboxMove, // If `MvboxMove` is true, this config is ignored. Currently only used in tests.
+
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
     #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
     MediaQuality,
 
+    /// If set to "1", on the first time `start_io()` is called after configuring,
+    /// the newest existing messages are fetched.
+    /// Existing recipients are added to the contact database regardless of this setting.
     #[strum(props(default = "1"))]
-    FetchExisting,
+    FetchExistingMsgs,
 
     #[strum(props(default = "0"))]
     KeyGenType,
@@ -113,6 +117,9 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
+    ConfiguredSpamFolder,
+    ConfiguredTimestamp,
+    ConfiguredProvider,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -133,6 +140,13 @@ pub enum Config {
 
     /// address to webrtc instance to use for videochats
     WebrtcInstance,
+
+    /// Timestamp of the last time housekeeping was run
+    LastHousekeeping,
+
+    /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
+    #[strum(props(default = "60"))]
+    ScanAllFoldersDebounceSecs,
 }
 
 impl Context {
@@ -159,7 +173,7 @@ impl Context {
 
         // Default values
         match key {
-            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::Selfstatus => Some(stock_str::status_line(self).await),
             Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
@@ -172,6 +186,20 @@ impl Context {
             .unwrap_or_default()
     }
 
+    pub async fn get_config_i64(&self, key: Config) -> i64 {
+        self.get_config(key)
+            .await
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
+    }
+
+    pub async fn get_config_u64(&self, key: Config) -> u64 {
+        self.get_config(key)
+            .await
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
+    }
+
     pub async fn get_config_bool(&self, key: Config) -> bool {
         self.get_config_int(key).await != 0
     }
@@ -188,6 +216,14 @@ impl Context {
         }
     }
 
+    /// Gets the configured provider, as saved in the `configured_provider` value.
+    ///
+    /// The provider is determined by `get_provider_info()` during configuration and then saved
+    /// to the db in `param.save_to_database()`, together with all the other `configured_*` values.
+    pub async fn get_configured_provider(&self) -> Option<&'static Provider> {
+        get_provider_by_id(&self.get_config(Config::ConfiguredProvider).await?)
+    }
+
     /// Gets configured "delete_device_after" value.
     ///
     /// `None` means never delete the message, `Some(x)` means delete
@@ -212,8 +248,8 @@ impl Context {
                     .await?;
                 match value {
                     Some(value) => {
-                        let blob = BlobObject::new_from_path(&self, value).await?;
-                        blob.recode_to_avatar_size(self)?;
+                        let blob = BlobObject::new_from_path(self, value).await?;
+                        blob.recode_to_avatar_size(self).await?;
                         self.sql
                             .set_raw_config(self, key, Some(blob.as_name()))
                             .await
@@ -222,7 +258,7 @@ impl Context {
                 }
             }
             Config::Selfstatus => {
-                let def = self.stock_str(StockMessage::StatusLine).await;
+                let def = stock_str::status_line(self).await;
                 let val = if value.is_none() || value.unwrap() == def {
                     None
                 } else {
@@ -252,6 +288,11 @@ impl Context {
             _ => self.sql.set_raw_config(self, key, value).await,
         }
     }
+
+    pub async fn set_config_bool(&self, key: Config, value: bool) -> crate::sql::Result<()> {
+        self.set_config(key, if value { Some("1") } else { None })
+            .await
+    }
 }
 
 /// Returns all available configuration keys concated together.
@@ -273,8 +314,8 @@ mod tests {
     use std::string::ToString;
 
     use crate::constants;
-    use crate::constants::AVATAR_SIZE;
-    use crate::test_utils::*;
+    use crate::constants::BALANCED_AVATAR_SIZE;
+    use crate::test_utils::TestContext;
     use image::GenericImageView;
     use num_traits::FromPrimitive;
     use std::fs::File;
@@ -292,11 +333,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_default_prop() {
-        assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
-    }
-
     #[async_std::test]
     async fn test_selfavatar_outside_blobdir() {
         let t = TestContext::new().await;
@@ -306,15 +342,14 @@ mod tests {
             .unwrap()
             .write_all(avatar_bytes)
             .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
+        let avatar_blob = t.get_blobdir().join("avatar.jpg");
         assert!(!avatar_blob.exists().await);
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
         assert!(avatar_blob.exists().await);
         assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
@@ -322,14 +357,14 @@ mod tests {
         assert_eq!(img.height(), 1000);
 
         let img = image::open(avatar_blob).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
     }
 
     #[async_std::test]
     async fn test_selfavatar_in_blobdir() {
         let t = TestContext::new().await;
-        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_src = t.get_blobdir().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
             .unwrap()
@@ -340,16 +375,15 @@ mod tests {
         assert_eq!(img.width(), 900);
         assert_eq!(img.height(), 900);
 
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
     }
 
     #[async_std::test]
@@ -361,10 +395,9 @@ mod tests {
             .unwrap()
             .write_all(avatar_bytes)
             .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_blob = t.get_blobdir().join("avatar.png");
         assert!(!avatar_blob.exists().await);
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
         assert!(avatar_blob.exists().await);
@@ -372,24 +405,21 @@ mod tests {
             std::fs::metadata(&avatar_blob).unwrap().len(),
             avatar_bytes.len() as u64
         );
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
     }
 
     #[async_std::test]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        let media_quality = t.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
         assert_eq!(media_quality, constants::MediaQuality::Balanced);
 
-        t.ctx
-            .set_config(Config::MediaQuality, Some("1"))
-            .await
-            .unwrap();
+        t.set_config(Config::MediaQuality, Some("1")).await.unwrap();
 
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        let media_quality = t.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 1);
         assert_eq!(constants::MediaQuality::Worse as i32, 1);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure/auto_mozilla.rs

@@ -251,10 +251,10 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
 
 pub(crate) async fn moz_autoconfigure(
     context: &Context,
-    url: &str,
+    url: impl AsRef<str>,
     param_in: &LoginParam,
 ) -> Result<Vec<ServerParams>, Error> {
-    let xml_raw = read_url(context, url).await?;
+    let xml_raw = read_url(context, url.as_ref()).await?;
 
     let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {

src/configure/auto_outlook.rs

@@ -73,7 +73,7 @@ fn parse_protocol<B: BufRead>(
                 }
             }
             Event::Text(ref e) => {
-                let val = e.unescape_and_decode(&reader).unwrap_or_default();
+                let val = e.unescape_and_decode(reader).unwrap_or_default();
 
                 if let Some(ref tag) = current_tag {
                     match tag.as_str() {
@@ -117,7 +117,7 @@ fn parse_redirecturl<B: BufRead>(
     let mut buf = Vec::new();
     match reader.read_event(&mut buf)? {
         Event::Text(ref e) => {
-            let val = e.unescape_and_decode(&reader).unwrap_or_default();
+            let val = e.unescape_and_decode(reader).unwrap_or_default();
             Ok(val.trim().to_string())
         }
         _ => Ok("".to_string()),
@@ -154,7 +154,7 @@ fn parse_xml_reader<B: BufRead>(
 }
 
 fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
-    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
     parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
@@ -187,9 +187,8 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
 
 pub(crate) async fn outlk_autodiscover(
     context: &Context,
-    url: &str,
+    mut url: String,
 ) -> Result<Vec<ServerParams>, Error> {
-    let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
         let xml_raw = read_url(context, &url).await?;

src/configure/mod.rs

@@ -12,17 +12,20 @@ use itertools::Itertools;
 use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
-use crate::config::Config;
-use crate::dc_tools::*;
+use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
 use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
-use crate::oauth2::*;
+use crate::oauth2::dc_get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
-use crate::stock::StockMessage;
+use crate::stock_str;
 use crate::{chat, e2ee, provider};
-use crate::{constants::*, job};
+use crate::{config::Config, dc_tools::time};
+use crate::{
+    constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2},
+    job,
+};
 use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
@@ -85,7 +88,7 @@ impl Context {
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
-        if let Some(provider) = provider::get_provider_info(&param.addr) {
+        if let Some(provider) = param.provider {
             if let Some(config_defaults) = &provider.config_defaults {
                 for def in config_defaults.iter() {
                     if !self.config_exists(def.key).await {
@@ -124,9 +127,10 @@ impl Context {
                     self,
                     0,
                     Some(
-                        self.stock_string_repl_str(
-                            StockMessage::ConfigurationFailed,
-                            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                        stock_str::configuration_failed(
+                            self,
+                            // We are using Anyhow's .context() and to show the
+                            // inner error, too, we need the {:#}:
                             format!("{:#}", err),
                         )
                         .await
@@ -163,8 +167,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         DC_LP_AUTH_NORMAL as i32
     };
 
-    //let ctx2 = ctx.clone();
-    //let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
+    let ctx2 = ctx.clone();
+    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
 
     // Step 1: Load the parameters and check email-address and password
 
@@ -205,9 +209,51 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     {
         // no advanced parameters entered by the user: query provider-database or do Autoconfig
 
-        if let Some(servers) = get_offline_autoconfig(ctx, &param.addr) {
-            param_autoconfig = Some(servers);
+        info!(
+            ctx,
+            "checking internal provider-info for offline autoconfig"
+        );
+
+        if let Some(provider) = provider::get_provider_info(&param_domain).await {
+            param.provider = Some(provider);
+            match provider.status {
+                provider::Status::OK | provider::Status::PREPARATION => {
+                    if provider.server.is_empty() {
+                        info!(ctx, "offline autoconfig found, but no servers defined");
+                        param_autoconfig = None;
+                    } else {
+                        info!(ctx, "offline autoconfig found");
+                        let servers = provider
+                            .server
+                            .iter()
+                            .map(|s| ServerParams {
+                                protocol: s.protocol,
+                                socket: s.socket,
+                                hostname: s.hostname.to_string(),
+                                port: s.port,
+                                username: match s.username_pattern {
+                                    UsernamePattern::EMAIL => param.addr.to_string(),
+                                    UsernamePattern::EMAILLOCALPART => {
+                                        if let Some(at) = param.addr.find('@') {
+                                            param.addr.split_at(at).0.to_string()
+                                        } else {
+                                            param.addr.to_string()
+                                        }
+                                    }
+                                },
+                            })
+                            .collect();
+
+                        param_autoconfig = Some(servers)
+                    }
+                }
+                provider::Status::BROKEN => {
+                    info!(ctx, "offline autoconfig found, provider is broken");
+                    param_autoconfig = None;
+                }
+            }
         } else {
+            info!(ctx, "no offline autoconfig found");
             param_autoconfig =
                 get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
         }
@@ -257,6 +303,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         .filter(|params| params.protocol == Protocol::SMTP)
         .cloned()
         .collect();
+    let provider_strict_tls = param.provider.map_or(false, |provider| provider.strict_tls);
 
     let smtp_config_task = task::spawn(async move {
         let mut smtp_configured = false;
@@ -267,8 +314,15 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             smtp_param.port = smtp_server.port;
             smtp_param.security = smtp_server.socket;
 
-            match try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp)
-                .await
+            match try_smtp_one_param(
+                &context_smtp,
+                &smtp_param,
+                &smtp_addr,
+                oauth2,
+                provider_strict_tls,
+                &mut smtp,
+            )
+            .await
             {
                 Ok(_) => {
                     smtp_configured = true;
@@ -288,7 +342,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 600);
 
     // Configure IMAP
-    let (_s, r) = async_std::sync::channel(1);
+    let (_s, r) = async_std::channel::bounded(1);
     let mut imap = Imap::new(r);
 
     let mut imap_configured = false;
@@ -304,7 +358,16 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
 
-        match try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
+        match try_imap_one_param(
+            ctx,
+            &param.imap,
+            &param.addr,
+            oauth2,
+            provider_strict_tls,
+            &mut imap,
+        )
+        .await
+        {
             Ok(_) => {
                 imap_configured = true;
                 break;
@@ -351,6 +414,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     // the trailing underscore is correct
     param.save_to_database(ctx, "configured_").await?;
     ctx.sql.set_raw_config_bool(ctx, "configured", true).await?;
+    ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
+        .await?;
 
     progress!(ctx, 920);
 
@@ -364,71 +429,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     .await;
 
     progress!(ctx, 940);
-    //update_device_chats_handle.await?;
+    update_device_chats_handle.await?;
 
     Ok(())
 }
 
-#[derive(Debug, PartialEq, Eq)]
-enum AutoconfigProvider {
-    Mozilla,
-    Outlook,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-struct AutoconfigSource {
-    provider: AutoconfigProvider,
-    url: String,
-}
-
-impl AutoconfigSource {
-    fn all(domain: &str, addr: &str) -> [Self; 5] {
-        [
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!(
-                    "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-                    domain, addr,
-                ),
-            },
-            // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!(
-                    "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
-                    domain, addr
-                ),
-            },
-            AutoconfigSource {
-                provider: AutoconfigProvider::Outlook,
-                url: format!("https://{}/autodiscover/autodiscover.xml", domain),
-            },
-            // Outlook uses always SSL but different domains (this comment describes the next two steps)
-            AutoconfigSource {
-                provider: AutoconfigProvider::Outlook,
-                url: format!(
-                    "https://autodiscover.{}/autodiscover/autodiscover.xml",
-                    domain
-                ),
-            },
-            // always SSL for Thunderbird's database
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!("https://autoconfig.thunderbird.net/v1.1/{}", domain),
-            },
-        ]
-    }
-
-    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<Vec<ServerParams>> {
-        let params = match self.provider {
-            AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
-            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url).await?,
-        };
-
-        Ok(params)
-    }
-}
-
 /// Retrieve available autoconfigurations.
 ///
 /// A Search configurations from the domain used in the email-address, prefer encrypted
@@ -439,74 +444,79 @@ async fn get_autoconfig(
     param_domain: &str,
     param_addr_urlencoded: &str,
 ) -> Option<Vec<ServerParams>> {
-    let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        format!(
+            "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
+            param_domain, param_addr_urlencoded
+        ),
+        param,
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 300);
 
-    let mut progress = 300;
-    for source in &sources {
-        let res = source.fetch(ctx, param).await;
-        progress!(ctx, progress);
-        progress += 10;
-        if let Ok(res) = res {
-            return Some(res);
-        }
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
+        format!(
+            "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
+            &param_domain, &param_addr_urlencoded
+        ),
+        param,
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 310);
+
+    // Outlook uses always SSL but different domains (this comment describes the next two steps)
+    if let Ok(res) = outlk_autodiscover(
+        ctx,
+        format!("https://{}/autodiscover/autodiscover.xml", &param_domain),
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 320);
+
+    if let Ok(res) = outlk_autodiscover(
+        ctx,
+        format!(
+            "https://autodiscover.{}/autodiscover/autodiscover.xml",
+            &param_domain
+        ),
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 330);
+
+    // always SSL for Thunderbird's database
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
+        param,
+    )
+    .await
+    {
+        return Some(res);
     }
 
     None
 }
 
-fn get_offline_autoconfig(context: &Context, addr: &str) -> Option<Vec<ServerParams>> {
-    info!(
-        context,
-        "checking internal provider-info for offline autoconfig"
-    );
-
-    if let Some(provider) = provider::get_provider_info(&addr) {
-        match provider.status {
-            provider::Status::OK | provider::Status::PREPARATION => {
-                if provider.server.is_empty() {
-                    info!(context, "offline autoconfig found, but no servers defined");
-                    None
-                } else {
-                    info!(context, "offline autoconfig found");
-                    let servers = provider
-                        .server
-                        .iter()
-                        .map(|s| ServerParams {
-                            protocol: s.protocol,
-                            socket: s.socket,
-                            hostname: s.hostname.to_string(),
-                            port: s.port,
-                            username: match s.username_pattern {
-                                UsernamePattern::EMAIL => addr.to_string(),
-                                UsernamePattern::EMAILLOCALPART => {
-                                    if let Some(at) = addr.find('@') {
-                                        addr.split_at(at).0.to_string()
-                                    } else {
-                                        addr.to_string()
-                                    }
-                                }
-                            },
-                        })
-                        .collect();
-                    Some(servers)
-                }
-            }
-            provider::Status::BROKEN => {
-                info!(context, "offline autoconfig found, provider is broken");
-                None
-            }
-        }
-    } else {
-        info!(context, "no offline autoconfig found");
-        None
-    }
-}
-
 async fn try_imap_one_param(
     context: &Context,
     param: &ServerLoginParam,
     addr: &str,
     oauth2: bool,
+    provider_strict_tls: bool,
     imap: &mut Imap,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
@@ -515,7 +525,10 @@ async fn try_imap_one_param(
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = imap.connect(context, param, addr, oauth2).await {
+    if let Err(err) = imap
+        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .await
+    {
         info!(context, "failure: {}", err);
         Err(ConfigurationError {
             config: inf,
@@ -532,6 +545,7 @@ async fn try_smtp_one_param(
     param: &ServerLoginParam,
     addr: &str,
     oauth2: bool,
+    provider_strict_tls: bool,
     smtp: &mut Smtp,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
@@ -540,7 +554,10 @@ async fn try_smtp_one_param(
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
+    if let Err(err) = smtp
+        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .await
+    {
         info!(context, "failure: {}", err);
         Err(ConfigurationError {
             config: inf,
@@ -573,10 +590,7 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
         .iter()
         .all(|e| e.msg.to_lowercase().contains("could not resolve"))
     {
-        return context
-            .stock_str(StockMessage::ErrorNoNetwork)
-            .await
-            .to_string();
+        return stock_str::error_no_network(context).await;
     }
 
     if errors.iter().all(|e| e.msg == first_err.msg) {
@@ -609,37 +623,16 @@ pub enum Error {
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
-    use super::*;
-    use crate::config::*;
-    use crate::test_utils::*;
+    use crate::config::Config;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_no_panic_on_bad_credentials() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::Addr, Some("probably@unexistant.addr"))
+        t.set_config(Config::Addr, Some("probably@unexistant.addr"))
             .await
             .unwrap();
-        t.ctx
-            .set_config(Config::MailPw, Some("123456"))
-            .await
-            .unwrap();
-        assert!(t.ctx.configure().await.is_err());
-    }
-
-    #[async_std::test]
-    async fn test_get_offline_autoconfig() {
-        let context = TestContext::new().await.ctx;
-
-        let addr = "someone123@example.org";
-        assert!(get_offline_autoconfig(&context, addr).is_none());
-
-        let addr = "someone123@nauta.cu";
-        let found_params = get_offline_autoconfig(&context, addr).unwrap();
-        assert_eq!(found_params.len(), 2);
-        assert_eq!(found_params[0].protocol, Protocol::IMAP);
-        assert_eq!(found_params[0].hostname, "imap.nauta.cu".to_string());
-        assert_eq!(found_params[1].protocol, Protocol::SMTP);
-        assert_eq!(found_params[1].hostname, "smtp.nauta.cu".to_string());
+        t.set_config(Config::MailPw, Some("123456")).await.unwrap();
+        assert!(t.configure().await.is_err());
     }
 }

src/constants.rs

@@ -1,5 +1,5 @@
 //! # Constants
-use deltachat_derive::*;
+use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
@@ -32,7 +32,9 @@ impl Default for Blocked {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum ShowEmails {
     Off = 0,
@@ -46,7 +48,9 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum MediaQuality {
     Balanced = 0,
@@ -59,7 +63,9 @@ impl Default for MediaQuality {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum KeyGenType {
     Default = 0,
@@ -73,7 +79,9 @@ impl Default for KeyGenType {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(i8)]
 pub enum VideochatType {
     Unknown = 0,
@@ -99,9 +107,10 @@ pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
 pub const DC_GCL_FOR_FORWARDING: usize = 0x08;
 
 pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
+pub const DC_GCM_INFO_ONLY: u32 = 0x02;
 
-pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
-pub const DC_GCL_ADD_SELF: usize = 0x02;
+pub const DC_GCL_VERIFIED_ONLY: u32 = 0x01;
+pub const DC_GCL_ADD_SELF: u32 = 0x02;
 
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
@@ -143,6 +152,7 @@ pub enum Chattype {
     Undefined = 0,
     Single = 100,
     Group = 120,
+    Mailinglist = 140,
 }
 
 impl Default for Chattype {
@@ -155,8 +165,34 @@ pub const DC_MSG_ID_MARKER1: u32 = 1;
 pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
-/// approx. max. length returned by dc_msg_get_text()
-pub const DC_MAX_GET_TEXT_LEN: usize = 30000;
+/// string that indicates sth. is left out or truncated
+pub const DC_ELLIPSE: &str = "[...]";
+
+/// to keep bubbles and chat flow usable,
+/// and to avoid problems with controls using very long texts,
+/// we limit the text length to DC_DESIRED_TEXT_LEN.
+/// if the text is longer, the full text can be retrieved usind has_html()/get_html().
+///
+/// we are using a bit less than DC_MAX_GET_TEXT_LEN to avoid cutting twice
+/// (a bit less as truncation may not be exact and ellipses may be added).
+///
+/// note, that DC_DESIRED_TEXT_LEN and DC_MAX_GET_TEXT_LEN
+/// define max. number of bytes, _not_ unicode graphemes.
+/// in general, that seems to be okay for such an upper limit,
+/// esp. as calculating the number of graphemes is not simple
+/// (one graphemes may be a sequence of code points which is a sequence of bytes).
+/// also even if we have the exact number of graphemes,
+/// that would not always help on getting an idea about the screen space used
+/// (to keep bubbles and chat flow usable).
+///
+/// therefore, the number of bytes is only a very rough estimation,
+/// however, the ~30K seems to work okayish for a while,
+/// if it turns out, it is too few for some alphabet, we can still increase.
+pub const DC_DESIRED_TEXT_LEN: usize = 29_000;
+
+/// approx. max. length (number of bytes) returned by dc_msg_get_text()
+pub const DC_MAX_GET_TEXT_LEN: usize = 30_000;
+
 /// approx. max. length returned by dc_get_msg_info()
 pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
@@ -196,7 +232,8 @@ pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
 pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
 
 // max. width/height of an avatar
-pub const AVATAR_SIZE: u32 = 192;
+pub const BALANCED_AVATAR_SIZE: u32 = 256;
+pub const WORSE_AVATAR_SIZE: u32 = 128;
 
 // max. width/height of images
 pub const BALANCED_IMAGE_SIZE: u32 = 1280;
@@ -205,6 +242,11 @@ pub const WORSE_IMAGE_SIZE: u32 = 640;
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
+// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is splitted to chunks.
+// this does not affect MIME'e `To:` header.
+// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
+pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
+
 #[derive(
     Debug,
     Display,

src/context.rs

@@ -3,25 +3,28 @@
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
+use std::time::{Instant, SystemTime};
 
-use async_std::path::{Path, PathBuf};
-use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
-use async_std::task;
+use anyhow::{bail, ensure, Result};
+use async_std::{
+    channel::{self, Receiver, Sender},
+    path::{Path, PathBuf},
+    sync::{Arc, Mutex, RwLock},
+    task,
+};
 
-use crate::chat::*;
+use crate::chat::{get_chat_cnt, ChatId};
 use crate::config::Config;
-use crate::constants::*;
-use crate::contact::*;
-use crate::dc_tools::duration_to_str;
-use crate::error::*;
+use crate::constants::DC_VERSION_STR;
+use crate::contact::Contact;
+use crate::dc_tools::{duration_to_str, time};
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::message::{self, MsgId};
+use crate::message::{self, MessageState, MsgId};
 use crate::scheduler::Scheduler;
 use crate::securejoin::Bob;
 use crate::sql::Sql;
-use std::time::SystemTime;
 
 #[derive(Clone, Debug)]
 pub struct Context {
@@ -44,7 +47,7 @@ pub struct InnerContext {
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
     pub(crate) os_name: Option<String>,
-    pub(crate) bob: RwLock<Bob>,
+    pub(crate) bob: Bob,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -59,7 +62,12 @@ pub struct InnerContext {
     pub(crate) scheduler: RwLock<Scheduler>,
     pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
-    /// Id for this context on the current device.
+    pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
+
+    /// ID for this `Context` in the current process.
+    ///
+    /// This allows for multiple `Context`s open in a single process where each context can
+    /// be identified by this ID.
     pub(crate) id: u32,
 
     creation_time: SystemTime,
@@ -121,7 +129,7 @@ impl Context {
             os_name: Some(os_name),
             running_state: RwLock::new(Default::default()),
             sql: Sql::new(),
-            bob: RwLock::new(Default::default()),
+            bob: Default::default(),
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
@@ -131,6 +139,7 @@ impl Context {
             scheduler: RwLock::new(Scheduler::Stopped),
             ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
+            last_full_folder_scan: Mutex::new(None),
         };
 
         let ctx = Context {
@@ -144,7 +153,7 @@ impl Context {
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
         info!(self, "starting IO");
-        if self.is_io_running().await {
+        if self.inner.is_io_running().await {
             info!(self, "IO is already running");
             return;
         }
@@ -155,18 +164,9 @@ impl Context {
         }
     }
 
-    /// Returns if the IO scheduler is running.
-    pub async fn is_io_running(&self) -> bool {
-        self.inner.is_io_running().await
-    }
-
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
-        if !self.is_io_running().await {
-            info!(self, "IO is not running");
-            return;
-        }
 
         self.inner.stop_io().await;
     }
@@ -197,7 +197,10 @@ impl Context {
         });
     }
 
-    /// Get the next queued event.
+    /// Returns a receiver for emitted events.
+    ///
+    /// Multiple emitters can be created, but note that in this case each emitted event will
+    /// only be received by one of the emitters, not by all of them.
     pub fn get_event_emitter(&self) -> EventEmitter {
         self.events.get_emitter()
     }
@@ -219,7 +222,7 @@ impl Context {
 
         s.ongoing_running = true;
         s.shall_stop_ongoing = false;
-        let (sender, receiver) = channel(1);
+        let (sender, receiver) = channel::bounded(1);
         s.cancel_sender = Some(sender);
 
         Ok(receiver)
@@ -246,7 +249,9 @@ impl Context {
         let s_a = &self.running_state;
         let mut s = s_a.write().await;
         if let Some(cancel) = s.cancel_sender.take() {
-            cancel.send(()).await;
+            if let Err(err) = cancel.send(()).await {
+                warn!(self, "could not cancel ongoing: {:?}", err);
+            }
         }
 
         if s.ongoing_running && !s.shall_stop_ongoing {
@@ -307,6 +312,7 @@ impl Context {
         let sentbox_watch = self.get_config_int(Config::SentboxWatch).await;
         let mvbox_watch = self.get_config_int(Config::MvboxWatch).await;
         let mvbox_move = self.get_config_int(Config::MvboxMove).await;
+        let sentbox_move = self.get_config_int(Config::SentboxMove).await;
         let folders_configured = self
             .sql
             .get_raw_config_int(self, "folders_configured")
@@ -323,6 +329,9 @@ impl Context {
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
+
+        // insert values
+        res.insert("bot", self.get_config_int(Config::Bot).await.to_string());
         res.insert("number_of_chats", chats.to_string());
         res.insert("number_of_chat_messages", real_msgs.to_string());
         res.insert("messages_in_contact_requests", deaddrop_msgs.to_string());
@@ -341,15 +350,30 @@ impl Context {
         res.insert("is_configured", is_configured.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
+        res.insert(
+            "fetch_existing_msgs",
+            self.get_config_int(Config::FetchExistingMsgs)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "show_emails",
+            self.get_config_int(Config::ShowEmails).await.to_string(),
+        );
         res.insert("inbox_watch", inbox_watch.to_string());
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_watch", mvbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
+        res.insert("sentbox_move", sentbox_move.to_string());
         res.insert("folders_configured", folders_configured.to_string());
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
         res.insert("mdns_enabled", mdns_enabled.to_string());
         res.insert("e2ee_enabled", e2ee_enabled.to_string());
+        res.insert(
+            "key_gen_type",
+            self.get_config_int(Config::KeyGenType).await.to_string(),
+        );
         res.insert("bcc_self", bcc_self.to_string());
         res.insert(
             "private_key_count",
@@ -360,6 +384,40 @@ impl Context {
             pub_key_cnt.unwrap_or_default().to_string(),
         );
         res.insert("fingerprint", fingerprint_str);
+        res.insert(
+            "webrtc_instance",
+            self.get_config(Config::WebrtcInstance)
+                .await
+                .unwrap_or_else(|| "<unset>".to_string()),
+        );
+        res.insert(
+            "media_quality",
+            self.get_config_int(Config::MediaQuality).await.to_string(),
+        );
+        res.insert(
+            "delete_device_after",
+            self.get_config_int(Config::DeleteDeviceAfter)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "delete_server_after",
+            self.get_config_int(Config::DeleteServerAfter)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "last_housekeeping",
+            self.get_config_int(Config::LastHousekeeping)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "scan_all_folders_debounce_secs",
+            self.get_config_int(Config::ScanAllFoldersDebounceSecs)
+                .await
+                .to_string(),
+        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
@@ -367,9 +425,15 @@ impl Context {
         res
     }
 
-    pub async fn get_fresh_msgs(&self) -> Vec<MsgId> {
-        let show_deaddrop: i32 = 0;
-        self.sql
+    /// Get a list of fresh, unmuted messages in any chat but deaddrop.
+    ///
+    /// The list starts with the most recent message
+    /// and is typically used to show notifications.
+    /// Moreover, the number of returned messages
+    /// can be used for a badge counter on the app icon.
+    pub async fn get_fresh_msgs(&self) -> Result<Vec<MsgId>> {
+        let ret = self
+            .sql
             .query_map(
                 concat!(
                     "SELECT m.id",
@@ -380,12 +444,13 @@ impl Context {
                     "        ON m.chat_id=c.id",
                     " WHERE m.state=?",
                     "   AND m.hidden=0",
-                    "   AND m.chat_id>?",
+                    "   AND m.chat_id>9",
                     "   AND ct.blocked=0",
-                    "   AND (c.blocked=0 OR c.blocked=?)",
+                    "   AND c.blocked=0",
+                    "   AND NOT(c.muted_until=-1 OR c.muted_until>?)",
                     " ORDER BY m.timestamp DESC,m.id DESC;"
                 ),
-                paramsv![10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
+                paramsv![MessageState::InFresh, time()],
                 |row| row.get::<_, MsgId>(0),
                 |rows| {
                     let mut ret = Vec::new();
@@ -395,52 +460,26 @@ impl Context {
                     Ok(ret)
                 },
             )
-            .await
-            .unwrap_or_default()
+            .await?;
+        Ok(ret)
     }
 
-    #[allow(non_snake_case)]
-    pub async fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
+    /// Searches for messages containing the query string.
+    ///
+    /// If `chat_id` is provided this searches only for messages in this chat, if `chat_id`
+    /// is `None` this searches messages from all chats.
+    pub async fn search_msgs(&self, chat_id: Option<ChatId>, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
         }
-        let strLikeInText = format!("%{}%", real_query);
-        let strLikeBeg = format!("{}%", real_query);
+        let str_like_in_text = format!("%{}%", real_query);
+        let str_like_beg = format!("{}%", real_query);
 
-        let query = if !chat_id.is_unset() {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " WHERE m.chat_id=?",
-                "   AND m.hidden=0",
-                "   AND ct.blocked=0",
-                "   AND (txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp,m.id;"
-            )
-        } else {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " LEFT JOIN chats c",
-                "        ON m.chat_id=c.id",
-                " WHERE m.chat_id>9",
-                "   AND m.hidden=0",
-                "   AND (c.blocked=0 OR c.blocked=?)",
-                "   AND ct.blocked=0",
-                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp DESC,m.id DESC;"
-            )
-        };
-
-        self.sql
-            .query_map(
+        let do_query = |query, params| {
+            self.sql.query_map(
                 query,
-                paramsv![chat_id, strLikeInText, strLikeBeg],
+                params,
                 |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
@@ -450,8 +489,42 @@ impl Context {
                     Ok(ret)
                 },
             )
+        };
+
+        if let Some(chat_id) = chat_id {
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
+                 FROM msgs m
+                 LEFT JOIN contacts ct
+                        ON m.from_id=ct.id
+                 WHERE m.chat_id=?
+                   AND m.hidden=0
+                   AND ct.blocked=0
+                   AND (txt LIKE ? OR ct.name LIKE ?)
+                 ORDER BY m.timestamp,m.id;",
+                paramsv![chat_id, str_like_in_text, str_like_beg],
+            )
             .await
             .unwrap_or_default()
+        } else {
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
+                 FROM msgs m
+                 LEFT JOIN contacts ct
+                        ON m.from_id=ct.id
+                 LEFT JOIN chats c
+                        ON m.chat_id=c.id
+                 WHERE m.chat_id>9
+                   AND m.hidden=0
+                   AND c.blocked=0
+                   AND ct.blocked=0
+                   AND (m.txt LIKE ? OR ct.name LIKE ?)
+                 ORDER BY m.timestamp DESC,m.id DESC;",
+                paramsv![str_like_in_text, str_like_beg],
+            )
+            .await
+            .unwrap_or_default()
+        }
     }
 
     pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
@@ -469,6 +542,11 @@ impl Context {
             == Some(folder_name.as_ref().to_string())
     }
 
+    pub async fn is_spam_folder(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredSpamFolder).await
+            == Some(folder_name.as_ref().to_string())
+    }
+
     pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -483,14 +561,19 @@ impl InnerContext {
     }
 
     async fn stop_io(&self) {
-        assert!(self.is_io_running().await, "context is already stopped");
-        let token = {
-            let lock = &*self.scheduler.read().await;
-            lock.pre_stop().await
-        };
-        {
-            let lock = &mut *self.scheduler.write().await;
-            lock.stop(token).await;
+        if self.is_io_running().await {
+            let token = {
+                let lock = &*self.scheduler.read().await;
+                lock.pre_stop().await
+            };
+            {
+                let lock = &mut *self.scheduler.write().await;
+                lock.stop(token).await;
+            }
+        }
+
+        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
+            ephemeral_task.cancel().await;
         }
     }
 }
@@ -513,7 +596,12 @@ pub fn get_version_str() -> &'static str {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::chat::{get_chat_contacts, get_chat_msgs, set_muted, Chat, MuteDuration};
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::dc_tools::dc_create_outgoing_rfc724_mid;
+    use crate::test_utils::TestContext;
+    use std::time::Duration;
+    use strum::IntoEnumIterator;
 
     #[async_std::test]
     async fn test_wrong_db() {
@@ -527,10 +615,140 @@ mod tests {
     #[async_std::test]
     async fn test_get_fresh_msgs() {
         let t = TestContext::new().await;
-        let fresh = t.ctx.get_fresh_msgs().await;
+        let fresh = t.get_fresh_msgs().await.unwrap();
         assert!(fresh.is_empty())
     }
 
+    async fn receive_msg(t: &TestContext, chat: &Chat) {
+        let members = get_chat_contacts(t, chat.id).await;
+        let contact = Contact::load_from_db(t, *members.first().unwrap())
+            .await
+            .unwrap();
+        let msg = format!(
+            "From: {}\n\
+             To: alice@example.com\n\
+             Message-ID: <{}>\n\
+             Chat-Version: 1.0\n\
+             Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+             \n\
+             hello\n",
+            contact.get_addr(),
+            dc_create_outgoing_rfc724_mid(None, contact.get_addr())
+        );
+        println!("{}", msg);
+        dc_receive_imf(t, msg.as_bytes(), "INBOX", 1, false)
+            .await
+            .unwrap();
+    }
+
+    #[async_std::test]
+    async fn test_get_fresh_msgs_and_muted_chats() {
+        // receive various mails in 3 chats
+        let t = TestContext::new_alice().await;
+        let bob = t.create_chat_with_contact("", "bob@g.it").await;
+        let claire = t.create_chat_with_contact("", "claire@g.it").await;
+        let dave = t.create_chat_with_contact("", "dave@g.it").await;
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+        receive_msg(&t, &bob).await;
+        assert_eq!(get_chat_msgs(&t, bob.id, 0, None).await.len(), 1);
+        assert_eq!(bob.id.get_fresh_msg_cnt(&t).await, 1);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        receive_msg(&t, &claire).await;
+        receive_msg(&t, &claire).await;
+        assert_eq!(get_chat_msgs(&t, claire.id, 0, None).await.len(), 2);
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await, 2);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 3);
+
+        receive_msg(&t, &dave).await;
+        receive_msg(&t, &dave).await;
+        receive_msg(&t, &dave).await;
+        assert_eq!(get_chat_msgs(&t, dave.id, 0, None).await.len(), 3);
+        assert_eq!(dave.id.get_fresh_msg_cnt(&t).await, 3);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6);
+
+        // mute one of the chats
+        set_muted(&t, claire.id, MuteDuration::Forever)
+            .await
+            .unwrap();
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await, 2);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 4); // muted claires messages are no longer counted
+
+        // receive more messages
+        receive_msg(&t, &bob).await;
+        receive_msg(&t, &claire).await;
+        receive_msg(&t, &dave).await;
+        assert_eq!(get_chat_msgs(&t, claire.id, 0, None).await.len(), 3);
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await, 3);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6); // muted claire is not counted
+
+        // unmute claire again
+        set_muted(&t, claire.id, MuteDuration::NotMuted)
+            .await
+            .unwrap();
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await, 3);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 9); // claire is counted again
+    }
+
+    #[async_std::test]
+    async fn test_get_fresh_msgs_and_muted_until() {
+        let t = TestContext::new_alice().await;
+        let bob = t.create_chat_with_contact("", "bob@g.it").await;
+        receive_msg(&t, &bob).await;
+        assert_eq!(get_chat_msgs(&t, bob.id, 0, None).await.len(), 1);
+
+        // chat is unmuted by default, here and in the following assert(),
+        // we check mainly that the SQL-statements in is_muted() and get_fresh_msgs()
+        // have the same view to the database.
+        assert!(!bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        // test get_fresh_msgs() with mute_until in the future
+        set_muted(
+            &t,
+            bob.id,
+            MuteDuration::Until(SystemTime::now() + Duration::from_secs(3600)),
+        )
+        .await
+        .unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+        // to test get_fresh_msgs() with mute_until in the past,
+        // we need to modify the database directly
+        t.sql
+            .execute(
+                "UPDATE chats SET muted_until=? WHERE id=?;",
+                paramsv![time() - 3600, bob.id],
+            )
+            .await
+            .unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(!bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        // test get_fresh_msgs() with "forever" mute_until
+        set_muted(&t, bob.id, MuteDuration::Forever).await.unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+        // to test get_fresh_msgs() with invalid mute_until (everything < -1),
+        // that results in "muted forever" by definition.
+        t.sql
+            .execute(
+                "UPDATE chats SET muted_until=-2 WHERE id=?;",
+                paramsv![bob.id],
+            )
+            .await
+            .unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(!bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+    }
+
     #[async_std::test]
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
@@ -586,14 +804,14 @@ mod tests {
     #[async_std::test]
     async fn no_crashes_on_context_deref() {
         let t = TestContext::new().await;
-        std::mem::drop(t.ctx);
+        std::mem::drop(t);
     }
 
     #[async_std::test]
     async fn test_get_info() {
         let t = TestContext::new().await;
 
-        let info = t.ctx.get_info().await;
+        let info = t.get_info().await;
         assert!(info.get("database_dir").is_some());
     }
 
@@ -604,4 +822,47 @@ mod tests {
         assert!(info.get("database_dir").is_none());
         assert_eq!(info.get("level").unwrap(), "awesome");
     }
+
+    #[async_std::test]
+    async fn test_get_info_completeness() {
+        // For easier debugging,
+        // get_info() shall return all important information configurable by the Config-values.
+        //
+        // There are exceptions for Config-values considered to be unimportant,
+        // too sensitive or summarized in another item.
+        let skip_from_get_info = vec![
+            "addr",
+            "displayname",
+            "imap_certificate_checks",
+            "mail_server",
+            "mail_user",
+            "mail_pw",
+            "mail_port",
+            "mail_security",
+            "notify_about_wrong_pw",
+            "save_mime_headers",
+            "selfstatus",
+            "send_server",
+            "send_user",
+            "send_pw",
+            "send_port",
+            "send_security",
+            "server_flags",
+            "smtp_certificate_checks",
+        ];
+        let t = TestContext::new().await;
+        let info = t.get_info().await;
+        for key in Config::iter() {
+            let key: String = key.to_string();
+            if !skip_from_get_info.contains(&&*key)
+                && !key.starts_with("configured")
+                && !key.starts_with("sys.")
+            {
+                assert!(
+                    info.contains_key(&*key),
+                    format!("'{}' missing in get_info() output", key)
+                );
+            }
+        }
+    }
 }

src/dc_tools.rs

@@ -12,26 +12,24 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::{bail, Error};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
-use crate::constants::{Viewtype, DC_OUTDATED_WARNING_DAYS};
+use crate::constants::{Viewtype, DC_ELLIPSE, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
-use crate::error::{bail, Error};
 use crate::events::EventType;
 use crate::message::Message;
 use crate::provider::get_provider_update_timestamp;
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
 #[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
-    let ellipse = "[...]";
-
     let count = buf.chars().count();
-    if approx_chars > 0 && count > approx_chars + ellipse.len() {
+    if approx_chars > 0 && count > approx_chars + DC_ELLIPSE.len() {
         let end_pos = buf
             .char_indices()
             .nth(approx_chars)
@@ -39,40 +37,15 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
             .unwrap_or_default();
 
         if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
-            Cow::Owned(format!("{}{}", &buf[..=index], ellipse))
+            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSE))
         } else {
-            Cow::Owned(format!("{}{}", &buf[..end_pos], ellipse))
+            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSE))
         }
     } else {
         Cow::Borrowed(buf)
     }
 }
 
-/// the colors must fulfill some criterions as:
-/// - contrast to black and to white
-/// - work as a text-color
-/// - being noticeable on a typical map
-/// - harmonize together while being different enough
-/// (therefore, we cannot just use random rgb colors :)
-const COLORS: [u32; 16] = [
-    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
-    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
-];
-
-#[allow(clippy::indexing_slicing)]
-pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
-    let str_lower = s.as_ref().to_lowercase();
-    let mut checksum = 0;
-    let bytes = str_lower.as_bytes();
-    for (i, byte) in bytes.iter().enumerate() {
-        checksum += (i + 1) * *byte as usize;
-        checksum %= 0x00ff_ffff;
-    }
-    let color_index = checksum % COLORS.len();
-
-    COLORS[color_index]
-}
-
 /* ******************************************************************************
  * date/time tools
  ******************************************************************************/
@@ -169,15 +142,14 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
     if now < known_past_timestamp {
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some(
-            context
-                .stock_string_repl_str(
-                    StockMessage::BadTimeMsgBody,
-                    Local
-                        .timestamp(now, 0)
-                        .format("%Y-%m-%d %H:%M:%S")
-                        .to_string(),
-                )
-                .await,
+            stock_str::bad_time_msg_body(
+                context,
+                Local
+                    .timestamp(now, 0)
+                    .format("%Y-%m-%d %H:%M:%S")
+                    .to_string(),
+            )
+            .await,
         );
         add_device_msg_with_importance(
             context,
@@ -201,12 +173,7 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
 async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time: i64) {
     if now > approx_compile_time + DC_OUTDATED_WARNING_DAYS * 24 * 60 * 60 {
         let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(
-            context
-                .stock_str(StockMessage::UpdateReminderMsgBody)
-                .await
-                .into(),
-        );
+        msg.text = Some(stock_str::update_reminder_msg_body(context).await);
         add_device_msg(
             context,
             Some(
@@ -537,34 +504,15 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn get_next_backup_path_old(
-    folder: impl AsRef<Path>,
-    backup_time: i64,
-) -> Result<PathBuf, Error> {
-    let folder = PathBuf::from(folder.as_ref());
-    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
-        .format("delta-chat-%Y-%m-%d")
-        .to_string();
-
-    // 64 backup files per day should be enough for everyone
-    for i in 0..64 {
-        let mut path = folder.clone();
-        path.push(format!("{}-{}.bak", stem, i));
-        if !path.exists().await {
-            return Ok(path);
-        }
-    }
-    bail!("could not create backup file, disk full?");
-}
-
 /// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
 /// it can be renamed to dest_path. This guarantees that the backup is complete.
-pub(crate) async fn get_next_backup_path_new(
+pub(crate) async fn get_next_backup_path(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<(PathBuf, PathBuf), Error> {
     let folder = PathBuf::from(folder.as_ref());
     let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
         .format("delta-chat-backup-%Y-%m-%d")
         .to_string();
 
@@ -692,13 +640,6 @@ impl rusqlite::types::ToSql for EmailAddress {
     }
 }
 
-/// Utility to check if a in the binary represantion of listflags
-/// the bit at position bitindex is 1.
-pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
-    let listflags = listflags as usize;
-    (listflags & bitindex) == bitindex
-}
-
 /// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
 pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
     input
@@ -721,15 +662,33 @@ where
     }
 }
 
+pub fn remove_subject_prefix(last_subject: &str) -> String {
+    let subject_start = if last_subject.starts_with("Chat:") {
+        0
+    } else {
+        // "Antw:" is the longest abbreviation in
+        // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+        // so look at the first _5_ characters:
+        match last_subject.chars().take(5).position(|c| c == ':') {
+            Some(prefix_end) => prefix_end + 1,
+            None => 0,
+        }
+    };
+    last_subject
+        .chars()
+        .skip(subject_start)
+        .collect::<String>()
+        .trim()
+        .to_string()
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
-    use std::convert::TryInto;
 
-    use crate::constants::*;
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_rust_ftoa() {
@@ -918,7 +877,7 @@ mod tests {
     #[async_std::test]
     async fn test_file_handling() {
         let t = TestContext::new().await;
-        let context = &t.ctx;
+        let context = &t;
         macro_rules! dc_file_exist {
             ($ctx:expr, $fname:expr) => {
                 $ctx.get_blobdir()
@@ -980,29 +939,15 @@ mod tests {
         assert!(!dc_file_exist!(context, &fn0).await);
     }
 
-    #[test]
-    fn test_listflags_has() {
-        let listflags: u32 = 0x1101;
-        assert!(listflags_has(listflags, 0x1));
-        assert!(!listflags_has(listflags, 0x10));
-        assert!(listflags_has(listflags, 0x100));
-        assert!(listflags_has(listflags, 0x1000));
-        let listflags: u32 = (DC_GCL_ADD_SELF | DC_GCL_VERIFIED_ONLY).try_into().unwrap();
-        assert!(listflags_has(listflags, DC_GCL_VERIFIED_ONLY));
-        assert!(listflags_has(listflags, DC_GCL_ADD_SELF));
-        let listflags: u32 = DC_GCL_VERIFIED_ONLY.try_into().unwrap();
-        assert!(!listflags_has(listflags, DC_GCL_ADD_SELF));
-    }
-
     #[async_std::test]
     async fn test_create_smeared_timestamp() {
         let t = TestContext::new().await;
         assert_ne!(
-            dc_create_smeared_timestamp(&t.ctx).await,
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t).await,
+            dc_create_smeared_timestamp(&t).await
         );
         assert!(
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t).await
                 >= SystemTime::now()
                     .duration_since(SystemTime::UNIX_EPOCH)
                     .unwrap()
@@ -1014,13 +959,13 @@ mod tests {
     async fn test_create_smeared_timestamps() {
         let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t, count as usize).await;
+        let next = dc_smeared_time(&t).await;
         assert!((start + count - 1) < next);
 
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t, count as usize).await;
+        let next = dc_smeared_time(&t).await;
         assert!((start + count - 1) < next);
     }
 
@@ -1094,53 +1039,53 @@ mod tests {
             / 1_000;
 
         // a correct time must not add a device message
-        maybe_warn_on_bad_time(&t.ctx, timestamp_now, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t, timestamp_now, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // we cannot find out if a date in the future is wrong - a device message is not added
-        maybe_warn_on_bad_time(&t.ctx, timestamp_future, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t, timestamp_future, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // a date in the past must add a device message
-        maybe_warn_on_bad_time(&t.ctx, timestamp_past, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t, timestamp_past, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         // the message should be added only once a day - test that an hour later and nearly a day later
         maybe_warn_on_bad_time(
-            &t.ctx,
+            &t,
             timestamp_past + 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         maybe_warn_on_bad_time(
-            &t.ctx,
+            &t,
             timestamp_past + 60 * 60 * 24 - 1,
             get_provider_update_timestamp(),
         )
         .await;
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         // next day, there should be another device message
         maybe_warn_on_bad_time(
-            &t.ctx,
+            &t,
             timestamp_past + 60 * 60 * 24,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         assert_eq!(device_chat_id, chats.get_chat_id(0));
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 2);
     }
 
@@ -1152,51 +1097,60 @@ mod tests {
         // in about 6 months, the app should not be outdated
         // (if this fails, provider-db is not updated since 6 months)
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + 180 * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // in 1 year, the app should be considered as outdated
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + 365 * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         // do not repeat the warning every day ...
+        // (we test that for the 2 subsequent days, this may be the next month, so the result should be 1 or 2 device message)
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + (365 + 1) * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
-        assert_eq!(msgs.len(), 1);
-
-        // ... but every month
         maybe_warn_on_outdated(
-            &t.ctx,
-            timestamp_now + (365 + 31) * 24 * 60 * 60,
+            &t,
+            timestamp_now + (365 + 2) * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
-        assert_eq!(msgs.len(), 2);
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
+        let test_len = msgs.len();
+        assert!(test_len == 1 || test_len == 2);
+
+        // ... but every month
+        // (forward generous 33 days to avoid being in the same month as in the previous check)
+        maybe_warn_on_outdated(
+            &t,
+            timestamp_now + (365 + 33) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), test_len + 1);
     }
 }

src/dehtml.rs

@@ -2,8 +2,13 @@
 //!
 //! A module to remove HTML tags from the email text
 
+use std::io::BufRead;
+
 use once_cell::sync::Lazy;
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+use quick_xml::{
+    events::{BytesEnd, BytesStart, BytesText},
+    Reader,
+};
 
 static LINE_RE: Lazy<regex::Regex> = Lazy::new(|| regex::Regex::new(r"(\r?\n)+").unwrap());
 
@@ -11,9 +16,37 @@ struct Dehtml {
     strbuilder: String,
     add_text: AddText,
     last_href: Option<String>,
+    /// GMX wraps a quote in `<div name="quote">`. After a `<div name="quote">`, this count is
+    /// increased at each `<div>` and decreased at each `</div>`. This way we know when the quote ends.
+    /// If this is > `0`, then we are inside a `<div name="quote">`
+    divs_since_quote_div: u32,
+    /// Everything between <div name="quote"> and <div name="quoted-content"> is usually metadata
+    /// If this is > `0`, then we are inside a `<div name="quoted-content">`.
+    divs_since_quoted_content_div: u32,
 }
 
-#[derive(Debug, PartialEq)]
+impl Dehtml {
+    fn line_prefix(&self) -> &str {
+        if self.divs_since_quoted_content_div > 0 {
+            "> "
+        } else {
+            ""
+        }
+    }
+    fn append_prefix(&self, line_end: impl AsRef<str>) -> String {
+        // line_end is e.g. "\n\n". We add "> " if necessary.
+        line_end.as_ref().to_owned() + self.line_prefix()
+    }
+    fn get_add_text(&self) -> AddText {
+        if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {
+            AddText::No // Everything between <div name="quoted"> and <div name="quoted_content"> is metadata which we don't want
+        } else {
+            self.add_text
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
 enum AddText {
     No,
     YesRemoveLineEnds,
@@ -41,6 +74,8 @@ pub fn dehtml_quick_xml(buf: &str) -> String {
         strbuilder: String::with_capacity(buf.len()),
         add_text: AddText::YesRemoveLineEnds,
         last_href: None,
+        divs_since_quote_div: 0,
+        divs_since_quoted_content_div: 0,
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
@@ -79,13 +114,16 @@ pub fn dehtml_quick_xml(buf: &str) -> String {
 }
 
 fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.add_text == AddText::YesPreserveLineEnds
-        || dehtml.add_text == AddText::YesRemoveLineEnds
+    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
+        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.add_text == AddText::YesRemoveLineEnds {
+        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
+        } else if !dehtml.line_prefix().is_empty() {
+            let l = dehtml.append_prefix("\n");
+            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -93,13 +131,16 @@ fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
 }
 
 fn dehtml_cdata_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.add_text == AddText::YesPreserveLineEnds
-        || dehtml.add_text == AddText::YesRemoveLineEnds
+    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
+        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.add_text == AddText::YesRemoveLineEnds {
+        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
+        } else if !dehtml.line_prefix().is_empty() {
+            let l = dehtml.append_prefix("\n");
+            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -110,8 +151,15 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "div" | "table" | "td" | "style" | "script" | "title" | "pre" => {
-            dehtml.strbuilder += "\n\n";
+        "p" | "table" | "td" | "style" | "script" | "title" | "pre" => {
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+            dehtml.add_text = AddText::YesRemoveLineEnds;
+        }
+        "div" => {
+            pop_tag(&mut dehtml.divs_since_quote_div);
+            pop_tag(&mut dehtml.divs_since_quoted_content_div);
+
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "a" => {
@@ -122,10 +170,14 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
             }
         }
         "b" | "strong" => {
-            dehtml.strbuilder += "*";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "*";
+            }
         }
         "i" | "em" => {
-            dehtml.strbuilder += "_";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "_";
+            }
         }
         _ => {}
     }
@@ -139,19 +191,27 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "div" | "table" | "td" => {
-            dehtml.strbuilder += "\n\n";
+        "p" | "table" | "td" => {
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+            dehtml.add_text = AddText::YesRemoveLineEnds;
+        }
+        #[rustfmt::skip]
+        "div" => {
+            maybe_push_tag(event, reader, "quote", &mut dehtml.divs_since_quote_div);
+            maybe_push_tag(event, reader, "quoted-content", &mut dehtml.divs_since_quoted_content_div);
+
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "br" => {
-            dehtml.strbuilder += "\n";
+            dehtml.strbuilder += &dehtml.append_prefix("\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "style" | "script" | "title" => {
             dehtml.add_text = AddText::No;
         }
         "pre" => {
-            dehtml.strbuilder += "\n\n";
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
@@ -172,15 +232,51 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             }
         }
         "b" | "strong" => {
-            dehtml.strbuilder += "*";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "*";
+            }
         }
         "i" | "em" => {
-            dehtml.strbuilder += "_";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "_";
+            }
         }
         _ => {}
     }
 }
 
+/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
+/// The `counts`s are stored in the `Dehtml` struct.
+fn pop_tag(count: &mut u32) {
+    if *count > 0 {
+        *count -= 1;
+    }
+}
+
+/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
+/// The `counts`s are stored in the `Dehtml` struct.
+fn maybe_push_tag(
+    event: &BytesStart,
+    reader: &Reader<impl BufRead>,
+    tag_name: &str,
+    count: &mut u32,
+) {
+    if *count > 0 || tag_contains_attr(event, reader, tag_name) {
+        *count += 1;
+    }
+}
+
+fn tag_contains_attr(event: &BytesStart, reader: &Reader<impl BufRead>, name: &str) -> bool {
+    event.attributes().any(|r| {
+        r.map(|a| {
+            a.unescape_and_decode_value(reader)
+                .map(|v| v == name)
+                .unwrap_or(false)
+        })
+        .unwrap_or(false)
+    })
+}
+
 pub fn dehtml_manually(buf: &str) -> String {
     // Just strip out everything between "<" and ">"
     let mut strbuilder = String::new();
@@ -288,4 +384,17 @@ mod tests {
         let txt = dehtml(input).unwrap();
         assert_eq!(txt.trim(), "lots of text");
     }
+
+    #[async_std::test]
+    async fn test_quote_div() {
+        let input = include_str!("../test-data/message/gmx-quote-body.eml");
+        let dehtml = dehtml(input).unwrap();
+        println!("{}", dehtml);
+        let (msg, forwarded, cut, top_quote, footer) = simplify(dehtml, false);
+        assert_eq!(msg, "Test");
+        assert_eq!(forwarded, false);
+        assert_eq!(cut, false);
+        assert_eq!(top_quote.as_deref(), Some("test"));
+        assert_eq!(footer, None);
+    }
 }

src/e2ee.rs

@@ -2,18 +2,18 @@
 
 use std::collections::HashSet;
 
+use anyhow::{bail, ensure, format_err, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
-use crate::aheader::*;
+use crate::aheader::{Aheader, EncryptPreference};
 use crate::config::Config;
 use crate::context::Context;
-use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
-use crate::keyring::*;
-use crate::peerstate::*;
+use crate::keyring::Keyring;
+use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::pgp;
 
 #[derive(Debug)]
@@ -113,7 +113,7 @@ impl EncryptHelper {
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: Vec<(Option<Peerstate<'_>>, &str)>,
+        peerstates: Vec<(Option<Peerstate>, &str)>,
     ) -> Result<String> {
         let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
 
@@ -153,7 +153,7 @@ pub async fn try_decrypt(
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
+        .and_then(|from_addr| mailparse::addrparse_header(from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -163,10 +163,10 @@ pub async fn try_decrypt(
     // Apply Autocrypt header
     if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
         if let Some(ref mut peerstate) = peerstate {
-            peerstate.apply_header(&header, message_time);
+            peerstate.apply_header(header, message_time);
             peerstate.save_to_db(&context.sql, false).await?;
         } else {
-            let p = Peerstate::from_header(context, header, message_time);
+            let p = Peerstate::from_header(header, message_time);
             p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
@@ -346,10 +346,10 @@ mod tests {
 
     use crate::chat;
     use crate::constants::Viewtype;
-    use crate::contact::{Contact, Origin};
     use crate::message::Message;
     use crate::param::Param;
-    use crate::test_utils::*;
+    use crate::peerstate::ToSave;
+    use crate::test_utils::{bob_keypair, TestContext};
 
     mod ensure_secret_key_exists {
         use super::*;
@@ -358,13 +358,13 @@ mod tests {
         async fn test_prexisting() {
             let t = TestContext::new().await;
             let test_addr = t.configure_alice().await;
-            assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
+            assert_eq!(ensure_secret_key_exists(&t).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
         async fn test_not_configured() {
             let t = TestContext::new().await;
-            assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
+            assert!(ensure_secret_key_exists(&t).await.is_err());
         }
     }
 
@@ -411,27 +411,12 @@ Sent with my Delta Chat Messenger: https://delta.chat";
     }
 
     #[async_std::test]
-    async fn test_encrypted_no_autocrypt() -> crate::error::Result<()> {
+    async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
-        let (contact_alice_id, _modified) = Contact::add_or_lookup(
-            &bob.ctx,
-            "Alice",
-            "alice@example.com",
-            Origin::ManuallyCreated,
-        )
-        .await?;
-        let (contact_bob_id, _modified) = Contact::add_or_lookup(
-            &alice.ctx,
-            "Bob",
-            "bob@example.net",
-            Origin::ManuallyCreated,
-        )
-        .await?;
-
-        let chat_alice = chat::create_by_contact_id(&alice.ctx, contact_bob_id).await?;
-        let chat_bob = chat::create_by_contact_id(&bob.ctx, contact_alice_id).await?;
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
 
         // Alice sends unencrypted message to Bob
         let mut msg = Message::new(Viewtype::Text);
@@ -512,14 +497,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         Ok(())
     }
 
-    fn new_peerstates(
-        ctx: &Context,
-        prefer_encrypt: EncryptPreference,
-    ) -> Vec<(Option<Peerstate<'_>>, &str)> {
+    fn new_peerstates(prefer_encrypt: EncryptPreference) -> Vec<(Option<Peerstate>, &'static str)> {
         let addr = "bob@foo.bar";
         let pub_key = bob_keypair().public;
         let peerstate = Peerstate {
-            context: &ctx,
             addr: addr.into(),
             last_seen: 13,
             last_seen_autocrypt: 14,
@@ -542,28 +523,28 @@ Sent with my Delta Chat Messenger: https://delta.chat";
     #[async_std::test]
     async fn test_should_encrypt() {
         let t = TestContext::new_alice().await;
-        let encrypt_helper = EncryptHelper::new(&t.ctx).await.unwrap();
+        let encrypt_helper = EncryptHelper::new(&t).await.unwrap();
 
         // test with EncryptPreference::NoPreference:
         // if e2ee_eguaranteed is unset, there is no encryption as not more than half of peers want encryption
-        let ps = new_peerstates(&t.ctx, EncryptPreference::NoPreference);
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = new_peerstates(EncryptPreference::NoPreference);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with EncryptPreference::Reset
-        let ps = new_peerstates(&t.ctx, EncryptPreference::Reset);
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = new_peerstates(EncryptPreference::Reset);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with EncryptPreference::Mutual (self is also Mutual)
-        let ps = new_peerstates(&t.ctx, EncryptPreference::Mutual);
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
-        assert!(encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = new_peerstates(EncryptPreference::Mutual);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with missing peerstate
         let mut ps = Vec::new();
         ps.push((None, "bob@foo.bar"));
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).is_err());
-        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).is_err());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
     }
 }

src/ephemeral.rs

@@ -56,24 +56,26 @@
 //! the database entries which are expired either according to their
 //! ephemeral message timers or global `delete_server_after` setting.
 
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use anyhow::{ensure, Error};
+use async_std::task;
+use serde::{Deserialize, Serialize};
+
 use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
 use crate::constants::{
     Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
 };
 use crate::context::Context;
 use crate::dc_tools::time;
-use crate::error::{ensure, Error};
 use crate::events::EventType;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::sql;
-use crate::stock::StockMessage;
-use async_std::task;
-use serde::{Deserialize, Serialize};
-use std::convert::{TryFrom, TryInto};
-use std::num::ParseIntError;
-use std::str::FromStr;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use crate::stock_str;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -211,21 +213,50 @@ pub(crate) async fn stock_ephemeral_timer_changed(
     timer: Timer,
     from_id: u32,
 ) -> String {
-    let stock_message = match timer {
-        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+    match timer {
+        Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
         Timer::Enabled { duration } => match duration {
-            60 => StockMessage::MsgEphemeralTimerMinute,
-            3600 => StockMessage::MsgEphemeralTimerHour,
-            86400 => StockMessage::MsgEphemeralTimerDay,
-            604_800 => StockMessage::MsgEphemeralTimerWeek,
-            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
-            _ => StockMessage::MsgEphemeralTimerEnabled,
+            0..=59 => {
+                stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id).await
+            }
+            60 => stock_str::msg_ephemeral_timer_minute(context, from_id).await,
+            61..=3599 => {
+                stock_str::msg_ephemeral_timer_minutes(
+                    context,
+                    format!("{}", (f64::from(duration) / 6.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            3600 => stock_str::msg_ephemeral_timer_hour(context, from_id).await,
+            3601..=86399 => {
+                stock_str::msg_ephemeral_timer_hours(
+                    context,
+                    format!("{}", (f64::from(duration) / 360.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            86400 => stock_str::msg_ephemeral_timer_day(context, from_id).await,
+            86401..=604_799 => {
+                stock_str::msg_ephemeral_timer_days(
+                    context,
+                    format!("{}", (f64::from(duration) / 8640.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            604_800 => stock_str::msg_ephemeral_timer_week(context, from_id).await,
+            _ => {
+                stock_str::msg_ephemeral_timer_weeks(
+                    context,
+                    format!("{}", (f64::from(duration) / 60480.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
         },
-    };
-
-    context
-        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
-        .await
+    }
 }
 
 impl MsgId {
@@ -277,11 +308,13 @@ pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, E
     let mut updated = context
         .sql
         .execute(
+            // If you change which information is removed here, also change MsgId::trash() and
+            // which information dc_receive_imf::add_parts() still adds to the db if the chat_id is TRASH
             "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
+             SET chat_id=?, txt='', subject='', txt_raw='', mime_headers='', from_id=0, to_id=0, param='' \
              WHERE \
              ephemeral_timestamp != 0 \
-             AND ephemeral_timestamp < ? \
+             AND ephemeral_timestamp <= ? \
              AND chat_id != ?",
             paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
         )
@@ -417,7 +450,7 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
             "SELECT id FROM msgs \
          WHERE ( \
          timestamp < ? \
-         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
          LIMIT 1",
@@ -459,12 +492,18 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()>
 
 #[cfg(test)]
 mod tests {
+    use async_std::task::sleep;
+
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
+    use crate::{
+        chat::{self, Chat, ChatItem},
+        dc_tools::IsNoneOrEmpty,
+    };
 
     #[async_std::test]
     async fn test_stock_ephemeral_messages() {
-        let context = TestContext::new().await.ctx;
+        let context = TestContext::new().await;
 
         assert_eq!(
             stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
@@ -472,24 +511,78 @@ mod tests {
         );
 
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
-            "Message deletion timer is disabled."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 1 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 s by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
-            "Message deletion timer is set to 1 s."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 30 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 30 s by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
-            "Message deletion timer is set to 30 s."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 minute by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
-            "Message deletion timer is set to 1 minute."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 90 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1.5 minutes by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
-            "Message deletion timer is set to 1 hour."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 30 * 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 30 minutes by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 60 * 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 hour by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 5400 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1.5 hours by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 2 * 60 * 60
+                },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 2 hours by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -497,10 +590,21 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 1 day."
+            "Message deletion timer is set to 1 day by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 2 * 24 * 60 * 60
+                },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 2 days by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -508,10 +612,10 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 1 week."
+            "Message deletion timer is set to 1 week by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -519,10 +623,119 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 4 weeks."
+            "Message deletion timer is set to 4 weeks by me."
         );
     }
+
+    #[async_std::test]
+    async fn test_ephemeral_timer() -> anyhow::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
+
+        // Alice sends message to Bob
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+
+        // Alice sends second message to Bob, with no timer
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Disabled
+        );
+
+        // Bob sets ephemeral timer and sends a message about timer change
+        chat_bob
+            .set_ephemeral_timer(&bob.ctx, Timer::Enabled { duration: 60 })
+            .await?;
+        let sent_timer_change = bob.pop_sent_msg().await;
+
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        // Bob receives message from Alice.
+        // Alice message has no timer. However, Bob should not disable timer,
+        // because Alice replies to old message.
+        bob.recv_msg(&sent).await;
+
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Disabled
+        );
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        // Alice receives message from Bob
+        alice.recv_msg(&sent_timer_change).await;
+
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_ephemeral_delete_msgs() {
+        let t = TestContext::new_alice().await;
+        let chat = t.get_self_chat().await;
+
+        t.send_text(chat.id, "Saved message, which we delete manually")
+            .await;
+        let msg = t.get_last_msg_in(chat.id).await;
+        msg.id.delete_from_db(&t).await.unwrap();
+        check_msg_was_deleted(&t, &chat, msg.id).await;
+
+        chat.id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
+            .await
+            .unwrap();
+        let msg = t
+            .send_text(chat.id, "Saved message, disappearing after 1s")
+            .await;
+
+        sleep(Duration::from_millis(1100)).await;
+
+        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
+    }
+
+    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+        let chat_items = chat::get_chat_msgs(t, chat.id, 0, None).await;
+        // Check that the chat is empty except for possibly info messages:
+        for item in &chat_items {
+            if let ChatItem::Message { msg_id } = item {
+                let msg = Message::load_from_db(t, *msg_id).await.unwrap();
+                assert!(msg.is_info())
+            }
+        }
+
+        // Check that if there is a message left, the text and metadata are gone
+        if let Ok(msg) = Message::load_from_db(t, msg_id).await {
+            assert_eq!(msg.from_id, 0);
+            assert_eq!(msg.to_id, 0);
+            assert!(msg.text.is_none_or_empty(), msg.text);
+            let rawtxt: Option<String> = t
+                .sql
+                .query_get_value(t, "SELECT txt_raw FROM msgs WHERE id=?;", paramsv![msg_id])
+                .await;
+            assert!(rawtxt.is_none_or_empty(), rawtxt);
+        }
+    }
 }

src/error.rs

@@ -1,7 +1,5 @@
 //! # Error handling
 
-pub use anyhow::{bail, ensure, format_err, Error, Result};
-
 #[macro_export]
 macro_rules! ensure_eq {
     ($left:expr, $right:expr) => ({

src/events.rs

@@ -2,8 +2,8 @@
 
 use std::ops::Deref;
 
+use async_std::channel::{self, Receiver, Sender, TrySendError};
 use async_std::path::PathBuf;
-use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
@@ -18,7 +18,7 @@ pub struct Events {
 
 impl Default for Events {
     fn default() -> Self {
-        let (sender, receiver) = channel(1_000);
+        let (sender, receiver) = channel::bounded(1_000);
 
         Self { receiver, sender }
     }
@@ -35,7 +35,7 @@ impl Events {
                 // try again
                 self.emit(event);
             }
-            Err(TrySendError::Disconnected(_)) => {
+            Err(TrySendError::Closed(_)) => {
                 unreachable!("unable to emit event, channel disconnected");
             }
         }
@@ -47,6 +47,17 @@ impl Events {
     }
 }
 
+/// A receiver of events from a [`Context`].
+///
+/// See [`Context::get_event_emitter`] to create an instance.  If multiple instances are
+/// created events emitted by the [`Context`] will only be delivered to one of the
+/// `EventEmitter`s.
+///
+/// The `EventEmitter` is also a [`Stream`], so a typical usage is in a `while let` loop.
+///
+/// [`Context`]: crate::context::Context
+/// [`Context::get_event_emitter`]: crate::context::Context::get_event_emitter
+/// [`Stream`]: async_std::stream::Stream
 #[derive(Debug, Clone)]
 pub struct EventEmitter(Receiver<Event>);
 
@@ -73,9 +84,27 @@ impl async_std::stream::Stream for EventEmitter {
     }
 }
 
+/// The event emitted by a [`Context`] from an [`EventEmitter`].
+///
+/// Events are documented on the C/FFI API in `deltachat.h` as `DC_EVENT_*` contants.  The
+/// context emits them in relation to various operations happening, a lot of these are again
+/// documented in `deltachat.h`.
+///
+/// This struct [`Deref`]s to the [`EventType`].
+///
+/// [`Context`]: crate::context::Context
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Event {
+    /// The ID of the [`Context`] which emitted this event.
+    ///
+    /// This allows using multiple [`Context`]s in a single process as they are identified
+    /// by this ID.
+    ///
+    /// [`Context`]: crate::context::Context
     pub id: u32,
+    /// The event payload.
+    ///
+    /// These are documented in `deltachat.h` as the `DC_EVENT_*` constants.
     pub typ: EventType,
 }
 
@@ -88,7 +117,9 @@ impl Deref for Event {
 }
 
 impl EventType {
-    /// Returns the corresponding Event id.
+    /// Returns the corresponding Event ID.
+    ///
+    /// These are the IDs used in the `DC_EVENT_*` constants in `deltachat.h`.
     pub fn as_id(&self) -> i32 {
         self.get_str("id")
             .expect("missing id")
@@ -100,8 +131,9 @@ impl EventType {
 #[derive(Debug, Clone, PartialEq, Eq, EnumProperty)]
 pub enum EventType {
     /// The library-user may write an informational string to the log.
-    /// Passed to the callback given to dc_context_new().
-    /// This event should not be reported to the end-user using a popup or something like that.
+    ///
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
     #[strum(props(id = "100"))]
     Info(String),
 
@@ -134,14 +166,13 @@ pub enum EventType {
     DeletedBlobFile(String),
 
     /// The library-user should write a warning string to the log.
-    /// Passed to the callback given to dc_context_new().
     ///
-    /// This event should not be reported to the end-user using a popup or something like that.
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
     #[strum(props(id = "300"))]
     Warning(String),
 
     /// The library-user should report an error to the end-user.
-    /// Passed to the callback given to dc_context_new().
     ///
     /// As most things are asynchronous, things may go wrong at any time and the user
     /// should not be disturbed by a dialog or so.  Instead, use a bubble or so.

src/headerdef.rs

@@ -11,16 +11,26 @@ pub enum HeaderDef {
     To,
     Cc,
     Disposition,
+
+    /// Used in the "Body Part Header" of MDNs as of RFC 8098.
+    /// Indicates the Message-ID of the message for which the MDN is being issued.
     OriginalMessageId,
 
     /// Delta Chat extension for message IDs in combined MDNs
     AdditionalMessageIds,
 
+    /// Outlook-SMTP-server replace the `Message-ID:`-header
+    /// and write the original ID to `X-Microsoft-Original-Message-ID`.
+    /// To sort things correctly and to not show outgoing messages twice,
+    /// we need to check that header as well.
+    XMicrosoftOriginalMessageId,
+
     ListId,
     References,
     InReplyTo,
     Precedence,
     ContentType,
+    ContentId,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -42,7 +52,9 @@ pub enum HeaderDef {
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
     SecureJoinAuth,
+    Sender,
     EphemeralTimer,
+    Received,
     _TestHeader,
 }
 

src/html.rs

@@ -0,0 +1,565 @@
+///! # Get message as HTML.
+///!
+///! Use `Message.has_html()` to check if the UI shall render a
+///! corresponding button and `MsgId.get_html()` to get the full message.
+///!
+///! Even when the original mime-message is not HTML,
+///! `MsgId.get_html()` will return HTML -
+///! this allows nice quoting, handling linebreaks properly etc.
+use futures::future::FutureExt;
+use std::future::Future;
+use std::pin::Pin;
+
+use anyhow::Result;
+use lettre_email::mime::{self, Mime};
+
+use crate::context::Context;
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::message::{Message, MsgId};
+use crate::mimeparser::parse_message_id;
+use crate::param::Param::SendHtml;
+use crate::plaintext::PlainText;
+use lettre_email::PartBuilder;
+use mailparse::ParsedContentType;
+
+impl Message {
+    /// Check if the message can be retrieved as HTML.
+    /// Typically, this is the case, when the mime structure of a Message is modified,
+    /// meaning that some text is cut or the original message
+    /// is in HTML and `simplify()` may hide some maybe important information.
+    /// The corresponding ffi-function is `dc_msg_has_html()`.
+    /// To get the HTML-code of the message, use `MsgId.get_html()`.
+    pub fn has_html(&self) -> bool {
+        self.mime_modified
+    }
+
+    /// Set HTML-part part of a message that is about to be sent.
+    /// The HTML-part is written to the database before sending and
+    /// used as the `text/html` part in the MIME-structure.
+    ///
+    /// Received HTML parts are handled differently,
+    /// they are saved together with the whole MIME-structure
+    /// in `mime_headers` and the HTML-part is extracted using `MsgId::get_html()`.
+    /// (To underline this asynchronicity, we are using the wording "SendHtml")
+    pub fn set_html(&mut self, html: Option<String>) {
+        if let Some(html) = html {
+            self.param.set(SendHtml, html);
+            self.mime_modified = true;
+        } else {
+            self.param.remove(SendHtml);
+            self.mime_modified = false;
+        }
+    }
+}
+
+/// Type defining a rough mime-type.
+/// This is mainly useful on iterating
+/// to decide whether a mime-part has subtypes.
+enum MimeMultipartType {
+    Multiple,
+    Single,
+    Message,
+}
+
+/// Function takes a content type from a ParsedMail structure
+/// and checks and returns the rough mime-type.
+async fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
+    let mimetype = ctype.mimetype.to_lowercase();
+    if mimetype.starts_with("multipart") && ctype.params.get("boundary").is_some() {
+        MimeMultipartType::Multiple
+    } else if mimetype == "message/rfc822" {
+        MimeMultipartType::Message
+    } else {
+        MimeMultipartType::Single
+    }
+}
+
+/// HtmlMsgParser converts a mime-message to HTML.
+#[derive(Debug)]
+struct HtmlMsgParser {
+    pub html: String,
+    pub plain: Option<PlainText>,
+}
+
+impl HtmlMsgParser {
+    /// Function takes a raw mime-message string,
+    /// searches for the main-text part
+    /// and returns that as parser.html
+    pub async fn from_bytes(context: &Context, rawmime: &[u8]) -> Result<Self> {
+        let mut parser = HtmlMsgParser {
+            html: "".to_string(),
+            plain: None,
+        };
+
+        let parsedmail = mailparse::parse_mail(rawmime)?;
+
+        parser.collect_texts_recursive(context, &parsedmail).await?;
+
+        if parser.html.is_empty() {
+            if let Some(plain) = &parser.plain {
+                parser.html = plain.to_html().await;
+            }
+        } else {
+            parser.cid_to_data_recursive(context, &parsedmail).await?;
+        }
+
+        Ok(parser)
+    }
+
+    /// Function iterates over all mime-parts
+    /// and searches for text/plain and text/html parts and saves the
+    /// first one found.
+    /// in the corresponding structure fields.
+    ///
+    /// Usually, there is at most one plain-text and one HTML-text part,
+    /// multiple plain-text parts might be used for mailinglist-footers,
+    /// therefore we use the first one.
+    fn collect_texts_recursive<'a>(
+        &'a mut self,
+        context: &'a Context,
+        mail: &'a mailparse::ParsedMail<'a>,
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
+        // Boxed future to deal with recursion
+        async move {
+            match get_mime_multipart_type(&mail.ctype).await {
+                MimeMultipartType::Multiple => {
+                    for cur_data in mail.subparts.iter() {
+                        self.collect_texts_recursive(context, cur_data).await?
+                    }
+                    Ok(())
+                }
+                MimeMultipartType::Message => {
+                    let raw = mail.get_body_raw()?;
+                    if raw.is_empty() {
+                        return Ok(());
+                    }
+                    let mail = mailparse::parse_mail(&raw).unwrap();
+                    self.collect_texts_recursive(context, &mail).await
+                }
+                MimeMultipartType::Single => {
+                    let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+                    if mimetype == mime::TEXT_HTML {
+                        if self.html.is_empty() {
+                            if let Ok(decoded_data) = mail.get_body() {
+                                self.html = decoded_data;
+                            }
+                        }
+                    } else if mimetype == mime::TEXT_PLAIN && self.plain.is_none() {
+                        if let Ok(decoded_data) = mail.get_body() {
+                            self.plain = Some(PlainText {
+                                text: decoded_data,
+                                flowed: if let Some(format) = mail.ctype.params.get("format") {
+                                    format.as_str().to_ascii_lowercase() == "flowed"
+                                } else {
+                                    false
+                                },
+                                delsp: if let Some(delsp) = mail.ctype.params.get("delsp") {
+                                    delsp.as_str().to_ascii_lowercase() == "yes"
+                                } else {
+                                    false
+                                },
+                            });
+                        }
+                    }
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+
+    /// Replace cid:-protocol by the data:-protocol where appropriate.
+    /// This allows the final html-file to be self-contained.
+    fn cid_to_data_recursive<'a>(
+        &'a mut self,
+        context: &'a Context,
+        mail: &'a mailparse::ParsedMail<'a>,
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
+        // Boxed future to deal with recursion
+        async move {
+            match get_mime_multipart_type(&mail.ctype).await {
+                MimeMultipartType::Multiple => {
+                    for cur_data in mail.subparts.iter() {
+                        self.cid_to_data_recursive(context, cur_data).await?;
+                    }
+                    Ok(())
+                }
+                MimeMultipartType::Message => {
+                    let raw = mail.get_body_raw()?;
+                    if raw.is_empty() {
+                        return Ok(());
+                    }
+                    let mail = mailparse::parse_mail(&raw).unwrap();
+                    self.cid_to_data_recursive(context, &mail).await
+                }
+                MimeMultipartType::Single => {
+                    let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+                    if mimetype.type_() == mime::IMAGE {
+                        if let Some(cid) = mail.headers.get_header_value(HeaderDef::ContentId) {
+                            if let Ok(cid) = parse_message_id(&cid) {
+                                if let Ok(replacement) = mimepart_to_data_url(mail).await {
+                                    let re_string = format!(
+                                        "(<img[^>]*src[^>]*=[^>]*)(cid:{})([^>]*>)",
+                                        regex::escape(&cid)
+                                    );
+                                    match regex::Regex::new(&re_string) {
+                                        Ok(re) => {
+                                            self.html = re
+                                                .replace_all(
+                                                    &*self.html,
+                                                    format!("${{1}}{}${{3}}", replacement).as_str(),
+                                                )
+                                                .as_ref()
+                                                .to_string()
+                                        }
+                                        Err(e) => warn!(
+                                            context,
+                                            "Cannot create regex for cid: {} throws {}",
+                                            re_string,
+                                            e
+                                        ),
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+}
+
+/// Convert a mime part to a data: url as defined in [RFC 2397](https://tools.ietf.org/html/rfc2397).
+async fn mimepart_to_data_url(mail: &mailparse::ParsedMail<'_>) -> Result<String> {
+    let data = mail.get_body_raw()?;
+    let data = base64::encode(&data);
+    Ok(format!("data:{};base64,{}", mail.ctype.mimetype, data))
+}
+
+impl MsgId {
+    /// Get HTML from a message-id.
+    /// This requires `mime_headers` field to be set for the message;
+    /// this is the case at least when `Message.has_html()` returns true
+    /// (we do not save raw mime unconditionally in the database to save space).
+    /// The corresponding ffi-function is `dc_get_msg_html()`.
+    pub async fn get_html(self, context: &Context) -> Option<String> {
+        let rawmime: Option<String> = context
+            .sql
+            .query_get_value(
+                context,
+                "SELECT mime_headers FROM msgs WHERE id=?;",
+                paramsv![self],
+            )
+            .await;
+
+        if let Some(rawmime) = rawmime {
+            if !rawmime.is_empty() {
+                match HtmlMsgParser::from_bytes(context, rawmime.as_bytes()).await {
+                    Err(err) => {
+                        warn!(context, "get_html: parser error: {}", err);
+                        None
+                    }
+                    Ok(parser) => Some(parser.html),
+                }
+            } else {
+                warn!(context, "get_html: empty mime for {}", self);
+                None
+            }
+        } else {
+            warn!(context, "get_html: no mime for {}", self);
+            None
+        }
+    }
+}
+
+/// Wraps HTML text into a new text/html mimepart structure.
+///
+/// Used on forwarding messages to avoid leaking the original mime structure
+/// and also to avoid sending too much, maybe large data.
+pub async fn new_html_mimepart(html: String) -> PartBuilder {
+    PartBuilder::new()
+        .content_type(&"text/html; charset=utf-8".parse::<mime::Mime>().unwrap())
+        .body(html)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chat;
+    use crate::chat::forward_msgs;
+    use crate::config::Config;
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::message::MessengerMessage;
+    use crate::test_utils::TestContext;
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_unspecified() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+This message does not have Content-Type nor Subject.<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_iso88591() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_iso88591.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+message with a non-UTF-8 encoding: äöüßÄÖÜ<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_flowed() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_flowed.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(parser.plain.unwrap().flowed);
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+This line ends with a space and will be merged with the next one due to format=flowed.<br/>
+<br/>
+This line does not end with a space<br/>
+and will be wrapped as usual.<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_plain() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+mime-modified should not be set set as there is no html and no special stuff;<br/>
+although not being a delta-message.<br/>
+test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x27; :)<br/>
+<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+
+        // on windows, `\r\n` linends are returned from mimeparser,
+        // however, rust multiline-strings use just `\n`;
+        // therefore, we just remove `\r` before comparison.
+        assert_eq!(
+            parser.html.replace("\r", ""),
+            r##"
+<html>
+  <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
+</html>"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            r##"<html>
+  <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
+</html>
+
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_plain_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            r##"<html>
+  <p>
+    this is <b>html</b>
+  </p>
+</html>
+
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_apple_cid_jpg() {
+        // load raw mime html-data with related image-part (cid:)
+        // and make sure, Content-Id has angle-brackets that are removed correctly.
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/apple_cid_jpg.eml");
+        let test = String::from_utf8_lossy(raw);
+        assert!(test.contains("Content-Id: <8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box>"));
+        assert!(test.contains("cid:8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box"));
+        assert!(test.find("data:").is_none());
+
+        // parsing converts cid: to data:
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(parser.html.contains("<html>"));
+        assert!(!parser.html.contains("Content-Id:"));
+        assert!(parser.html.contains("data:image/jpeg;base64,/9j/4AAQ"));
+        assert!(!parser.html.contains("cid:"));
+    }
+
+    #[async_std::test]
+    async fn test_get_html_empty() {
+        let t = TestContext::new().await;
+        let msg_id = MsgId::new_unset();
+        assert!(msg_id.get_html(&t).await.is_none())
+    }
+
+    #[async_std::test]
+    async fn test_html_forwarding() {
+        // alice receives a non-delta html-message
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("2")).await.ok();
+        let chat = alice
+            .create_chat_with_contact("", "sender@testrun.org")
+            .await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::No);
+        assert!(!msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+
+        // alice: create chat with bob and forward received html-message there
+        let chat = alice.create_chat_with_contact("", "bob@example.net").await;
+        forward_msgs(&alice, &[msg.get_id()], chat.get_id())
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+
+        // bob: check that bob also got the html-part of the forwarded message
+        let bob = TestContext::new_bob().await;
+        let chat = bob.create_chat_with_contact("", "alice@example.com").await;
+        bob.recv_msg(&alice.pop_sent_msg().await).await;
+        let msg = bob.get_last_msg_in(chat.get_id()).await;
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&bob).await.unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+    }
+
+    #[async_std::test]
+    async fn test_html_forwarding_encrypted() {
+        // Alice receives a non-delta html-message
+        // (`ShowEmails=1` lets Alice actually receive non-delta messages for known contacts,
+        // the contact is marked as known by creating a chat using `chat_with_contact()`)
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("1")).await.ok();
+        let chat = alice
+            .create_chat_with_contact("", "sender@testrun.org")
+            .await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+
+        // forward the message to saved-messages,
+        // this will encrypt the message as new_alice() has set up keys
+        let chat = alice.get_self_chat().await;
+        forward_msgs(&alice, &[msg.get_id()], chat.get_id())
+            .await
+            .unwrap();
+        let msg = alice.pop_sent_msg().await;
+
+        // receive the message on another device
+        let alice = TestContext::new_alice().await;
+        assert_eq!(alice.get_config_int(Config::ShowEmails).await, 0); // set to "1" above, make sure it is another db
+        alice.recv_msg(&msg).await;
+        let chat = alice.get_self_chat().await;
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.get_showpadlock());
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+    }
+
+    #[async_std::test]
+    async fn test_set_html() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // alice sends a message with html-part to bob
+        let chat_id = alice.create_chat(&bob).await.id;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("plain text".to_string()));
+        msg.set_html(Some("<b>html</b> text".to_string()));
+        assert!(msg.mime_modified);
+        chat::send_msg(&alice, chat_id, &mut msg).await.unwrap();
+
+        // check the message is written correctly to alice's db
+        let msg = alice.get_last_msg_in(chat_id).await;
+        assert_eq!(msg.get_text(), Some("plain text".to_string()));
+        assert!(!msg.is_forwarded());
+        assert!(msg.mime_modified);
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.contains("<b>html</b> text"));
+
+        // let bob receive the message
+        let chat_id = bob.create_chat(&alice).await.id;
+        bob.recv_msg(&alice.pop_sent_msg().await).await;
+        let msg = bob.get_last_msg_in(chat_id).await;
+        assert_eq!(msg.get_text(), Some("plain text".to_string()));
+        assert!(!msg.is_forwarded());
+        assert!(msg.mime_modified);
+        let html = msg.get_id().get_html(&bob).await.unwrap();
+        assert!(html.contains("<b>html</b> text"));
+    }
+}

src/imap/idle.rs

@@ -1,11 +1,11 @@
 use super::Imap;
 
+use anyhow::{bail, format_err, Result};
 use async_imap::extensions::idle::IdleResponse;
 use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
-use crate::error::{bail, format_err, Result};
 use crate::{context::Context, scheduler::InterruptInfo};
 
 use super::session::Session;
@@ -50,6 +50,12 @@ impl Imap {
                 return Ok(info);
             }
 
+            if let Ok(info) = self.idle_interrupt.try_recv() {
+                info!(context, "skip idle, got interrupt {:?}", info);
+                self.session = Some(session);
+                return Ok(info);
+            }
+
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
                 bail!("IMAP IDLE protocol failed to init/complete: {}", err);
@@ -62,14 +68,18 @@ impl Imap {
                 Interrupt(InterruptInfo),
             }
 
-            info!(context, "Idle entering wait-on-remote state");
+            info!(
+                context,
+                "{}: Idle entering wait-on-remote state",
+                watch_folder.as_deref().unwrap_or("None")
+            );
             let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
-                let probe_network = self.idle_interrupt.recv().await;
+                let info = self.idle_interrupt.recv().await;
 
                 // cancel imap idle connection properly
                 drop(interrupt);
 
-                Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+                Ok(Event::Interrupt(info.unwrap_or_default()))
             });
 
             match fut.await {
@@ -115,10 +125,12 @@ impl Imap {
         let fake_idle_start_time = SystemTime::now();
 
         // Do not poll, just wait for an interrupt when no folder is passed in.
-        if watch_folder.is_none() {
+        let watch_folder = if let Some(watch_folder) = watch_folder {
+            watch_folder
+        } else {
             info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
             return self.idle_interrupt.recv().await.unwrap_or_default();
-        }
+        };
         info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
         // check every minute if there are new messages
@@ -160,19 +172,17 @@ impl Imap {
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
 
-                    if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_new_messages(context, watch_folder, false).await {
-                            Ok(res) => {
-                                info!(context, "fetch_new_messages returned {:?}", res);
-                                if res {
-                                    break InterruptInfo::new(false, None);
-                                }
-                            }
-                            Err(err) => {
-                                error!(context, "could not fetch from folder: {}", err);
-                                self.trigger_reconnect()
+                    match self.fetch_new_messages(context, &watch_folder, false).await {
+                        Ok(res) => {
+                            info!(context, "fetch_new_messages returned {:?}", res);
+                            if res {
+                                break InterruptInfo::new(false, None);
                             }
                         }
+                        Err(err) => {
+                            error!(context, "could not fetch from folder: {:#}", err);
+                            self.trigger_reconnect()
+                        }
                     }
                 }
                 Event::Interrupt(info) => {

src/imap/scan_folders.rs

@@ -0,0 +1,105 @@
+use std::time::Instant;
+
+use anyhow::{Context as _, Result};
+
+use crate::config::Config;
+use crate::context::Context;
+use crate::imap::Imap;
+use async_std::prelude::*;
+
+use super::{get_folder_meaning, get_folder_meaning_by_name, FolderMeaning};
+
+impl Imap {
+    pub async fn scan_folders(&mut self, context: &Context) -> Result<()> {
+        // First of all, debounce to once per minute:
+        let mut last_scan = context.last_full_folder_scan.lock().await;
+        if let Some(last_scan) = *last_scan {
+            let elapsed_secs = last_scan.elapsed().as_secs();
+            let debounce_secs = context
+                .get_config_u64(Config::ScanAllFoldersDebounceSecs)
+                .await;
+
+            if elapsed_secs < debounce_secs {
+                return Ok(());
+            }
+        }
+        info!(context, "Starting full folder scan");
+
+        self.setup_handle(context).await?;
+        let session = self.session.as_mut();
+        let session = session.context("scan_folders(): IMAP No Connection established")?;
+        let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
+        let watched_folders = get_watched_folders(context).await;
+
+        let mut sentbox_folder = None;
+        let mut spam_folder = None;
+
+        for folder in folders {
+            let folder = match folder {
+                Ok(f) => f,
+                Err(e) => {
+                    warn!(context, "Can't get folder: {}", e);
+                    continue;
+                }
+            };
+
+            let foldername = folder.name();
+            let folder_meaning = get_folder_meaning(&folder);
+            let folder_name_meaning = get_folder_meaning_by_name(foldername);
+
+            if folder_meaning == FolderMeaning::SentObjects {
+                // Always takes precedent
+                sentbox_folder = Some(folder.name().to_string());
+            } else if folder_meaning == FolderMeaning::Spam {
+                spam_folder = Some(folder.name().to_string());
+            } else if folder_name_meaning == FolderMeaning::SentObjects {
+                // only set iff none has been already set
+                if sentbox_folder.is_none() {
+                    sentbox_folder = Some(folder.name().to_string());
+                }
+            } else if folder_name_meaning == FolderMeaning::Spam && spam_folder.is_none() {
+                spam_folder = Some(folder.name().to_string());
+            }
+
+            if watched_folders.contains(&foldername.to_string()) {
+                info!(
+                    context,
+                    "Not scanning folder {} as it is watched anyway", foldername
+                );
+            } else {
+                info!(context, "Scanning folder: {}", foldername);
+
+                if let Err(e) = self.fetch_new_messages(context, foldername, false).await {
+                    warn!(context, "Can't fetch new msgs in scanned folder: {:#}", e);
+                }
+            }
+        }
+
+        context
+            .set_config(Config::ConfiguredSentboxFolder, sentbox_folder.as_deref())
+            .await?;
+        context
+            .set_config(Config::ConfiguredSpamFolder, spam_folder.as_deref())
+            .await?;
+
+        last_scan.replace(Instant::now());
+        Ok(())
+    }
+}
+
+async fn get_watched_folders(context: &Context) -> Vec<String> {
+    let mut res = Vec::new();
+    let folder_watched_configured = &[
+        (Config::SentboxWatch, Config::ConfiguredSentboxFolder),
+        (Config::MvboxWatch, Config::ConfiguredMvboxFolder),
+        (Config::InboxWatch, Config::ConfiguredInboxFolder),
+    ];
+    for (watched, configured) in folder_watched_configured {
+        if context.get_config_bool(*watched).await {
+            if let Some(folder) = context.get_config(*configured).await {
+                res.push(folder);
+            }
+        }
+    }
+    res
+}

src/imap/select_folder.rs

@@ -61,11 +61,12 @@ impl Imap {
 
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
+    /// Returns whether a new folder was selected.
     pub(super) async fn select_folder<S: AsRef<str>>(
         &mut self,
         context: &Context,
         folder: Option<S>,
-    ) -> Result<()> {
+    ) -> Result<NewlySelected> {
         if self.session.is_none() {
             self.config.selected_folder = None;
             self.config.selected_folder_needs_expunge = false;
@@ -78,7 +79,7 @@ impl Imap {
         if let Some(ref folder) = folder {
             if let Some(ref selected_folder) = self.config.selected_folder {
                 if folder.as_ref() == selected_folder {
-                    return Ok(());
+                    return Ok(NewlySelected::No);
                 }
             }
         }
@@ -99,7 +100,7 @@ impl Imap {
                     Ok(mailbox) => {
                         self.config.selected_folder = Some(folder.as_ref().to_string());
                         self.config.selected_mailbox = Some(mailbox);
-                        Ok(())
+                        Ok(NewlySelected::Yes)
                     }
                     Err(async_imap::error::Error::ConnectionLost) => {
                         self.trigger_reconnect();
@@ -119,7 +120,15 @@ impl Imap {
                 Err(Error::NoSession)
             }
         } else {
-            Ok(())
+            Ok(NewlySelected::No)
         }
     }
 }
+#[derive(PartialEq, Debug, Copy, Clone, Eq)]
+pub(super) enum NewlySelected {
+    /// The folder was newly selected during this call to select_folder().
+    Yes,
+    /// No SELECT command was run because the folder already was selected
+    /// and self.config.selected_mailbox was not updated (so, e.g. it may contain an outdated uid_next)
+    No,
+}

src/imex.rs

@@ -1,12 +1,9 @@
 //! # Import/export module
 
 use std::any::Any;
-use std::{
-    cmp::{max, min},
-    ffi::OsStr,
-};
+use std::ffi::OsStr;
 
-use anyhow::Context as _;
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use async_std::path::{Path, PathBuf};
 use async_std::{
     fs::{self, File},
@@ -14,23 +11,26 @@ use async_std::{
 };
 use rand::{thread_rng, Rng};
 
-use crate::blob::BlobObject;
 use crate::chat;
 use crate::chat::delete_and_reset_all_device_msgs;
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{
+    dc_copy_file, dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
+    dc_open_file_std, dc_read_file, dc_write_file, get_next_backup_path, time, EmailAddress,
+};
 use crate::e2ee;
-use crate::error::*;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::param::*;
+use crate::param::Param;
 use crate::pgp;
 use crate::sql::{self, Sql};
-use crate::stock::StockMessage;
+use crate::stock_str;
+use crate::{blob::BlobObject, log::LogExt};
+use ::pgp::types::KeyTrait;
 use async_tar::Archive;
 
 // Name of the database file in the backup.
@@ -78,11 +78,7 @@ pub enum ImexMode {
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
-pub async fn imex(
-    context: &Context,
-    what: ImexMode,
-    param1: Option<impl AsRef<Path>>,
-) -> Result<()> {
+pub async fn imex(context: &Context, what: ImexMode, param1: impl AsRef<Path>) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
 
     let res = async {
@@ -162,6 +158,7 @@ pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Re
     let dir_name = dir_name.as_ref();
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
+    let mut newest_backup_name = "".to_string();
     let mut newest_backup_path: Option<PathBuf> = None;
     while let Some(dirent) = dir_iter.next().await {
         if let Ok(dirent) = dirent {
@@ -183,10 +180,22 @@ pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Re
                         info!(context, "backup_time of {} is {}", name, curr_backup_time);
                         sql.close().await;
                     }
-                    Err(e) => warn!(
-                        context,
-                        "Found backup file {} which could not be opened: {}", name, e
-                    ),
+                    Err(e) => {
+                        warn!(
+                            context,
+                            "Found backup file {} which could not be opened: {}", name, e
+                        );
+                        // On some Android devices we can't open sql files that are not in our private directory
+                        // (see https://github.com/deltachat/deltachat-android/issues/1768). So, compare names
+                        // to still find the newest backup.
+                        let name: String = name.into();
+                        if newest_backup_time == 0
+                            && (newest_backup_name.is_empty() || name > newest_backup_name)
+                        {
+                            newest_backup_path = Some(path);
+                            newest_backup_name = name;
+                        }
+                    }
                 }
             }
         }
@@ -267,7 +276,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
     };
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes()).await?;
+    let encr = pgp::symm_encrypt(passphrase, private_key_asc.as_bytes()).await?;
 
     let replacement = format!(
         concat!(
@@ -279,8 +288,8 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
-    let msg_subj = context.stock_str(StockMessage::AcSetupMsgSubject).await;
-    let msg_body = context.stock_str(StockMessage::AcSetupMsgBody).await;
+    let msg_subj = stock_str::ac_setup_msg_subject(context).await;
+    let msg_body = stock_str::ac_setup_msg_body(context).await;
     let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
     Ok(format!(
         concat!(
@@ -413,6 +422,8 @@ async fn set_self_key(
         },
     )
     .await?;
+
+    info!(context, "stored self key: {:?}", keypair.secret.key_id());
     Ok(())
 }
 
@@ -429,7 +440,7 @@ async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
 pub fn normalize_setup_code(s: &str) -> String {
     let mut out = String::new();
     for c in s.chars() {
-        if c >= '0' && c <= '9' {
+        if ('0'..='9').contains(&c) {
             out.push(c);
             if let 4 | 9 | 14 | 19 | 24 | 29 | 34 | 39 = out.len() {
                 out += "-"
@@ -439,19 +450,11 @@ pub fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-async fn imex_inner(
-    context: &Context,
-    what: ImexMode,
-    param: Option<impl AsRef<Path>>,
-) -> Result<()> {
-    ensure!(param.is_some(), "No Import/export dir/file given.");
-
-    info!(context, "Import/export process started.");
+async fn imex_inner(context: &Context, what: ImexMode, path: impl AsRef<Path>) -> Result<()> {
+    info!(context, "Import/export dir: {}", path.as_ref().display());
+    ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
 
-    ensure!(context.sql.is_open().await, "Database not opened.");
-
-    let path = param.ok_or_else(|| format_err!("Imex: Param was None"))?;
     if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
         if e2ee::ensure_secret_key_exists(context).await.is_err() {
@@ -465,9 +468,7 @@ async fn imex_inner(
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
 
-        // TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old().
-        // (now is 07/2020)
-        ImexMode::ExportBackup => export_backup_old(context, path).await,
+        ImexMode::ExportBackup => export_backup(context, path).await,
         // import_backup() will call import_backup_old() if this is an old backup.
         ImexMode::ImportBackup => import_backup(context, path).await,
     }
@@ -495,6 +496,10 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         !context.is_configured().await,
         "Cannot import backups to accounts in use."
     );
+    ensure!(
+        !context.scheduler.read().await.is_running(),
+        "cannot import backup, IO already running"
+    );
     context.sql.close().await;
     dc_delete_file(context, context.get_dbfile()).await;
     ensure!(
@@ -503,10 +508,20 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     );
 
     let backup_file = File::open(backup_to_import).await?;
+    let file_size = backup_file.metadata().await?.len();
     let archive = Archive::new(backup_file);
+
     let mut entries = archive.entries()?;
     while let Some(file) = entries.next().await {
         let f = &mut file?;
+
+        let current_pos = f.raw_file_position();
+        let progress = 1000 * current_pos / file_size;
+        if progress > 10 && progress < 1000 {
+            // We already emitted ImexProgress(10) above
+            context.emit_event(EventType::ImexProgress(progress as usize));
+        }
+
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
             // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
             f.unpack_in(context.get_blobdir()).await?;
@@ -515,7 +530,6 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
                 context.get_dbfile(),
             )
             .await?;
-            context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
         } else {
             // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
             f.unpack_in(context.get_blobdir()).await?;
@@ -532,11 +546,11 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
 
     context
         .sql
-        .open(&context, &context.get_dbfile(), false)
+        .open(context, &context.get_dbfile(), false)
         .await
         .context("Could not re-open db")?;
 
-    delete_and_reset_all_device_msgs(&context).await?;
+    delete_and_reset_all_device_msgs(context).await?;
 
     Ok(())
 }
@@ -553,6 +567,10 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
         !context.is_configured().await,
         "Cannot import backups to accounts in use."
     );
+    ensure!(
+        !context.scheduler.read().await.is_running(),
+        "cannot import backup, IO already running"
+    );
     context.sql.close().await;
     dc_delete_file(context, context.get_dbfile()).await;
     ensure!(
@@ -568,11 +586,11 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
     /* re-open copied database file */
     context
         .sql
-        .open(&context, &context.get_dbfile(), false)
+        .open(context, &context.get_dbfile(), false)
         .await
         .context("Could not re-open db")?;
 
-    delete_and_reset_all_device_msgs(&context).await?;
+    delete_and_reset_all_device_msgs(context).await?;
 
     let total_files_cnt = context
         .sql
@@ -651,14 +669,14 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
 async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
-    let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+    let (temp_path, dest_path) = get_next_backup_path(dir, now).await?;
     let _d = DeleteOnDrop(temp_path.clone());
 
     context
         .sql
         .set_raw_config_int(context, "backup_time", now as i32)
         .await?;
-    sql::housekeeping(context).await;
+    sql::housekeeping(context).await.ok_or_log(context);
 
     context
         .sql
@@ -666,6 +684,11 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         .await
         .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
 
+    ensure!(
+        !context.scheduler.read().await.is_running(),
+        "cannot export backup, IO already running"
+    );
+
     // we close the database during the export
     context.sql.close().await;
 
@@ -681,7 +704,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // we re-open the database after export is finished
     context
         .sql
-        .open(&context, &context.get_dbfile(), false)
+        .open(context, &context.get_dbfile(), false)
         .await;
 
     match &res {
@@ -715,131 +738,37 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
         .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
         .await?;
 
-    context.emit_event(EventType::ImexProgress(500));
+    let read_dir: Vec<_> = fs::read_dir(context.get_blobdir()).await?.collect().await;
+    let count = read_dir.len();
+    let mut written_files = 0;
 
-    builder
-        .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
-        .await?;
+    for entry in read_dir.into_iter() {
+        let entry = entry?;
+        let name = entry.file_name();
+        if !entry.file_type().await?.is_file() {
+            warn!(
+                context,
+                "Export: Found dir entry {} that is not a file, ignoring",
+                name.to_string_lossy()
+            );
+            continue;
+        }
+        let mut file = File::open(entry.path()).await?;
+        let path_in_archive = PathBuf::from(BLOBS_BACKUP_NAME).join(name);
+        builder.append_file(path_in_archive, &mut file).await?;
+
+        written_files += 1;
+        let progress = 1000 * written_files / count;
+        if progress > 10 && progress < 1000 {
+            // We already emitted ImexProgress(10) above
+            emit_event!(context, EventType::ImexProgress(progress));
+        }
+    }
 
     builder.finish().await?;
     Ok(())
 }
 
-async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
-    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
-    // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
-    // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
-    let now = time();
-    let dest_path_filename = get_next_backup_path_old(dir, now).await?;
-    let dest_path_string = dest_path_filename.to_string_lossy().to_string();
-
-    sql::housekeeping(context).await;
-
-    context.sql.execute("VACUUM;", paramsv![]).await.ok();
-
-    // we close the database during the copy of the dbfile
-    context.sql.close().await;
-    info!(
-        context,
-        "Backup '{}' to '{}'.",
-        context.get_dbfile().display(),
-        dest_path_filename.display(),
-    );
-    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename).await;
-    context
-        .sql
-        .open(&context, &context.get_dbfile(), false)
-        .await?;
-
-    if !copied {
-        bail!(
-            "could not copy file from '{}' to '{}'",
-            context.get_dbfile().display(),
-            dest_path_string
-        );
-    }
-    let dest_sql = Sql::new();
-    dest_sql
-        .open(context, &dest_path_filename, false)
-        .await
-        .with_context(|| format!("could not open exported database {}", dest_path_string))?;
-
-    let res = match add_files_to_export(context, &dest_sql).await {
-        Err(err) => {
-            dc_delete_file(context, &dest_path_filename).await;
-            error!(context, "backup failed: {}", err);
-            Err(err)
-        }
-        Ok(()) => {
-            dest_sql
-                .set_raw_config_int(context, "backup_time", now as i32)
-                .await?;
-            context.emit_event(EventType::ImexFileWritten(dest_path_filename));
-            Ok(())
-        }
-    };
-    dest_sql.close().await;
-
-    Ok(res?)
-}
-
-async fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
-    // add all files as blobs to the database copy (this does not require
-    // the source to be locked, neigher the destination as it is used only here)
-    if !sql.table_exists("backup_blobs").await? {
-        sql.execute(
-            "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
-            paramsv![],
-        )
-        .await?;
-    }
-    // copy all files from BLOBDIR into backup-db
-    let mut total_files_cnt = 0;
-    let dir = context.get_blobdir();
-    let dir_handle = async_std::fs::read_dir(&dir).await?;
-    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count().await;
-
-    info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
-
-    sql.with_conn_async(|conn| async move {
-        // scan directory, pass 2: copy files
-        let mut dir_handle = async_std::fs::read_dir(&dir).await?;
-
-        let mut processed_files_cnt = 0;
-        while let Some(entry) = dir_handle.next().await {
-            let entry = entry?;
-            if context.shall_stop_ongoing().await {
-                return Ok(());
-            }
-            processed_files_cnt += 1;
-            let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
-            context.emit_event(EventType::ImexProgress(permille));
-
-            let name_f = entry.file_name();
-            let name = name_f.to_string_lossy();
-            if name.starts_with("delta-chat") && name.ends_with(".bak") {
-                continue;
-            }
-            info!(context, "EXPORT: copying filename={}", name);
-            let curr_path_filename = context.get_blobdir().join(entry.file_name());
-            if let Ok(buf) = dc_read_file(context, &curr_path_filename).await {
-                if buf.is_empty() {
-                    continue;
-                }
-                // bail out if we can't insert
-                let mut stmt = conn.prepare_cached(
-                    "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
-                )?;
-                stmt.execute(paramsv![name, buf])?;
-            }
-        }
-        Ok(())
-    })
-    .await?;
-
-    Ok(())
-}
-
 /*******************************************************************************
  * Classic key import
  ******************************************************************************/
@@ -875,6 +804,12 @@ async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
                 continue;
             }
         }
+        info!(
+            context,
+            "considering key file: {}",
+            path_plus_name.display()
+        );
+
         match dc_read_file(context, &path_plus_name).await {
             Ok(buf) => {
                 let armored = std::string::String::from_utf8_lossy(&buf);
@@ -964,7 +899,7 @@ where
         let any_key = key as &dyn Any;
         let kind = if any_key.downcast_ref::<SignedPublicKey>().is_some() {
             "public"
-        } else if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+        } else if any_key.downcast_ref::<SignedSecretKey>().is_some() {
             "private"
         } else {
             "unknown"
@@ -972,7 +907,12 @@ where
         let id = id.map_or("default".into(), |i| i.to_string());
         dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
     };
-    info!(context, "Exporting key {}", file_name.display());
+    info!(
+        context,
+        "Exporting key {:?} to {}",
+        key.key_id(),
+        file_name.display()
+    );
     dc_delete_file(context, &file_name).await;
 
     let content = key.to_asc(None).into_bytes();
@@ -988,8 +928,11 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+
     use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
-    use crate::test_utils::*;
+    use crate::stock_str::StockMessage;
+    use crate::test_utils::{alice_keypair, TestContext};
+
     use ::pgp::armor::BlockType;
 
     #[async_std::test]
@@ -997,7 +940,7 @@ mod tests {
         let t = TestContext::new().await;
 
         t.configure_alice().await;
-        let msg = render_setup_file(&t.ctx, "hello").await.unwrap();
+        let msg = render_setup_file(&t, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
         // In particular note the mixing of `\r\n` and `\n` depending
@@ -1014,12 +957,11 @@ mod tests {
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
+        t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
         t.configure_alice().await;
-        let msg = render_setup_file(&t.ctx, "pw").await.unwrap();
+        let msg = render_setup_file(&t, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
     }
@@ -1027,7 +969,7 @@ mod tests {
     #[async_std::test]
     async fn test_create_setup_code() {
         let t = TestContext::new().await;
-        let setupcode = create_setup_code(&t.ctx);
+        let setupcode = create_setup_code(&t);
         assert_eq!(setupcode.len(), 44);
         assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
         assert_eq!(setupcode.chars().nth(9).unwrap(), '-');
@@ -1040,7 +982,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_export_key_to_asc_file() {
+    async fn test_export_public_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
@@ -1054,19 +996,35 @@ mod tests {
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
+    #[async_std::test]
+    async fn test_export_private_key_to_asc_file() {
+        let context = TestContext::new().await;
+        let key = alice_keypair().secret;
+        let blobdir = "$BLOBDIR";
+        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
+            .await
+            .is_ok());
+        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        let filename = format!("{}/private-key-default.asc", blobdir);
+        let bytes = async_std::fs::read(&filename).await.unwrap();
+
+        assert_eq!(bytes, key.to_asc(None).into_bytes());
+    }
+
     #[async_std::test]
     async fn test_export_and_import_key() {
         let context = TestContext::new().await;
         context.configure_alice().await;
-        let blobdir = "$BLOBDIR";
-        assert!(imex(&context.ctx, ImexMode::ExportSelfKeys, Some(blobdir))
-            .await
-            .is_ok());
-
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
-        assert!(imex(&context.ctx, ImexMode::ImportSelfKeys, Some(blobdir))
-            .await
-            .is_ok());
+        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir).await {
+            panic!("got error on export: {:?}", err);
+        }
+
+        let context2 = TestContext::new().await;
+        context2.configure_alice().await;
+        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir).await {
+            panic!("got error on import: {:?}", err);
+        }
     }
 
     #[test]

src/job.rs

@@ -2,41 +2,43 @@
 //!
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
-
-use std::fmt;
 use std::future::Future;
+use std::{fmt, time::Duration};
 
+use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
+use async_smtp::smtp::response::{Category, Code, Detail};
+use async_std::task::sleep;
 use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
 
-use async_smtp::smtp::response::Category;
-use async_smtp::smtp::response::Code;
-use async_smtp::smtp::response::Detail;
-
-use crate::chat::{self, ChatId};
-use crate::config::Config;
-use crate::contact::Contact;
-use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_delete_file, dc_read_file, time};
 use crate::ephemeral::load_imap_deletion_msgid;
-use crate::error::{bail, ensure, format_err, Error, Result};
 use crate::events::EventType;
-use crate::imap::*;
+use crate::imap::{Imap, ImapActionResult};
 use crate::location;
 use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
-use crate::param::*;
+use crate::param::{Param, Params};
 use crate::smtp::Smtp;
 use crate::{blob::BlobObject, contact::normalize_name, contact::Modifier, contact::Origin};
+use crate::{
+    chat::{self, Chat, ChatId, ChatItem},
+    constants::DC_CHAT_ID_DEADDROP,
+};
+use crate::{config::Config, constants::Blocked};
+use crate::{constants::Chattype, contact::Contact};
+use crate::{context::Context, log::LogExt};
 use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
 
 /// Thread IDs
-#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(i32)]
 pub(crate) enum Thread {
     Unknown = 0,
@@ -248,7 +250,7 @@ impl Job {
             info!(context, "smtp-sending out mime message:");
             println!("{}", String::from_utf8_lossy(&message));
         }
-        match smtp.send(context, recipients, message, job_id).await {
+        let status = match smtp.send(context, recipients, message, job_id).await {
             Err(crate::smtp::send::Error::SendError(err)) => {
                 // Remote error, retry later.
                 warn!(context, "SMTP failed to send: {}", err);
@@ -273,7 +275,7 @@ impl Job {
                                 // Other enhanced status codes, such as Postfix
                                 // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
                                 // are not ignored.
-                                response.message.get(0) == Some(&"5.5.0".to_string())
+                                response.first_word() == Some(&"5.5.0".to_string())
                             }
                             _ => false,
                         };
@@ -285,25 +287,30 @@ impl Job {
                             // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
                             // should definitely go here, because user has to open the link to
                             // resume message sending.
-                            let msg_id = MsgId::new(self.foreign_id);
-                            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
-                            match Message::load_from_db(context, msg_id).await {
-                                Ok(message) => {
-                                    chat::add_info_msg(context, message.chat_id, err.to_string())
-                                        .await
-                                }
-                                Err(e) => error!(
-                                    context,
-                                    "couldn't load chat_id to inform user about SMTP error: {}", e
-                                ),
-                            };
                             Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                         }
                     }
-                    async_smtp::smtp::error::Error::Transient(_) => {
+                    async_smtp::smtp::error::Error::Transient(ref response) => {
                         // We got a transient 4xx response from SMTP server.
                         // Give some time until the server-side error maybe goes away.
-                        Status::RetryLater
+
+                        if let Some(first_word) = response.first_word() {
+                            if first_word.ends_with(".1.1")
+                                || first_word.ends_with(".1.2")
+                                || first_word.ends_with(".1.3")
+                            {
+                                // Sometimes we receive transient errors that should be permanent.
+                                // Any extended smtp status codes like x.1.1, x.1.2 or x.1.3 that we
+                                // receive as a transient error are misconfigurations of the smtp server.
+                                // See https://tools.ietf.org/html/rfc3463#section-3.2
+                                info!(context, "Smtp-job #{} Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", self.job_id, first_word);
+                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } else {
+                                Status::RetryLater
+                            }
+                        } else {
+                            Status::RetryLater
+                        }
                     }
                     _ => {
                         if smtp.has_maybe_stale_connection().await {
@@ -336,7 +343,14 @@ impl Job {
                 job_try!(success_cb().await);
                 Status::Finished(Ok(()))
             }
+        };
+
+        if let Status::Finished(Err(err)) = &status {
+            // We couldn't send the message, so mark it as failed
+            let msg_id = MsgId::new(self.foreign_id);
+            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
         }
+        status
     }
 
     pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
@@ -477,7 +491,7 @@ impl Job {
         let msg = job_try!(Message::load_from_db(context, msg_id).await);
         let mimefactory =
             job_try!(MimeFactory::from_mdn(context, &msg, additional_rfc724_mids).await);
-        let rendered_msg = job_try!(mimefactory.render().await);
+        let rendered_msg = job_try!(mimefactory.render(context).await);
         let body = rendered_msg.message;
 
         let addr = contact.get_addr();
@@ -508,11 +522,27 @@ impl Job {
         }
 
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await;
+        let server_folder = &job_try!(msg
+            .server_folder
+            .context("Can't move message out of folder if we don't know the current folder"));
+
+        let move_res = msg.id.needs_move(context, server_folder).await;
+        let dest_folder = match move_res {
+            Err(e) => {
+                warn!(context, "could not load dest folder: {}", e);
+                return Status::RetryLater;
+            }
+            Ok(None) => {
+                warn!(
+                    context,
+                    "msg {} does not need to be moved from {}", msg.id, server_folder
+                );
+                return Status::Finished(Ok(()));
+            }
+            Ok(Some(config)) => context.get_config(config).await,
+        };
 
         if let Some(dest_folder) = dest_folder {
-            let server_folder = msg.server_folder.as_ref().unwrap();
-
             match imap
                 .mv(context, server_folder, msg.server_uid, &dest_folder)
                 .await
@@ -639,7 +669,7 @@ impl Job {
         add_all_recipients_as_contacts(context, imap, Config::ConfiguredMvboxFolder).await;
         add_all_recipients_as_contacts(context, imap, Config::ConfiguredInboxFolder).await;
 
-        if context.get_config_bool(Config::FetchExisting).await {
+        if context.get_config_bool(Config::FetchExistingMsgs).await {
             for config in &[
                 Config::ConfiguredMvboxFolder,
                 Config::ConfiguredInboxFolder,
@@ -654,6 +684,39 @@ impl Job {
                 }
             }
         }
+
+        // Make sure that if there now is a chat with a contact (created by an outgoing
+        // message), then group contact requests from this contact should also be unblocked.
+        // See https://github.com/deltachat/deltachat-core-rust/issues/2097.
+        for item in chat::get_chat_msgs(context, ChatId::new(DC_CHAT_ID_DEADDROP), 0, None).await {
+            if let ChatItem::Message { msg_id } = item {
+                let msg = match Message::load_from_db(context, msg_id).await {
+                    Err(e) => {
+                        warn!(context, "can't get msg: {:#}", e);
+                        return Status::RetryLater;
+                    }
+                    Ok(m) => m,
+                };
+                let chat = match Chat::load_from_db(context, msg.chat_id).await {
+                    Err(e) => {
+                        warn!(context, "can't get chat: {:#}", e);
+                        return Status::RetryLater;
+                    }
+                    Ok(c) => c,
+                };
+                match chat.typ {
+                    Chattype::Group | Chattype::Mailinglist => {
+                        if let Ok((_1to1_chat, Blocked::Not)) =
+                            chat::lookup_by_contact_id(context, msg.from_id).await
+                        {
+                            chat.id.unblock(context).await;
+                        }
+                    }
+                    Chattype::Single | Chattype::Undefined => {}
+                }
+            }
+        }
+
         info!(context, "Done fetching existing messages.");
         Status::Finished(Ok(()))
     }
@@ -714,6 +777,7 @@ impl Job {
             // retry. If the message was moved, we will create another
             // job to mark the message as seen later. If it was
             // deleted, there is nothing to do.
+            info!(context, "Can't mark message as seen: No UID");
             ImapActionResult::Failed
         } else {
             imap.set_seen(context, folder, msg.server_uid).await
@@ -805,7 +869,8 @@ async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, fold
         return;
     };
     if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
-        warn!(context, "Could not select {}: {}", mailbox, e);
+        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+        warn!(context, "Could not select {}: {:#}", mailbox, e);
         return;
     }
     match imap.get_all_recipients(context).await {
@@ -893,7 +958,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
         return Ok(None);
     }
 
-    let rendered_msg = match mimefactory.render().await {
+    let rendered_msg = match mimefactory.render(context).await {
         Ok(res) => Ok(res),
         Err(err) => {
             message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
@@ -954,6 +1019,9 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     param.set(Param::File, blob.as_name());
     param.set(Param::Recipients, &recipients);
 
+    msg.subject = rendered_msg.subject.clone();
+    msg.update_subject(context).await;
+
     let job = create(Action::SendMsgToSmtp, msg_id.to_u32() as i32, param, 0)?;
 
     Ok(Some(job))
@@ -1092,7 +1160,7 @@ async fn perform_job_action(
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
         Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::Housekeeping => {
-            sql::housekeeping(context).await;
+            sql::housekeeping(context).await.ok_or_log(context);
             Status::Finished(Ok(()))
         }
     };
@@ -1176,6 +1244,18 @@ pub async fn add(context: &Context, job: Job) {
     }
 }
 
+async fn load_housekeeping_job(context: &Context) -> Option<Job> {
+    let last_time = context.get_config_i64(Config::LastHousekeeping).await;
+
+    let next_time = last_time + (60 * 60 * 24);
+    if next_time <= time() {
+        kill_action(context, Action::Housekeeping).await;
+        Some(Job::new(Action::Housekeeping, 0, Params::new(), 0))
+    } else {
+        None
+    }
+}
+
 /// Load jobs from the database.
 ///
 /// Load jobs for this "[Thread]", i.e. either load SMTP jobs or load
@@ -1189,6 +1269,17 @@ pub(crate) async fn load_next(
 ) -> Option<Job> {
     info!(context, "loading job for {}-thread", thread);
 
+    while !context.sql.is_open().await {
+        // The db is closed, which means that this thread should not be running.
+        // Wait until the db is re-opened (if we returned None, this thread might do further damage)
+        warn!(
+            context,
+            "{}: load_next() was called but the db was not opened, THIS SHOULD NOT HAPPEN. Waiting...",
+            thread
+        );
+        sleep(Duration::from_millis(500)).await;
+    }
+
     let query;
     let params;
     let t = time();
@@ -1292,8 +1383,10 @@ LIMIT 1;
                 } else {
                     Some(job)
                 }
+            } else if let Some(job) = load_imap_deletion_job(context).await.unwrap_or_default() {
+                Some(job)
             } else {
-                load_imap_deletion_job(context).await.unwrap_or_default()
+                load_housekeeping_job(context).await
             }
         }
         Thread::Smtp => job,
@@ -1304,7 +1397,7 @@ LIMIT 1;
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     async fn insert_job(context: &Context, foreign_id: i64) {
         let now = time();
@@ -1333,18 +1426,19 @@ mod tests {
         // fails to load from the database instead of failing to load
         // all jobs.
         let t = TestContext::new().await;
-        insert_job(&t.ctx, -1).await; // This can not be loaded into Job struct.
+        insert_job(&t, -1).await; // This can not be loaded into Job struct.
         let jobs = load_next(
-            &t.ctx,
+            &t,
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
         .await;
-        assert!(jobs.is_none());
+        // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
+        assert!(jobs.unwrap().action == Action::Housekeeping);
 
-        insert_job(&t.ctx, 1).await;
+        insert_job(&t, 1).await;
         let jobs = load_next(
-            &t.ctx,
+            &t,
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
@@ -1356,10 +1450,10 @@ mod tests {
     async fn test_load_next_job_one() {
         let t = TestContext::new().await;
 
-        insert_job(&t.ctx, 1).await;
+        insert_job(&t, 1).await;
 
         let jobs = load_next(
-            &t.ctx,
+            &t,
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )

src/key.rs

@@ -12,7 +12,7 @@ use pgp::types::{KeyTrait, SecretKeyTrait};
 use thiserror::Error;
 
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::KeyGenType;
 use crate::context::Context;
 use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
 use crate::sql;
@@ -407,7 +407,7 @@ impl std::str::FromStr for Fingerprint {
         let hex_repr: String = input
             .to_uppercase()
             .chars()
-            .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
+            .filter(|&c| ('0'..='9').contains(&c) || ('A'..='F').contains(&c))
             .collect();
         let v: Vec<u8> = hex::decode(hex_repr)?;
         let fp = Fingerprint::new(v)?;
@@ -426,7 +426,7 @@ pub enum FingerprintError {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::{alice_keypair, TestContext};
 
     use std::error::Error;
 
@@ -558,31 +558,29 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         let alice = alice_keypair();
         let t = TestContext::new().await;
         t.configure_alice().await;
-        let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
+        let pubkey = SignedPublicKey::load_self(&t).await.unwrap();
         assert_eq!(alice.public, pubkey);
-        let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
+        let seckey = SignedSecretKey::load_self(&t).await.unwrap();
         assert_eq!(alice.secret, seckey);
     }
 
     #[async_std::test]
     async fn test_load_self_generate_public() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
-        let key = SignedPublicKey::load_self(&t.ctx).await;
+        let key = SignedPublicKey::load_self(&t).await;
         assert!(key.is_ok());
     }
 
     #[async_std::test]
     async fn test_load_self_generate_secret() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
-        let key = SignedSecretKey::load_self(&t.ctx).await;
+        let key = SignedSecretKey::load_self(&t).await;
         assert!(key.is_ok());
     }
 
@@ -591,17 +589,17 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         use std::thread;
 
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
-        let ctx = t.ctx.clone();
-        let ctx0 = ctx.clone();
-        let thr0 =
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
-        let ctx1 = ctx;
-        let thr1 =
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
+        let thr0 = {
+            let ctx = t.clone();
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+        };
+        let thr1 = {
+            let ctx = t.clone();
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+        };
         let res0 = thr0.join().unwrap();
         let res1 = thr1.join().unwrap();
         assert_eq!(res0.unwrap(), res1.unwrap());
@@ -618,12 +616,11 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         // Saving the same key twice should result in only one row in
         // the keypairs table.
         let t = TestContext::new().await;
-        let ctx = Arc::new(t.ctx);
+        let ctx = Arc::new(t);
 
-        let ctx1 = ctx.clone();
         let nrows = || async {
-            ctx1.sql
-                .query_get_value::<u32>(&ctx1, "SELECT COUNT(*) FROM keypairs;", paramsv![])
+            ctx.sql
+                .query_get_value::<u32>(&ctx, "SELECT COUNT(*) FROM keypairs;", paramsv![])
                 .await
                 .unwrap()
         };

src/keyring.rs

@@ -62,7 +62,7 @@ where
 mod tests {
     use super::*;
     use crate::key::{SignedPublicKey, SignedSecretKey};
-    use crate::test_utils::*;
+    use crate::test_utils::{alice_keypair, TestContext};
 
     #[test]
     fn test_keyring_add_keys() {
@@ -83,10 +83,10 @@ mod tests {
         t.configure_alice().await;
         let alice = alice_keypair();
 
-        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t).await.unwrap();
         assert_eq!(pub_ring.keys(), [alice.public]);
 
-        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t).await.unwrap();
         assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 }

src/lib.rs

@@ -3,7 +3,9 @@
     clippy::correctness,
     missing_debug_implementations,
     clippy::all,
-    clippy::indexing_slicing
+    clippy::indexing_slicing,
+    clippy::wildcard_imports,
+    clippy::needless_borrow
 )]
 #![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
@@ -72,10 +74,13 @@ pub mod qr;
 pub mod securejoin;
 mod simplify;
 mod smtp;
-pub mod stock;
+pub mod stock_str;
 mod token;
 #[macro_use]
 mod dehtml;
+mod color;
+pub mod html;
+pub mod plaintext;
 
 pub mod dc_receive_imf;
 pub mod dc_tools;

src/location.rs

@@ -1,20 +1,20 @@
 //! Location handling
 
+use anyhow::{ensure, Error};
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::chat::{self, ChatId};
 use crate::config::Config;
-use crate::constants::*;
-use crate::context::*;
-use crate::dc_tools::*;
-use crate::error::{ensure, Error};
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+use crate::context::Context;
+use crate::dc_tools::time;
 use crate::events::EventType;
 use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::param::*;
-use crate::stock::StockMessage;
+use crate::param::Params;
+use crate::stock_str;
 
 /// Location record
 #[derive(Debug, Clone, Default)]
@@ -193,7 +193,8 @@ impl Kml {
 pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
     let now = time();
     if !(seconds < 0 || chat_id.is_special()) {
-        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id).await;
+        let is_sending_locations_before =
+            is_sending_locations_to_chat(context, Some(chat_id)).await;
         if context
             .sql
             .execute(
@@ -212,19 +213,13 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
         {
             if 0 != seconds && !is_sending_locations_before {
                 let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some(
-                    context
-                        .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
-                        .await,
-                );
+                msg.text = Some(stock_str::msg_location_enabled(context).await);
                 msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
                 chat::send_msg(context, chat_id, &mut msg)
                     .await
                     .unwrap_or_default();
             } else if 0 == seconds && is_sending_locations_before {
-                let stock_str = context
-                    .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
-                    .await;
+                let stock_str = stock_str::msg_location_disabled(context).await;
                 chat::add_info_msg(context, chat_id, stock_str).await;
             }
             context.emit_event(EventType::ChatModified(chat_id));
@@ -255,15 +250,29 @@ async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool)
     };
 }
 
-pub async fn is_sending_locations_to_chat(context: &Context, chat_id: ChatId) -> bool {
-    context
-        .sql
-        .exists(
-            "SELECT id  FROM chats  WHERE (? OR id=?)   AND locations_send_until>?;",
-            paramsv![if chat_id.is_unset() { 1 } else { 0 }, chat_id, time()],
-        )
-        .await
-        .unwrap_or_default()
+/// Returns whether `chat_id` or any chat is sending locations.
+///
+/// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
+/// is sending locations.
+pub async fn is_sending_locations_to_chat(context: &Context, chat_id: Option<ChatId>) -> bool {
+    match chat_id {
+        Some(chat_id) => context
+            .sql
+            .exists(
+                "SELECT id  FROM chats  WHERE id=?  AND locations_send_until>?;",
+                paramsv![chat_id, time()],
+            )
+            .await
+            .unwrap_or_default(),
+        None => context
+            .sql
+            .exists(
+                "SELECT id  FROM chats  WHERE locations_send_until>?;",
+                paramsv![time()],
+            )
+            .await
+            .unwrap_or_default(),
+    }
 }
 
 pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
@@ -311,14 +320,22 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
 
 pub async fn get_range(
     context: &Context,
-    chat_id: ChatId,
-    contact_id: u32,
+    chat_id: Option<ChatId>,
+    contact_id: Option<u32>,
     timestamp_from: i64,
     mut timestamp_to: i64,
 ) -> Vec<Location> {
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
+    let (disable_chat_id, chat_id) = match chat_id {
+        Some(chat_id) => (0, chat_id),
+        None => (1, ChatId::new(0)), // this ChatId is unused
+    };
+    let (disable_contact_id, contact_id) = match contact_id {
+        Some(contact_id) => (0, contact_id),
+        None => (1, 0), // this contact_id is unused
+    };
     context
         .sql
         .query_map(
@@ -329,9 +346,9 @@ pub async fn get_range(
              AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
              ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
             paramsv![
-                if chat_id.is_unset() { 1 } else { 0 },
+                disable_chat_id,
                 chat_id,
-                if contact_id == 0 { 1 } else { 0 },
+                disable_contact_id,
                 contact_id as i32,
                 timestamp_from,
                 timestamp_to,
@@ -372,7 +389,12 @@ pub async fn get_range(
 }
 
 fn is_marker(txt: &str) -> bool {
-    txt.len() == 1 && !txt.starts_with(' ')
+    let mut chars = txt.chars();
+    if let Some(c) = chars.next() {
+        !c.is_whitespace() && chars.next().is_none()
+    } else {
+        false
+    }
 }
 
 /// Deletes all locations from the database.
@@ -711,9 +733,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
                 paramsv![chat_id],
             ).await);
 
-            let stock_str = context
-                .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
-                .await;
+            let stock_str = stock_str::msg_location_disabled(context).await;
             chat::add_info_msg(context, chat_id, stock_str).await;
             context.emit_event(EventType::ChatModified(chat_id));
         }
@@ -777,4 +797,13 @@ mod tests {
         assert!(locations_ref[0].accuracy.abs() < f64::EPSILON);
         assert_eq!(locations_ref[0].timestamp, timestamp);
     }
+
+    #[test]
+    fn test_is_marker() {
+        assert!(is_marker("f"));
+        assert!(!is_marker("foo"));
+        assert!(is_marker("🏠"));
+        assert!(!is_marker(" "));
+        assert!(!is_marker("\t"));
+    }
 }

src/log.rs

@@ -1,4 +1,5 @@
-//! # Logging macros
+//! # Logging
+use crate::context::Context;
 
 #[macro_export]
 macro_rules! info {
@@ -58,3 +59,95 @@ macro_rules! emit_event {
         $ctx.emit_event($event);
     };
 }
+
+pub trait LogExt<T, E>
+where
+    Self: std::marker::Sized,
+{
+    #[track_caller]
+    fn log_err_inner(self, context: &Context, msg: Option<&str>) -> Result<T, E>;
+
+    /// Emits a warning if the receiver contains an Err value.
+    ///
+    /// Thanks to the [track_caller](https://blog.rust-lang.org/2020/08/27/Rust-1.46.0.html#track_caller)
+    /// feature, the location of the caller is printed to the log, just like with the warn!() macro.
+    ///
+    /// Unfortunately, the track_caller feature does not work on async functions (as of Rust 1.50).
+    /// Once it is, you can add `#[track_caller]` to helper functions that use one of the log helpers here
+    /// so that the location of the caller can be seen in the log. (this won't work with the macros,
+    /// like warn!(), since the file!() and line!() macros don't work with track_caller)  
+    /// See https://github.com/rust-lang/rust/issues/78840 for progress on this.
+    #[track_caller]
+    fn log_err(self, context: &Context, msg: &str) -> Result<T, E> {
+        self.log_err_inner(context, Some(msg))
+    }
+
+    /// Emits a warning if the receiver contains an Err value and returns an [`Option<T>`].
+    ///
+    /// Example:
+    /// ```text
+    /// if let Err(e) = do_something() {
+    ///     warn!(context, "{:#}", e);
+    /// }
+    /// ```
+    /// is equivalent to:
+    /// ```text
+    /// do_something().ok_or_log(context);
+    /// ```
+    ///
+    /// For a note on the `track_caller` feature, see the doc comment on `log_err()`.
+    #[track_caller]
+    fn ok_or_log(self, context: &Context) -> Option<T> {
+        self.log_err_inner(context, None).ok()
+    }
+
+    /// Like `ok_or_log()`, but you can pass an extra message that is prepended in the log.
+    ///
+    /// Example:
+    /// ```text
+    /// if let Err(e) = do_something() {
+    ///     warn!(context, "Something went wrong: {:#}", e);
+    /// }
+    /// ```
+    /// is equivalent to:
+    /// ```text
+    /// do_something().ok_or_log_msg(context, "Something went wrong");
+    /// ```
+    /// and is also equivalent to:
+    /// ```text
+    /// use anyhow::Context as _;
+    /// do_something().context("Something went wrong").ok_or_log(context);
+    /// ```
+    ///
+    /// For a note on the `track_caller` feature, see the doc comment on `log_err()`.
+    #[track_caller]
+    fn ok_or_log_msg(self, context: &Context, msg: &'static str) -> Option<T> {
+        self.log_err_inner(context, Some(msg)).ok()
+    }
+}
+
+impl<T: Default, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
+    #[track_caller]
+    fn log_err_inner(self, context: &Context, msg: Option<&str>) -> Result<T, E> {
+        if let Err(e) = &self {
+            let location = std::panic::Location::caller();
+
+            let separator = if msg.is_none() { "" } else { ": " };
+            let msg = msg.unwrap_or_default();
+
+            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+            let full = format!(
+                "{file}:{line}: {msg}{separator}{e:#}",
+                file = location.file(),
+                line = location.line(),
+                msg = msg,
+                separator = separator,
+                e = e
+            );
+            // We can't use the warn!() macro here as the file!() and line!() macros
+            // don't work with #[track_caller]
+            emit_event!(context, crate::EventType::Warning(full));
+        };
+        self
+    }
+}

src/login_param.rs

@@ -3,6 +3,7 @@
 use std::borrow::Cow;
 use std::fmt;
 
+use crate::provider::{get_provider_by_id, Provider};
 use crate::{context::Context, provider::Socket};
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
@@ -48,6 +49,7 @@ pub struct LoginParam {
     pub imap: ServerLoginParam,
     pub smtp: ServerLoginParam,
     pub server_flags: i32,
+    pub provider: Option<&'static Provider>,
 }
 
 impl LoginParam {
@@ -130,6 +132,12 @@ impl LoginParam {
             .await
             .unwrap_or_default();
 
+        let key = format!("{}provider", prefix);
+        let provider = sql
+            .get_raw_config(context, key)
+            .await
+            .and_then(|provider_id| get_provider_by_id(&provider_id));
+
         LoginParam {
             addr,
             imap: ServerLoginParam {
@@ -148,6 +156,7 @@ impl LoginParam {
                 security: send_security,
                 certificate_checks: smtp_certificate_checks,
             },
+            provider,
             server_flags,
         }
     }
@@ -216,6 +225,11 @@ impl LoginParam {
         sql.set_raw_config_int(context, key, self.server_flags)
             .await?;
 
+        if let Some(provider) = self.provider {
+            let key = format!("{}provider", prefix);
+            sql.set_raw_config(context, key, Some(provider.id)).await?;
+        }
+
         Ok(())
     }
 }

src/lot.rs

@@ -22,7 +22,9 @@ pub struct Lot {
 }
 
 #[repr(u8)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 pub enum Meaning {
     None = 0,
     Text1Draft = 1,
@@ -67,7 +69,9 @@ impl Lot {
 }
 
 #[repr(i32)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 pub enum LotState {
     // Default
     Undefined = 0,

src/oauth2.rs

@@ -1,14 +1,12 @@
 //! OAuth 2 module
 
-use regex::Regex;
 use std::collections::HashMap;
 
-use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::time;
 use crate::provider;
 use crate::provider::Oauth2Authorizer;
 
@@ -19,7 +17,6 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
-    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -29,11 +26,8 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
-    mx_pattern: None,
 };
 
-const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
-
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -41,7 +35,6 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
-    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -276,47 +269,16 @@ impl Oauth2 {
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            if let Some(provider) = provider::get_provider_info(&addr_normalized) {
-                match &provider.oauth2_authorizer {
-                    Some(Oauth2Authorizer::Gmail) => Some(OAUTH2_GMAIL),
-                    Some(Oauth2Authorizer::Yandex) => Some(OAUTH2_YANDEX),
-                    None => None, // provider known to not support oauth2, no mx-lookup required
-                }
-            } else {
-                Oauth2::lookup_mx(domain).await
-            }
-        } else {
-            None
-        }
-    }
-
-    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
-        if let Ok(resolver) = resolver(
-            config::ResolverConfig::default(),
-            config::ResolverOpts::default(),
-        )
-        .await
-        {
-            for provider in OAUTH2_PROVIDERS.iter() {
-                if let Some(pattern) = provider.mx_pattern {
-                    let re = Regex::new(pattern).unwrap();
-
-                    let mut fqdn: String = String::from(domain.as_ref());
-                    if !fqdn.ends_with('.') {
-                        fqdn.push('.');
-                    }
-
-                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
-                        for rr in res.iter() {
-                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
-                                return Some(provider.clone());
-                            }
-                        }
-                    }
-                }
+            if let Some(oauth2_authorizer) = provider::get_provider_info(domain)
+                .await
+                .and_then(|provider| provider.oauth2_authorizer.as_ref())
+            {
+                return Some(match oauth2_authorizer {
+                    Oauth2Authorizer::Gmail => OAUTH2_GMAIL,
+                    Oauth2Authorizer::Yandex => OAUTH2_YANDEX,
+                });
             }
         }
-
         None
     }
 
@@ -386,7 +348,7 @@ fn normalize_addr(addr: &str) -> &str {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_normalize_addr() {

src/param.rs

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
 use std::fmt;
 use std::str;
 
+use anyhow::{bail, Error};
 use async_std::path::PathBuf;
 use itertools::Itertools;
 use num_traits::FromPrimitive;
@@ -9,7 +10,6 @@ use serde::{Deserialize, Serialize};
 
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
-use crate::error::{self, bail};
 use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 
@@ -22,6 +22,11 @@ pub enum Param {
     /// For messages and jobs
     File = b'f',
 
+    /// For messages: This name should be shown instead of contact.get_display_name()
+    /// (used if this is a mailinglist
+    /// or explictly set using set_override_sender_name(), eg. by bots)
+    OverrideSenderDisplayname = b'O',
+
     /// For Messages
     Width = b'w',
 
@@ -34,6 +39,11 @@ pub enum Param {
     /// For Messages
     MimeType = b'm',
 
+    /// For Messages: HTML to be written to the database and to be send.
+    /// `SendHtml` param is not used for received messages.
+    /// Use `MsgId::get_html()` to get HTML of received messages.
+    SendHtml = b'T',
+
     /// For Messages: message is encrypted, outgoing: guarantee E2EE or the message is not send
     GuaranteeE2ee = b'c',
 
@@ -50,7 +60,8 @@ pub enum Param {
     /// For Messages
     WantsMdn = b'r',
 
-    /// For Messages
+    /// For Messages: unset or 0=not forwarded,
+    /// 1=forwarded from unknown msg_id, >9 forwarded from msg_id
     Forwarded = b'a',
 
     /// For Messages: quoted text.
@@ -115,21 +126,14 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
-    /// For Chats: So that on sending a new message we can sent the subject to "Re: <last subject>"
+    /// For Chats: On sending a new message we set the subject to "Re: <last subject>".
+    /// Usually we just use the subject of the parent message, but if the parent message
+    /// is deleted, we use the LastSubject of the chat.
     LastSubject = b't',
 
     /// For Chats
     Devicetalk = b'D',
 
-    /// For QR
-    Auth = b's',
-
-    /// For QR
-    GroupId = b'x',
-
-    /// For QR
-    GroupName = b'g',
-
     /// For MDN-sending job
     MsgId = b'I',
 }
@@ -162,7 +166,7 @@ impl fmt::Display for Params {
 }
 
 impl str::FromStr for Params {
-    type Err = error::Error;
+    type Err = Error;
 
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let mut inner = BTreeMap::new();
@@ -379,7 +383,7 @@ mod tests {
     use async_std::fs;
     use async_std::path::Path;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_dc_param() {
@@ -425,18 +429,10 @@ mod tests {
         assert_eq!(params.to_string().parse::<Params>().unwrap(), params);
     }
 
-    #[test]
-    fn test_regression() {
-        let p1: Params = "a=cli%40deltachat.de\nn=\ni=TbnwJ6lSvD5\ns=0ejvbdFSQxB"
-            .parse()
-            .unwrap();
-        assert_eq!(p1.get(Param::Forwarded).unwrap(), "cli%40deltachat.de");
-    }
-
     #[async_std::test]
     async fn test_params_file_fs_path() {
         let t = TestContext::new().await;
-        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
+        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t, "/foo/bar/baz").unwrap() {
             assert_eq!(p, Path::new("/foo/bar/baz"));
         } else {
             panic!("Wrong enum variant");
@@ -446,7 +442,7 @@ mod tests {
     #[async_std::test]
     async fn test_params_file_blob() {
         let t = TestContext::new().await;
-        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
+        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t, "$BLOBDIR/foo").unwrap() {
             assert_eq!(b.as_name(), "$BLOBDIR/foo");
         } else {
             panic!("Wrong enum variant");
@@ -461,51 +457,33 @@ mod tests {
         let mut p = Params::new();
         p.set(Param::File, fname.to_str().unwrap());
 
-        let file = p.get_file(Param::File, &t.ctx).unwrap().unwrap();
+        let file = p.get_file(Param::File, &t).unwrap().unwrap();
         assert_eq!(file, ParamsFile::FsPath(fname.clone().into()));
 
-        let path: PathBuf = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        let path: PathBuf = p.get_path(Param::File, &t).unwrap().unwrap();
         let fname: PathBuf = fname.into();
         assert_eq!(path, fname);
 
         // Blob does not exist yet, expect BlobError.
-        let err = p.get_blob(Param::File, &t.ctx, false).await.unwrap_err();
+        let err = p.get_blob(Param::File, &t, false).await.unwrap_err();
         match err {
             BlobError::WrongBlobdir { .. } => (),
             _ => panic!("wrong error type/variant: {:?}", err),
         }
 
         fs::write(fname, b"boo").await.unwrap();
-        let blob = p
-            .get_blob(Param::File, &t.ctx, true)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(
-            blob,
-            BlobObject::from_name(&t.ctx, "foo".to_string()).unwrap()
-        );
+        let blob = p.get_blob(Param::File, &t, true).await.unwrap().unwrap();
+        assert_eq!(blob, BlobObject::from_name(&t, "foo".to_string()).unwrap());
 
         // Blob in blobdir, expect blob.
-        let bar_path = t.ctx.get_blobdir().join("bar");
+        let bar_path = t.get_blobdir().join("bar");
         p.set(Param::File, bar_path.to_str().unwrap());
-        let blob = p
-            .get_blob(Param::File, &t.ctx, false)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(
-            blob,
-            BlobObject::from_name(&t.ctx, "bar".to_string()).unwrap()
-        );
+        let blob = p.get_blob(Param::File, &t, false).await.unwrap().unwrap();
+        assert_eq!(blob, BlobObject::from_name(&t, "bar".to_string()).unwrap());
 
         p.remove(Param::File);
-        assert!(p.get_file(Param::File, &t.ctx).unwrap().is_none());
-        assert!(p.get_path(Param::File, &t.ctx).unwrap().is_none());
-        assert!(p
-            .get_blob(Param::File, &t.ctx, false)
-            .await
-            .unwrap()
-            .is_none());
+        assert!(p.get_file(Param::File, &t).unwrap().is_none());
+        assert!(p.get_path(Param::File, &t).unwrap().is_none());
+        assert!(p.get_blob(Param::File, &t, false).await.unwrap().is_none());
     }
 }

src/peerstate.rs

@@ -1,18 +1,19 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
+
 use std::collections::HashSet;
 use std::fmt;
 
+use anyhow::{bail, Result};
 use num_traits::FromPrimitive;
 
-use crate::aheader::*;
+use crate::aheader::{Aheader, EncryptPreference};
 use crate::chat;
 use crate::constants::Blocked;
 use crate::context::Context;
-use crate::error::{bail, Result};
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 #[derive(Debug)]
 pub enum PeerstateKeyType {
@@ -29,8 +30,7 @@ pub enum PeerstateVerifiedStatus {
 }
 
 /// Peerstate represents the state of an Autocrypt peer.
-pub struct Peerstate<'a> {
-    pub context: &'a Context,
+pub struct Peerstate {
     pub addr: String,
     pub last_seen: i64,
     pub last_seen_autocrypt: i64,
@@ -46,7 +46,7 @@ pub struct Peerstate<'a> {
     pub fingerprint_changed: bool,
 }
 
-impl<'a> PartialEq for Peerstate<'a> {
+impl PartialEq for Peerstate {
     fn eq(&self, other: &Peerstate) -> bool {
         self.addr == other.addr
             && self.last_seen == other.last_seen
@@ -64,9 +64,9 @@ impl<'a> PartialEq for Peerstate<'a> {
     }
 }
 
-impl<'a> Eq for Peerstate<'a> {}
+impl Eq for Peerstate {}
 
-impl<'a> fmt::Debug for Peerstate<'a> {
+impl fmt::Debug for Peerstate {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         f.debug_struct("Peerstate")
             .field("addr", &self.addr)
@@ -93,10 +93,9 @@ pub enum ToSave {
     All = 0x02,
 }
 
-impl<'a> Peerstate<'a> {
-    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
+impl Peerstate {
+    pub fn from_header(header: &Aheader, message_time: i64) -> Self {
         Peerstate {
-            context,
             addr: header.addr.clone(),
             last_seen: message_time,
             last_seen_autocrypt: message_time,
@@ -113,13 +112,20 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn from_gossip(context: &'a Context, gossip_header: &Aheader, message_time: i64) -> Self {
+    pub fn from_gossip(gossip_header: &Aheader, message_time: i64) -> Self {
         Peerstate {
-            context,
             addr: gossip_header.addr.clone(),
             last_seen: 0,
             last_seen_autocrypt: 0,
-            prefer_encrypt: Default::default(),
+
+            // Non-standard extension. According to Autocrypt 1.1.0 gossip headers SHOULD NOT
+            // contain encryption preference.
+            //
+            // Delta Chat includes encryption preference to ensure new users introduced to a group
+            // learn encryption preferences of other members immediately and don't send unencrypted
+            // messages to a group where everyone prefers encryption.
+            prefer_encrypt: gossip_header.prefer_encrypt,
+
             public_key: None,
             public_key_fingerprint: None,
             gossip_key: Some(gossip_header.public_key.clone()),
@@ -132,7 +138,7 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub async fn from_addr(context: &'a Context, addr: &str) -> Result<Option<Peerstate<'a>>> {
+    pub async fn from_addr(context: &Context, addr: &str) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint \
@@ -142,10 +148,10 @@ impl<'a> Peerstate<'a> {
     }
 
     pub async fn from_fingerprint(
-        context: &'a Context,
+        context: &Context,
         _sql: &Sql,
         fingerprint: &Fingerprint,
-    ) -> Result<Option<Peerstate<'a>>> {
+    ) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint \
@@ -158,10 +164,10 @@ impl<'a> Peerstate<'a> {
     }
 
     async fn from_stmt(
-        context: &'a Context,
+        context: &Context,
         query: &str,
         params: Vec<&dyn crate::ToSql>,
-    ) -> Result<Option<Peerstate<'a>>> {
+    ) -> Result<Option<Peerstate>> {
         let peerstate = context
             .sql
             .query_row_optional(query, params, |row| {
@@ -172,7 +178,6 @@ impl<'a> Peerstate<'a> {
                 */
 
                 let res = Peerstate {
-                    context,
                     addr: row.get(0)?,
                     last_seen: row.get(1)?,
                     last_seen_autocrypt: row.get(2)?,
@@ -272,9 +277,7 @@ impl<'a> Peerstate<'a> {
                         .await
                         .unwrap_or_default();
 
-                let msg = context
-                    .stock_string_repl_str(StockMessage::ContactSetupChanged, self.addr.clone())
-                    .await;
+                let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
 
                 chat::add_info_msg(context, contact_chat_id, msg).await;
                 emit_event!(context, EventType::ChatModified(contact_chat_id));
@@ -427,31 +430,31 @@ impl<'a> Peerstate<'a> {
         if self.to_save == Some(ToSave::All) || create {
             sql.execute(
                 if create {
-                "INSERT INTO acpeerstates (last_seen, last_seen_autocrypt, prefer_encrypted, \
+                    "INSERT INTO acpeerstates (last_seen, last_seen_autocrypt, prefer_encrypted, \
                  public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                  verified_key, verified_key_fingerprint, addr \
                 ) VALUES(?,?,?,?,?,?,?,?,?,?,?)"
                 } else {
-                "UPDATE acpeerstates \
+                    "UPDATE acpeerstates \
                  SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
                  public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
                  verified_key=?, verified_key_fingerprint=? \
                  WHERE addr=?"
                 },
                 paramsv![
-                    self.last_seen,
-                    self.last_seen_autocrypt,
-                    self.prefer_encrypt as i64,
-                    self.public_key.as_ref().map(|k| k.to_bytes()),
-                    self.gossip_timestamp,
-                    self.gossip_key.as_ref().map(|k| k.to_bytes()),
-                    self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
-                    self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
-                    self.verified_key.as_ref().map(|k| k.to_bytes()),
-                    self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
-                    self.addr,
+                self.last_seen,
+                self.last_seen_autocrypt,
+                self.prefer_encrypt as i64,
+                self.public_key.as_ref().map(|k| k.to_bytes()),
+                self.gossip_timestamp,
+                self.gossip_key.as_ref().map(|k| k.to_bytes()),
+                self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                self.verified_key.as_ref().map(|k| k.to_bytes()),
+                self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                self.addr,
                 ],
-            ).await?;
+                ).await?;
         } else if self.to_save == Some(ToSave::Timestamps) {
             sql.execute(
                 "UPDATE acpeerstates SET last_seen=?, last_seen_autocrypt=?, gossip_timestamp=? \
@@ -487,7 +490,7 @@ impl From<crate::key::FingerprintError> for rusqlite::Error {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::alice_keypair;
     use pretty_assertions::assert_eq;
 
     #[async_std::test]
@@ -498,7 +501,6 @@ mod tests {
         let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -542,7 +544,6 @@ mod tests {
         let pub_key = alice_keypair().public;
 
         let peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -576,7 +577,6 @@ mod tests {
         let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -637,13 +637,11 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_degrade_reordering() {
-        let context = crate::test_utils::TestContext::new().await.ctx;
         let addr = "example@example.org";
         let pub_key = alice_keypair().public;
         let header = Aheader::new(addr.to_string(), pub_key, EncryptPreference::Mutual);
 
         let mut peerstate = Peerstate {
-            context: &context,
             addr: addr.to_string(),
             last_seen: 0,
             last_seen_autocrypt: 0,

src/pgp.rs

@@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashSet};
 use std::io;
 use std::io::Cursor;
 
+use anyhow::{bail, ensure, format_err, Result};
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
@@ -17,7 +18,6 @@ use rand::{thread_rng, CryptoRng, Rng};
 
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
-use crate::error::{bail, ensure, format_err, Result};
 use crate::key::{DcKey, Fingerprint};
 use crate::keyring::Keyring;
 
@@ -380,7 +380,7 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::{alice_keypair, bob_keypair};
     use once_cell::sync::Lazy;
 
     #[test]

src/plaintext.rs

@@ -0,0 +1,252 @@
+///! Handle plain text together with some attributes.
+use crate::simplify::split_lines;
+use once_cell::sync::Lazy;
+
+#[derive(Debug)]
+pub struct PlainText {
+    pub text: String,
+
+    /// Text may "flowed" as defined in [RFC 2646](https://tools.ietf.org/html/rfc2646).
+    /// At a glance, that means, if a line ends with a space, it is merged with the next one
+    /// and the first leading spaces is ignored
+    /// (to allow lines starting with `>` that normally indicates a quote)
+    pub flowed: bool,
+
+    /// If set together with "flowed",
+    /// The space indicating merging two lines is removed.
+    pub delsp: bool,
+}
+
+impl PlainText {
+    /// Convert plain text to HTML.
+    /// The function handles quotes, links, fixed and floating text paragraphs.
+    pub async fn to_html(&self) -> String {
+        static LINKIFY_MAIL_RE: Lazy<regex::Regex> =
+            Lazy::new(|| regex::Regex::new(r#"\b([\w.\-+]+@[\w.\-]+)\b"#).unwrap());
+
+        static LINKIFY_URL_RE: Lazy<regex::Regex> = Lazy::new(|| {
+            regex::Regex::new(r#"\b((http|https|ftp|ftps):[\w.,:;$/@!?&%\-~=#+]+)"#).unwrap()
+        });
+
+        let lines = split_lines(&self.text);
+
+        let mut ret =
+            "<!DOCTYPE html>\n<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" /></head><body>\n".to_string();
+
+        for line in lines {
+            let is_quote = line.starts_with('>');
+
+            // we need to do html-entity-encoding after linkify, as otherwise encapsulated links
+            // as <http://example.org> cannot be handled correctly
+            // (they would become &lt;http://example.org&gt; where the trailing &gt; would become a valid url part).
+            // to avoid double encoding, we escape our html-entities by \r that must not be used in the string elsewhere.
+            let line = line.to_string().replace("\r", "");
+
+            let mut line = LINKIFY_MAIL_RE
+                .replace_all(&*line, "\rLTa href=\rQUOTmailto:$1\rQUOT\rGT$1\rLT/a\rGT")
+                .as_ref()
+                .to_string();
+
+            line = LINKIFY_URL_RE
+                .replace_all(&*line, "\rLTa href=\rQUOT$1\rQUOT\rGT$1\rLT/a\rGT")
+                .as_ref()
+                .to_string();
+
+            // encode html-entities after linkify the raw string
+            line = escaper::encode_minimal(&line);
+
+            // make our escaped html-entities real after encoding all others
+            line = line.replace("\rLT", "<");
+            line = line.replace("\rGT", ">");
+            line = line.replace("\rQUOT", "\"");
+
+            if self.flowed {
+                // flowed text as of RFC 3676 -
+                // a leading space shall be removed
+                // and is only there to allow > at the beginning of a line that is no quote.
+                line = line.strip_prefix(" ").unwrap_or(&line).to_string();
+                if is_quote {
+                    line = "<em>".to_owned() + &line + "</em>";
+                }
+
+                // a trailing space indicates that the line can be merged with the next one;
+                // for sake of simplicity, we skip merging for quotes (quotes may be combined with
+                // delsp, so `> >` is different from `>>` etc. see RFC 3676 for details)
+                if line.ends_with(' ') && !is_quote {
+                    if self.delsp {
+                        line.pop();
+                    }
+                } else {
+                    line += "<br/>\n";
+                }
+            } else {
+                // normal, fixed text
+                if is_quote {
+                    line = "<em>".to_owned() + &line + "</em>";
+                }
+                line += "<br/>\n";
+            }
+
+            ret += &*line;
+        }
+        ret += "</body></html>\n";
+        ret
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[async_std::test]
+    async fn test_plain_to_html() {
+        let html = PlainText {
+            text: r##"line 1
+line 2
+line with https://link-mid-of-line.org and http://link-end-of-line.com/file?foo=bar%20
+http://link-at-start-of-line.org
+"##
+            .to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line 1<br/>
+line 2<br/>
+line with <a href="https://link-mid-of-line.org">https://link-mid-of-line.org</a> and <a href="http://link-end-of-line.com/file?foo=bar%20">http://link-end-of-line.com/file?foo=bar%20</a><br/>
+<a href="http://link-at-start-of-line.org">http://link-at-start-of-line.org</a><br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_encapsulated() {
+        let html = PlainText {
+            text: r#"line with <http://encapsulated.link/?foo=_bar> here!"#.to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line with &lt;<a href="http://encapsulated.link/?foo=_bar">http://encapsulated.link/?foo=_bar</a>&gt; here!<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_nolink() {
+        let html = PlainText {
+            text: r#"line with nohttp://no.link here"#.to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line with nohttp://no.link here<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_mailto() {
+        let html = PlainText {
+            text: r#"just an address: foo@bar.org another@one.de"#.to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+just an address: <a href="mailto:foo@bar.org">foo@bar.org</a> <a href="mailto:another@one.de">another@one.de</a><br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_flowed() {
+        let html = PlainText {
+            text: "line \nstill line\n>quote \n>still quote\n >no quote".to_string(),
+            flowed: true,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line still line<br/>
+<em>&gt;quote </em><br/>
+<em>&gt;still quote</em><br/>
+&gt;no quote<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_flowed_delsp() {
+        let html = PlainText {
+            text: "line \nstill line\n>quote \n>still quote\n >no quote".to_string(),
+            flowed: true,
+            delsp: true,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+linestill line<br/>
+<em>&gt;quote </em><br/>
+<em>&gt;still quote</em><br/>
+&gt;no quote<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_fixed() {
+        let html = PlainText {
+            text: "line \nstill line\n>quote \n>still quote\n >no quote".to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line <br/>
+still line<br/>
+<em>&gt;quote </em><br/>
+<em>&gt;still quote</em><br/>
+ &gt;no quote<br/>
+</body></html>
+"#
+        );
+    }
+}

src/provider/data.rs

@@ -3,13 +3,14 @@
 use crate::provider::Protocol::*;
 use crate::provider::Socket::*;
 use crate::provider::UsernamePattern::*;
-use crate::provider::*;
+use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status};
 use std::collections::HashMap;
 
 use once_cell::sync::Lazy;
 
 // aktivix.org.md: aktivix.org
 static P_AKTIVIX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "aktivix.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -31,27 +32,33 @@ static P_AKTIVIX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // aol.md: aol.com
 static P_AOL: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "aol",
     status: Status::PREPARATION,
     before_login_hint: "To log in to AOL with Delta Chat, you need to set up an app password in the AOL web interface.",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/aol",
     server: vec![
+        Server { protocol: IMAP, socket: SSL, hostname: "imap.aol.com", port: 993, username_pattern: EMAIL },
+        Server { protocol: SMTP, socket: SSL, hostname: "smtp.aol.com", port: 465, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
 });
 
 // arcor.de.md: arcor.de
 static P_ARCOR_DE: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "arcor.de",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -73,12 +80,14 @@ static P_ARCOR_DE: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // autistici.org.md: autistici.org
 static P_AUTISTICI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "autistici.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -100,12 +109,14 @@ static P_AUTISTICI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // bluewin.ch.md: bluewin.ch
 static P_BLUEWIN_CH: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "bluewin.ch",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -127,12 +138,14 @@ static P_BLUEWIN_CH: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // buzon.uy.md: buzon.uy
 static P_BUZON_UY: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "buzon.uy",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -141,25 +154,27 @@ static P_BUZON_UY: Lazy<Provider> = Lazy::new(|| Provider {
         Server {
             protocol: IMAP,
             socket: STARTTLS,
-            hostname: "buzon.uy",
+            hostname: "mail.buzon.uy",
             port: 143,
             username_pattern: EMAIL,
         },
         Server {
             protocol: SMTP,
             socket: STARTTLS,
-            hostname: "buzon.uy",
+            hostname: "mail.buzon.uy",
             port: 587,
             username_pattern: EMAIL,
         },
     ],
     config_defaults: None,
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // chello.at.md: chello.at
 static P_CHELLO_AT: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "chello.at",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -181,36 +196,42 @@ static P_CHELLO_AT: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // comcast.md: xfinity.com, comcast.net
 static P_COMCAST: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "comcast",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/comcast",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // dismail.de.md: dismail.de
 static P_DISMAIL_DE: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "dismail.de",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/dismail-de",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // disroot.md: disroot.org
 static P_DISROOT: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "disroot",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -218,11 +239,13 @@ static P_DISROOT: Lazy<Provider> = Lazy::new(|| Provider {
     server: vec![],
     config_defaults: None,
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // dubby.org.md: dubby.org
 static P_DUBBY_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "dubby.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -269,12 +292,28 @@ static P_DUBBY_ORG: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ]),
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
-// example.com.md: example.com, example.org
+// espiv.net.md: espiv.net
+static P_ESPIV_NET: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "espiv.net",
+    status: Status::OK,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/espiv-net",
+    server: vec![],
+    config_defaults: None,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
+    oauth2_authorizer: None,
+});
+
+// example.com.md: example.com, example.org, example.net
 static P_EXAMPLE_COM: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "example.com",
     status: Status::BROKEN,
     before_login_hint: "Hush this provider doesn't exist!",
     after_login_hint: "This provider doesn't really exist, so you can't use it :/ If you need an email provider for Delta Chat, take a look at providers.delta.chat!",
@@ -284,13 +323,15 @@ static P_EXAMPLE_COM: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
 });
 
 // fastmail.md: fastmail.com
 static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "fastmail",
     status: Status::PREPARATION,
     before_login_hint:
         "You must create an app-specific password for Delta Chat before you can log in.",
@@ -298,13 +339,15 @@ static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
     overview_page: "https://providers.delta.chat/fastmail",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // firemail.de.md: firemail.at, firemail.de
 static P_FIREMAIL_DE: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "firemail.de",
     status: Status::PREPARATION,
     before_login_hint: "Firemail erlaubt nur bei bezahlten Accounts den vollen Zugriff auf das E-Mail-Protokoll. Wenn Sie nicht für Firemail bezahlen, verwenden Sie bitte einen anderen E-Mail-Anbieter.",
     after_login_hint: "Leider schränkt Firemail die maximale Gruppengröße ein. Je nach Bezahlmodell sind nur 5 bis 30 Gruppenmitglieder erlaubt.",
@@ -312,13 +355,15 @@ static P_FIREMAIL_DE: Lazy<Provider> = Lazy::new(|| {
     server: vec![
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
 });
 
 // five.chat.md: five.chat
 static P_FIVE_CHAT: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "five.chat",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -343,11 +388,13 @@ static P_FIVE_CHAT: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ]),
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // freenet.de.md: freenet.de
 static P_FREENET_DE: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "freenet.de",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -369,13 +416,15 @@ static P_FREENET_DE: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
-// gmail.md: gmail.com, googlemail.com
+// gmail.md: gmail.com, googlemail.com, google.com
 static P_GMAIL: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "gmail",
     status: Status::PREPARATION,
     before_login_hint: "For Gmail accounts, you need to create an app-password if you have \"2-Step Verification\" enabled. If this setting is not available, you need to enable \"less secure apps\".",
     after_login_hint: "",
@@ -386,12 +435,14 @@ static P_GMAIL: Lazy<Provider> = Lazy::new(|| {
     ],
     config_defaults: None,
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: Some(Oauth2Authorizer::Gmail),
 }
 });
 
 // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
 static P_GMX_NET: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "gmx.net",
     status: Status::PREPARATION,
     before_login_hint: "You must allow IMAP access to your account before you can login.",
     after_login_hint: "",
@@ -420,12 +471,14 @@ static P_GMX_NET: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // hermes.radio.md: hermes.radio
 static P_HERMES_RADIO: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "hermes.radio",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -450,12 +503,14 @@ static P_HERMES_RADIO: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ]),
     strict_tls: false,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // hey.com.md: hey.com
 static P_HEY_COM: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "hey.com",
     status: Status::BROKEN,
     before_login_hint: "hey.com does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to hey.com.",
     after_login_hint: "",
@@ -463,25 +518,29 @@ static P_HEY_COM: Lazy<Provider> = Lazy::new(|| {
     server: vec![
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
 });
 
 // i.ua.md: i.ua
 static P_I_UA: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "i.ua",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/i-ua",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // icloud.md: icloud.com, me.com, mac.com
 static P_ICLOUD: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "icloud",
     status: Status::PREPARATION,
     before_login_hint:
         "You must create an app-specific password for Delta Chat before you can login.",
@@ -504,48 +563,56 @@ static P_ICLOUD: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // kolst.com.md: kolst.com
 static P_KOLST_COM: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "kolst.com",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/kolst-com",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // kontent.com.md: kontent.com
 static P_KONTENT_COM: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "kontent.com",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/kontent-com",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // mail.ru.md: mail.ru, inbox.ru, bk.ru, list.ru
 static P_MAIL_RU: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "mail.ru",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/mail-ru",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // mailbox.org.md: mailbox.org, secure.mailbox.org
 static P_MAILBOX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "mailbox.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -553,12 +620,14 @@ static P_MAILBOX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     server: vec![],
     config_defaults: None,
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // nauta.cu.md: nauta.cu
 static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "nauta.cu",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "Atención - con nauta.cu, puede enviar mensajes sólo a un máximo de 20 personas a la vez. En grupos más grandes, no puede enviar mensajes o abandonar el grupo.",
@@ -575,9 +644,10 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| {
         ConfigDefault { key: Config::MvboxMove, value: "0" },
         ConfigDefault { key: Config::E2eeEnabled, value: "0" },
         ConfigDefault { key: Config::MediaQuality, value: "1" },
-        ConfigDefault { key: Config::FetchExisting, value: "0" },
+        ConfigDefault { key: Config::FetchExistingMsgs, value: "0" },
     ]),
     strict_tls: false,
+    max_smtp_rcpt_to: Some(20),
     oauth2_authorizer: None,
 }
 });
@@ -585,6 +655,7 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| {
 // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
 static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "outlook.com",
     status: Status::BROKEN,
     before_login_hint: "Outlook.com email addresses will not work as expected as these servers remove some important transport information. Hopefully sooner or later there will be a fix, for now we suggest to use another email address.",
     after_login_hint: "Outlook.com email addresses will not work as expected as these servers remove some important transport information. Unencrypted 1-on-1 chats kind of work, but groups and encryption don't. Hopefully sooner or later there will be a fix, for now we suggest to use another email address.",
@@ -594,13 +665,15 @@ static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
 });
 
 // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
 static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "posteo",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -623,12 +696,14 @@ static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
     ],
     config_defaults: None,
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // protonmail.md: protonmail.com, protonmail.ch
 static P_PROTONMAIL: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "protonmail",
     status: Status::BROKEN,
     before_login_hint: "Protonmail does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to Protonmail.",
     after_login_hint: "To use Delta Chat with Protonmail, the IMAP bridge must be running in the background. If you have connectivity issues, double check whether it works as expected.",
@@ -636,13 +711,15 @@ static P_PROTONMAIL: Lazy<Provider> = Lazy::new(|| {
     server: vec![
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
 });
 
 // riseup.net.md: riseup.net
 static P_RISEUP_NET: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "riseup.net",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -650,23 +727,27 @@ static P_RISEUP_NET: Lazy<Provider> = Lazy::new(|| Provider {
     server: vec![],
     config_defaults: None,
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // rogers.com.md: rogers.com
 static P_ROGERS_COM: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "rogers.com",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/rogers-com",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // systemli.org.md: systemli.org
 static P_SYSTEMLI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "systemli.org",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -674,26 +755,32 @@ static P_SYSTEMLI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     server: vec![],
     config_defaults: None,
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // t-online.md: t-online.de, magenta.de
 static P_T_ONLINE: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "t-online",
     status: Status::PREPARATION,
     before_login_hint: "To use Delta Chat with a T-Online email address, you need to create an app password in the web interface.",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/t-online",
     server: vec![
+        Server { protocol: IMAP, socket: SSL, hostname: "secureimap.t-online.de", port: 993, username_pattern: EMAIL },
+        Server { protocol: SMTP, socket: SSL, hostname: "securesmtp.t-online.de", port: 465, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
 });
 
 // testrun.md: testrun.org
 static P_TESTRUN: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "testrun",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -740,11 +827,13 @@ static P_TESTRUN: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ]),
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // tiscali.it.md: tiscali.it
 static P_TISCALI_IT: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "tiscali.it",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -766,24 +855,28 @@ static P_TISCALI_IT: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // ukr.net.md: ukr.net
 static P_UKR_NET: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "ukr.net",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/ukr-net",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // undernet.uy.md: undernet.uy
 static P_UNDERNET_UY: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "undernet.uy",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -806,23 +899,27 @@ static P_UNDERNET_UY: Lazy<Provider> = Lazy::new(|| Provider {
     ],
     config_defaults: None,
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // vfemail.md: vfemail.net
 static P_VFEMAIL: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "vfemail",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/vfemail",
     server: vec![],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // vodafone.de.md: vodafone.de, vodafonemail.de
 static P_VODAFONE_DE: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "vodafone.de",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -844,13 +941,15 @@ static P_VODAFONE_DE: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
 // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms
 static P_WEB_DE: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "web.de",
     status: Status::PREPARATION,
     before_login_hint: "You must allow IMAP access to your account before you can login.",
     after_login_hint: "Note: if you have your web.de spam settings too strict, you won't receive contact requests from new people. If you want to receive contact requests, you should disable the \"3-Wege-Spamschutz\" in the web.de settings.  Read how: https://hilfe.web.de/email/spam-und-viren/spamschutz-einstellungen.html",
@@ -861,7 +960,8 @@ static P_WEB_DE: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
 });
@@ -869,6 +969,7 @@ static P_WEB_DE: Lazy<Provider> = Lazy::new(|| {
 // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
 static P_YAHOO: Lazy<Provider> = Lazy::new(|| {
     Provider {
+    id: "yahoo",
     status: Status::PREPARATION,
     before_login_hint: "To use Delta Chat with your Yahoo email address you have to create an \"App-Password\" in the account security screen.",
     after_login_hint: "",
@@ -878,13 +979,15 @@ static P_YAHOO: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 }
 });
 
 // yandex.ru.md: yandex.com, yandex.by, yandex.kz, yandex.ru, yandex.ua, ya.ru, narod.ru
 static P_YANDEX_RU: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "yandex.ru",
     status: Status::PREPARATION,
     before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
     after_login_hint: "",
@@ -907,11 +1010,13 @@ static P_YANDEX_RU: Lazy<Provider> = Lazy::new(|| Provider {
     ],
     config_defaults: None,
     strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: Some(Oauth2Authorizer::Yandex),
 });
 
 // ziggo.nl.md: ziggo.nl
 static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "ziggo.nl",
     status: Status::OK,
     before_login_hint: "",
     after_login_hint: "",
@@ -933,11 +1038,12 @@ static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
         },
     ],
     config_defaults: None,
-    strict_tls: false,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
 });
 
-pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
+pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
     [
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol.com", &*P_AOL),
@@ -951,8 +1057,10 @@ pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy:
         ("dismail.de", &*P_DISMAIL_DE),
         ("disroot.org", &*P_DISROOT),
         ("dubby.org", &*P_DUBBY_ORG),
+        ("espiv.net", &*P_ESPIV_NET),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
+        ("example.net", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
         ("firemail.at", &*P_FIREMAIL_DE),
         ("firemail.de", &*P_FIREMAIL_DE),
@@ -960,6 +1068,7 @@ pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy:
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
+        ("google.com", &*P_GMAIL),
         ("gmx.net", &*P_GMX_NET),
         ("gmx.de", &*P_GMX_NET),
         ("gmx.at", &*P_GMX_NET),
@@ -1108,5 +1217,58 @@ pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy:
     .collect()
 });
 
+pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
+    [
+        ("aktivix.org", &*P_AKTIVIX_ORG),
+        ("aol", &*P_AOL),
+        ("arcor.de", &*P_ARCOR_DE),
+        ("autistici.org", &*P_AUTISTICI_ORG),
+        ("bluewin.ch", &*P_BLUEWIN_CH),
+        ("buzon.uy", &*P_BUZON_UY),
+        ("chello.at", &*P_CHELLO_AT),
+        ("comcast", &*P_COMCAST),
+        ("dismail.de", &*P_DISMAIL_DE),
+        ("disroot", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
+        ("espiv.net", &*P_ESPIV_NET),
+        ("example.com", &*P_EXAMPLE_COM),
+        ("fastmail", &*P_FASTMAIL),
+        ("firemail.de", &*P_FIREMAIL_DE),
+        ("five.chat", &*P_FIVE_CHAT),
+        ("freenet.de", &*P_FREENET_DE),
+        ("gmail", &*P_GMAIL),
+        ("gmx.net", &*P_GMX_NET),
+        ("hermes.radio", &*P_HERMES_RADIO),
+        ("hey.com", &*P_HEY_COM),
+        ("i.ua", &*P_I_UA),
+        ("icloud", &*P_ICLOUD),
+        ("kolst.com", &*P_KOLST_COM),
+        ("kontent.com", &*P_KONTENT_COM),
+        ("mail.ru", &*P_MAIL_RU),
+        ("mailbox.org", &*P_MAILBOX_ORG),
+        ("nauta.cu", &*P_NAUTA_CU),
+        ("outlook.com", &*P_OUTLOOK_COM),
+        ("posteo", &*P_POSTEO),
+        ("protonmail", &*P_PROTONMAIL),
+        ("riseup.net", &*P_RISEUP_NET),
+        ("rogers.com", &*P_ROGERS_COM),
+        ("systemli.org", &*P_SYSTEMLI_ORG),
+        ("t-online", &*P_T_ONLINE),
+        ("testrun", &*P_TESTRUN),
+        ("tiscali.it", &*P_TISCALI_IT),
+        ("ukr.net", &*P_UKR_NET),
+        ("undernet.uy", &*P_UNDERNET_UY),
+        ("vfemail", &*P_VFEMAIL),
+        ("vodafone.de", &*P_VODAFONE_DE),
+        ("web.de", &*P_WEB_DE),
+        ("yahoo", &*P_YAHOO),
+        ("yandex.ru", &*P_YANDEX_RU),
+        ("ziggo.nl", &*P_ZIGGO_NL),
+    ]
+    .iter()
+    .copied()
+    .collect()
+});
+
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd(2020, 10, 17));
+    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 1, 8));

src/provider/mod.rs

@@ -3,8 +3,8 @@
 mod data;
 
 use crate::config::Config;
-use crate::dc_tools::EmailAddress;
-use crate::provider::data::{PROVIDER_DATA, PROVIDER_UPDATED};
+use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED};
+use async_std_resolver::{config, resolver};
 use chrono::{NaiveDateTime, NaiveTime};
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
@@ -68,6 +68,8 @@ pub struct ConfigDefault {
 
 #[derive(Debug)]
 pub struct Provider {
+    /// Unique ID, corresponding to provider database filename.
+    pub id: &'static str,
     pub status: Status,
     pub before_login_hint: &'static str,
     pub after_login_hint: &'static str,
@@ -75,23 +77,88 @@ pub struct Provider {
     pub server: Vec<Server>,
     pub config_defaults: Option<Vec<ConfigDefault>>,
     pub strict_tls: bool,
+    pub max_smtp_rcpt_to: Option<u16>,
     pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
-pub fn get_provider_info(addr: &str) -> Option<&Provider> {
-    let domain = match addr.parse::<EmailAddress>() {
-        Ok(addr) => addr.domain,
-        Err(_err) => return None,
-    }
-    .to_lowercase();
+/// Returns provider for the given domain.
+///
+/// This function looks up domain in offline database first. If not
+/// found, it queries MX record for the domain and looks up offline
+/// database for MX domains.
+///
+/// For compatibility, email address can be passed to this function
+/// instead of the domain.
+pub async fn get_provider_info(domain: &str) -> Option<&'static Provider> {
+    let domain = domain.rsplitn(2, '@').next()?;
 
-    if let Some(provider) = PROVIDER_DATA.get(domain.as_str()) {
+    if let Some(provider) = get_provider_by_domain(domain) {
+        return Some(provider);
+    }
+
+    if let Some(provider) = get_provider_by_mx(domain).await {
+        return Some(provider);
+    }
+
+    None
+}
+
+/// Finds a provider in offline database based on domain.
+pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
+    if let Some(provider) = PROVIDER_DATA.get(domain.to_lowercase().as_str()) {
         return Some(*provider);
     }
 
     None
 }
 
+/// Finds a provider based on MX record for the given domain.
+///
+/// For security reasons, only Gmail can be configured this way.
+pub async fn get_provider_by_mx(domain: impl AsRef<str>) -> Option<&'static Provider> {
+    if let Ok(resolver) = resolver(
+        config::ResolverConfig::default(),
+        config::ResolverOpts::default(),
+    )
+    .await
+    {
+        let mut fqdn: String = String::from(domain.as_ref());
+        if !fqdn.ends_with('.') {
+            fqdn.push('.');
+        }
+
+        if let Ok(mx_domains) = resolver.mx_lookup(fqdn).await {
+            for (provider_domain, provider) in PROVIDER_DATA.iter() {
+                if provider.id != "gmail" {
+                    // MX lookup is limited to Gmail for security reasons
+                    continue;
+                }
+
+                let provider_fqdn = provider_domain.to_string() + ".";
+                let provider_fqdn_dot = ".".to_string() + &provider_fqdn;
+
+                for mx_domain in mx_domains.iter() {
+                    let mx_domain = mx_domain.exchange().to_lowercase().to_utf8();
+
+                    if mx_domain == provider_fqdn || mx_domain.ends_with(&provider_fqdn_dot) {
+                        return Some(provider);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+pub fn get_provider_by_id(id: &str) -> Option<&'static Provider> {
+    if let Some(provider) = PROVIDER_IDS.get(id) {
+        Some(provider)
+    } else {
+        None
+    }
+}
+
 // returns update timestamp in seconds, UTC, compatible for comparison with time() and database times
 pub fn get_provider_update_timestamp() -> i64 {
     NaiveDateTime::new(*PROVIDER_UPDATED, NaiveTime::from_hms(0, 0, 0)).timestamp_millis() / 1_000
@@ -106,24 +173,21 @@ mod tests {
     use chrono::NaiveDate;
 
     #[test]
-    fn test_get_provider_info_unexistant() {
-        let provider = get_provider_info("user@unexistant.org");
+    fn test_get_provider_by_domain_unexistant() {
+        let provider = get_provider_by_domain("unexistant.org");
         assert!(provider.is_none());
     }
 
     #[test]
-    fn test_get_provider_info_mixed_case() {
-        let provider = get_provider_info("uSer@nAUta.Cu").unwrap();
+    fn test_get_provider_by_domain_mixed_case() {
+        let provider = get_provider_by_domain("nAUta.Cu").unwrap();
         assert!(provider.status == Status::OK);
     }
 
     #[test]
-    fn test_get_provider_info() {
-        let provider = get_provider_info("nauta.cu"); // this is no email address
-        assert!(provider.is_none());
-
-        let addr = "user@nauta.cu";
-        let provider = get_provider_info(addr).unwrap();
+    fn test_get_provider_by_domain() {
+        let addr = "nauta.cu";
+        let provider = get_provider_by_domain(addr).unwrap();
         assert!(provider.status == Status::OK);
         let server = &provider.server[0];
         assert_eq!(server.protocol, Protocol::IMAP);
@@ -138,15 +202,30 @@ mod tests {
         assert_eq!(server.port, 25);
         assert_eq!(server.username_pattern, UsernamePattern::EMAIL);
 
-        let provider = get_provider_info("user@gmail.com").unwrap();
+        let provider = get_provider_by_domain("gmail.com").unwrap();
         assert!(provider.status == Status::PREPARATION);
         assert!(!provider.before_login_hint.is_empty());
         assert!(!provider.overview_page.is_empty());
 
-        let provider = get_provider_info("user@googlemail.com").unwrap();
+        let provider = get_provider_by_domain("googlemail.com").unwrap();
         assert!(provider.status == Status::PREPARATION);
     }
 
+    #[test]
+    fn test_get_provider_by_id() {
+        let provider = get_provider_by_id("gmail").unwrap();
+        assert!(provider.id == "gmail");
+    }
+
+    #[async_std::test]
+    async fn test_get_provider_info() {
+        assert!(get_provider_info("").await.is_none());
+        assert!(get_provider_info("google.com").await.unwrap().id == "gmail");
+
+        // get_provider_info() accepts email addresses for backwards compatibility
+        assert!(get_provider_info("example@google.com").await.unwrap().id == "gmail");
+    }
+
     #[test]
     fn test_get_provider_update_timestamp() {
         let timestamp_past = NaiveDateTime::new(

src/provider/update.py

@@ -8,7 +8,8 @@ import datetime
 
 out_all = ""
 out_domains = ""
-domains_dict = {}
+out_ids = ""
+domains_set = set()
 
 def camel(name):
     words = name.split("_")
@@ -22,15 +23,19 @@ def cleanstr(s):
     return s
 
 
+def file2id(f):
+    return os.path.basename(f).replace(".md", "")
+
+
 def file2varname(f):
-    f = f[f.rindex("/")+1:].replace(".md", "")
+    f = file2id(f)
     f = f.replace(".", "_")
     f = f.replace("-", "_")
     return "P_" + f.upper()
 
 
 def file2url(f):
-    f = f[f.rindex("/")+1:].replace(".md", "")
+    f = file2id(f)
     f = f.replace(".", "-")
     return "https://providers.delta.chat/" + f
 
@@ -61,14 +66,16 @@ def process_data(data, file):
         if domain == "" or domain.count(".") < 1 or domain.lower() != domain:
             raise TypeError("bad domain: " + domain)
 
-        global domains_dict
-        if domains_dict.get(domain, False):
+        global domains_set
+        if domain in domains_set:
             raise TypeError("domain used twice: " + domain)
-        domains_dict[domain] = True
+        domains_set.add(domain)
 
         domains += "    (\"" + domain + "\", &*" + file2varname(file) + "),\n"
         comment += domain + ", "
 
+    ids = ""
+    ids += "    (\"" + file2id(file) + "\", &*" + file2varname(file) + "),\n"
 
     server = ""
     has_imap = False
@@ -101,9 +108,12 @@ def process_data(data, file):
 
     config_defaults = process_config_defaults(data)
 
-    strict_tls = data.get("strict_tls", False)
+    strict_tls = data.get("strict_tls", True)
     strict_tls = "true" if strict_tls else "false"
 
+    max_smtp_rcpt_to = data.get("max_smtp_rcpt_to", 0)
+    max_smtp_rcpt_to = "Some(" + str(max_smtp_rcpt_to) + ")" if max_smtp_rcpt_to != 0 else "None"
+
     oauth2 = data.get("oauth2", "")
     oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
 
@@ -112,6 +122,7 @@ def process_data(data, file):
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
     if (not has_imap and not has_smtp) or (has_imap and has_smtp):
         provider += "static " + file2varname(file) + ": Lazy<Provider> = Lazy::new(|| Provider {\n"
+        provider += "    id: \"" + file2id(file) + "\",\n"
         provider += "    status: Status::" + status + ",\n"
         provider += "    before_login_hint: \"" + before_login_hint + "\",\n"
         provider += "    after_login_hint: \"" + after_login_hint + "\",\n"
@@ -119,6 +130,7 @@ def process_data(data, file):
         provider += "    server: vec![\n" + server + "    ],\n"
         provider += "    config_defaults: " + config_defaults + ",\n"
         provider += "    strict_tls: " + strict_tls + ",\n"
+        provider += "    max_smtp_rcpt_to: " + max_smtp_rcpt_to + ",\n"
         provider += "    oauth2_authorizer: " + oauth2 + ",\n"
         provider += "});\n\n"
     else:
@@ -128,13 +140,14 @@ def process_data(data, file):
         raise TypeError("status PREPARATION or BROKEN requires before_login_hint: " + file)
 
     # finally, add the provider
-    global out_all, out_domains
+    global out_all, out_domains, out_ids
     out_all += "// " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
 
     # also add provider with no special things to do -
     # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
     out_all += provider
     out_domains += domains
+    out_ids += ids
 
 
 def process_file(file):
@@ -168,10 +181,14 @@ if __name__ == "__main__":
 
     process_dir(sys.argv[1])
 
-    out_all += "pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
+    out_all += "pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
     out_all += out_domains;
     out_all += "].iter().copied().collect());\n\n"
 
+    out_all += "pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
+    out_all += out_ids;
+    out_all += "].iter().copied().collect());\n\n"
+
     now = datetime.datetime.utcnow()
     out_all += "pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> = "\
                "Lazy::new(|| chrono::NaiveDate::from_ymd("+str(now.year)+", "+str(now.month)+", "+str(now.day)+"));\n"

src/qr.rs

@@ -1,20 +1,20 @@
 //! # QR code module
 
+use anyhow::{bail, ensure, format_err, Error};
 use once_cell::sync::Lazy;
 use percent_encoding::percent_decode_str;
 use serde::Deserialize;
+use std::collections::BTreeMap;
 
 use crate::chat;
-use crate::config::*;
+use crate::config::Config;
 use crate::constants::Blocked;
-use crate::contact::*;
+use crate::contact::{addr_normalize, may_be_valid_addr, Contact, Origin};
 use crate::context::Context;
-use crate::error::{bail, ensure, format_err, Error};
 use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::message::Message;
-use crate::param::*;
-use crate::peerstate::*;
+use crate::peerstate::Peerstate;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
@@ -93,16 +93,18 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         }
     };
 
-    // replace & with \n to match expected param format
-    let fragment = fragment.replace('&', "\n");
+    let param: BTreeMap<&str, &str> = fragment
+        .split('&')
+        .filter_map(|s| {
+            if let [key, value] = s.splitn(2, '=').collect::<Vec<_>>()[..] {
+                Some((key, value))
+            } else {
+                None
+            }
+        })
+        .collect();
 
-    // Then parse the parameters
-    let param: Params = match fragment.parse() {
-        Ok(params) => params,
-        Err(err) => return err.into(),
-    };
-
-    let addr = if let Some(addr) = param.get(Param::Forwarded) {
+    let addr = if let Some(addr) = param.get("a") {
         match normalize_address(addr) {
             Ok(addr) => Some(addr),
             Err(err) => return err.into(),
@@ -112,7 +114,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     };
 
     // what is up with that param name?
-    let name = if let Some(encoded_name) = param.get(Param::SetLongitude) {
+    let name = if let Some(encoded_name) = param.get("n") {
         let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
         match percent_decode_str(&encoded_name).decode_utf8() {
             Ok(name) => name.to_string(),
@@ -122,12 +124,12 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         "".to_string()
     };
 
-    let invitenumber = param.get(Param::ProfileImage).map(|s| s.to_string());
-    let auth = param.get(Param::Auth).map(|s| s.to_string());
-    let grpid = param.get(Param::GroupId).map(|s| s.to_string());
+    let invitenumber = param.get("i").map(|s| s.to_string());
+    let auth = param.get("s").map(|s| s.to_string());
+    let grpid = param.get("x").map(|s| s.to_string());
 
     let grpname = if grpid.is_some() {
-        if let Some(encoded_name) = param.get(Param::GroupName) {
+        if let Some(encoded_name) = param.get("g") {
             let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
             match percent_decode_str(&encoded_name).decode_utf8() {
                 Ok(name) => Some(name.to_string()),
@@ -363,9 +365,9 @@ static VCARD_NAME_RE: Lazy<regex::Regex> =
 static VCARD_EMAIL_RE: Lazy<regex::Regex> =
     Lazy::new(|| regex::Regex::new(r"(?m)^EMAIL([^:\n]*):([^;\n]*)").unwrap());
 
-/// Extract address for the matmsg scheme.
+/// Extract address for the vcard scheme.
 ///
-/// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;
+/// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;`
 #[allow(clippy::indexing_slicing)]
 async fn decode_vcard(context: &Context, qr: &str) -> Lot {
     let name = VCARD_NAME_RE
@@ -425,7 +427,7 @@ fn normalize_address(addr: &str) -> Result<String, Error> {
     let new_addr = percent_decode_str(addr).decode_utf8()?;
     let new_addr = addr_normalize(&new_addr);
 
-    ensure!(may_be_valid_addr(&new_addr), "Bad e-mail address");
+    ensure!(may_be_valid_addr(new_addr), "Bad e-mail address");
 
     Ok(new_addr.to_string())
 }
@@ -488,6 +490,8 @@ mod tests {
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
         assert_eq!(contact.get_name(), "First Last");
+        assert_eq!(contact.get_authname(), "");
+        assert_eq!(contact.get_display_name(), "First Last");
     }
 
     #[async_std::test]
@@ -563,6 +567,7 @@ mod tests {
         assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
         // Test it again with lowercased "openpgp4fpr:" uri scheme
+        let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
             "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
@@ -603,6 +608,21 @@ mod tests {
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
         assert_eq!(contact.get_name(), "Jörn P. P.");
+
+        // Regression test
+        let ctx = TestContext::new().await;
+        let res = check_qr(
+            &ctx.ctx,
+            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
+        ).await;
+
+        println!("{:?}", res);
+        assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
+        assert_ne!(res.get_id(), 0);
+
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
+        assert_eq!(contact.get_addr(), "cli@deltachat.de");
+        assert_eq!(contact.get_name(), "");
     }
 
     #[async_std::test]

src/scheduler.rs

@@ -1,12 +1,16 @@
 use async_std::prelude::*;
-use async_std::sync::{channel, Receiver, Sender};
-use async_std::task;
+use async_std::{
+    channel::{self, Receiver, Sender},
+    task,
+};
 
+use crate::config::Config;
 use crate::context::Context;
 use crate::dc_tools::maybe_add_time_based_warnings;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
-use crate::{config::Config, message::MsgId, smtp::Smtp};
+use crate::message::MsgId;
+use crate::smtp::Smtp;
 
 pub(crate) struct StopToken;
 
@@ -54,7 +58,10 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started.send(()).await;
+        started
+            .send(())
+            .await
+            .expect("inbox loop, missing started receiver");
         let ctx = ctx1;
 
         // track number of continously executed jobs
@@ -69,9 +76,11 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                 }
                 Some(job) => {
                     // Let the fetch run, but return back to the job afterwards.
-                    info!(ctx, "postponing imap-job {} to run fetch...", job);
                     jobs_loaded = 0;
-                    fetch(&ctx, &mut connection).await;
+                    if ctx.get_config_bool(Config::InboxWatch).await {
+                        info!(ctx, "postponing imap-job {} to run fetch...", job);
+                        fetch(&ctx, &mut connection).await;
+                    }
                 }
                 None => {
                     jobs_loaded = 0;
@@ -87,6 +96,9 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                     info = if ctx.get_config_bool(Config::InboxWatch).await {
                         fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
                     } else {
+                        if let Err(err) = connection.scan_folders(&ctx).await {
+                            warn!(ctx, "{}", err);
+                        }
                         connection.fake_idle(&ctx, None).await
                     };
                 }
@@ -101,26 +113,29 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         })
         .race(fut)
         .await;
-    shutdown_sender.send(()).await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("inbox loop, missing shutdown receiver");
 }
 
 async fn fetch(ctx: &Context, connection: &mut Imap) {
     match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
-            if let Err(err) = connection.connect_configured(&ctx).await {
+            if let Err(err) = connection.connect_configured(ctx).await {
                 error_network!(ctx, "{}", err);
                 return;
             }
 
             // fetch
-            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+            if let Err(err) = connection.fetch(ctx, &watch_folder).await {
                 connection.trigger_reconnect();
-                warn!(ctx, "{}", err);
+                warn!(ctx, "{:#}", err);
             }
         }
         None => {
             warn!(ctx, "Can not fetch inbox folder, not set");
-            connection.fake_idle(&ctx, None).await;
+            connection.fake_idle(ctx, None).await;
         }
     }
 }
@@ -129,21 +144,30 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
     match ctx.get_config(folder).await {
         Some(watch_folder) => {
             // connect and fake idle if unable to connect
-            if let Err(err) = connection.connect_configured(&ctx).await {
+            if let Err(err) = connection.connect_configured(ctx).await {
                 warn!(ctx, "imap connection failed: {}", err);
-                return connection.fake_idle(&ctx, Some(watch_folder)).await;
+                return connection.fake_idle(ctx, Some(watch_folder)).await;
             }
 
             // fetch
-            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+            if let Err(err) = connection.fetch(ctx, &watch_folder).await {
                 connection.trigger_reconnect();
-                warn!(ctx, "{}", err);
+                warn!(ctx, "{:#}", err);
+            }
+
+            if folder == Config::ConfiguredInboxFolder {
+                // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
+                if let Err(err) = connection.scan_folders(ctx).await {
+                    // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
+                    // but maybe just one folder can't be selected or something
+                    warn!(ctx, "{}", err);
+                }
             }
 
             // idle
             if connection.can_idle() {
                 connection
-                    .idle(&ctx, Some(watch_folder))
+                    .idle(ctx, Some(watch_folder))
                     .await
                     .unwrap_or_else(|err| {
                         connection.trigger_reconnect();
@@ -151,12 +175,12 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                         InterruptInfo::new(false, None)
                     })
             } else {
-                connection.fake_idle(&ctx, Some(watch_folder)).await
+                connection.fake_idle(ctx, Some(watch_folder)).await
             }
         }
         None => {
             warn!(ctx, "Can not watch {} folder, not set", folder);
-            connection.fake_idle(&ctx, None).await
+            connection.fake_idle(ctx, None).await
         }
     }
 }
@@ -179,7 +203,10 @@ async fn simple_imap_loop(
     let ctx1 = ctx.clone();
 
     let fut = async move {
-        started.send(()).await;
+        started
+            .send(())
+            .await
+            .expect("simple imap loop, missing started receive");
         let ctx = ctx1;
 
         loop {
@@ -194,7 +221,10 @@ async fn simple_imap_loop(
         })
         .race(fut)
         .await;
-    shutdown_sender.send(()).await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("simple imap loop, missing shutdown receiver");
 }
 
 async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnectionHandlers) {
@@ -210,7 +240,10 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started.send(()).await;
+        started
+            .send(())
+            .await
+            .expect("smtp loop, missing started receiver");
         let ctx = ctx1;
 
         let mut interrupt_info = Default::default();
@@ -238,7 +271,10 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
         })
         .race(fut)
         .await;
-    shutdown_sender.send(()).await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("smtp loop, missing shutdown receiver");
 }
 
 impl Scheduler {
@@ -249,23 +285,25 @@ impl Scheduler {
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
         let (inbox, inbox_handlers) = ImapConnectionState::new();
 
-        let (inbox_start_send, inbox_start_recv) = channel(1);
-        let (mvbox_start_send, mvbox_start_recv) = channel(1);
+        let (inbox_start_send, inbox_start_recv) = channel::bounded(1);
+        let (mvbox_start_send, mvbox_start_recv) = channel::bounded(1);
         let mut mvbox_handle = None;
-        let (sentbox_start_send, sentbox_start_recv) = channel(1);
+        let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1);
         let mut sentbox_handle = None;
-        let (smtp_start_send, smtp_start_recv) = channel(1);
+        let (smtp_start_send, smtp_start_recv) = channel::bounded(1);
 
-        let ctx1 = ctx.clone();
-        let inbox_handle = Some(task::spawn(async move {
-            inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
-        }));
+        let inbox_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                inbox_loop(ctx, inbox_start_send, inbox_handlers).await
+            }))
+        };
 
         if ctx.get_config_bool(Config::MvboxWatch).await {
-            let ctx1 = ctx.clone();
+            let ctx = ctx.clone();
             mvbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
-                    ctx1,
+                    ctx,
                     mvbox_start_send,
                     mvbox_handlers,
                     Config::ConfiguredMvboxFolder,
@@ -273,14 +311,17 @@ impl Scheduler {
                 .await
             }));
         } else {
-            mvbox_start_send.send(()).await;
+            mvbox_start_send
+                .send(())
+                .await
+                .expect("mvbox start send, missing receiver");
         }
 
         if ctx.get_config_bool(Config::SentboxWatch).await {
-            let ctx1 = ctx.clone();
+            let ctx = ctx.clone();
             sentbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
-                    ctx1,
+                    ctx,
                     sentbox_start_send,
                     sentbox_handlers,
                     Config::ConfiguredSentboxFolder,
@@ -288,13 +329,18 @@ impl Scheduler {
                 .await
             }));
         } else {
-            sentbox_start_send.send(()).await;
+            sentbox_start_send
+                .send(())
+                .await
+                .expect("sentbox start send, missing receiver");
         }
 
-        let ctx1 = ctx.clone();
-        let smtp_handle = Some(task::spawn(async move {
-            smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
-        }));
+        let smtp_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                smtp_loop(ctx, smtp_start_send, smtp_handlers).await
+            }))
+        };
 
         *self = Scheduler::Running {
             inbox,
@@ -365,17 +411,27 @@ impl Scheduler {
             }
             Scheduler::Running {
                 inbox,
+                inbox_handle,
                 mvbox,
+                mvbox_handle,
                 sentbox,
+                sentbox_handle,
                 smtp,
+                smtp_handle,
                 ..
             } => {
-                inbox
-                    .stop()
-                    .join(mvbox.stop())
-                    .join(sentbox.stop())
-                    .join(smtp.stop())
-                    .await;
+                if inbox_handle.is_some() {
+                    inbox.stop().await;
+                }
+                if mvbox_handle.is_some() {
+                    mvbox.stop().await;
+                }
+                if sentbox_handle.is_some() {
+                    sentbox.stop().await;
+                }
+                if smtp_handle.is_some() {
+                    smtp.stop().await;
+                }
 
                 StopToken
             }
@@ -434,7 +490,10 @@ impl ConnectionState {
     /// Shutdown this connection completely.
     async fn stop(&self) {
         // Trigger shutdown of the run loop.
-        self.stop_sender.send(()).await;
+        self.stop_sender
+            .send(())
+            .await
+            .expect("stop, missing receiver");
         // Wait for a notification that the run loop has been shutdown.
         self.shutdown_receiver.recv().await.ok();
     }
@@ -452,9 +511,9 @@ pub(crate) struct SmtpConnectionState {
 
 impl SmtpConnectionState {
     fn new() -> (Self, SmtpConnectionHandlers) {
-        let (stop_sender, stop_receiver) = channel(1);
-        let (shutdown_sender, shutdown_receiver) = channel(1);
-        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+        let (stop_sender, stop_receiver) = channel::bounded(1);
+        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = SmtpConnectionHandlers {
             connection: Smtp::new(),
@@ -500,9 +559,9 @@ pub(crate) struct ImapConnectionState {
 impl ImapConnectionState {
     /// Construct a new connection.
     fn new() -> (Self, ImapConnectionHandlers) {
-        let (stop_sender, stop_receiver) = channel(1);
-        let (shutdown_sender, shutdown_receiver) = channel(1);
-        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+        let (stop_sender, stop_receiver) = channel::bounded(1);
+        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = ImapConnectionHandlers {
             connection: Imap::new(idle_interrupt_receiver),

src/securejoin/bobstate.rs

@@ -0,0 +1,525 @@
+//! Secure-Join protocol state machine for Bob, the joiner-side.
+//!
+//! This module contains the state machine to run the Secure-Join handshake for Bob and does
+//! not do any user interaction required by the protocol.  Instead the state machine
+//! provides all the information to its driver so it can perform the correct interactions.
+//!
+//! The [`BobState`] is only directly used to initially create it when starting the
+//! protocol.  Afterwards it must be stored in a mutex and the [`BobStateHandle`] should be
+//! used to work with the state.
+
+use anyhow::{Error, Result};
+use async_std::sync::MutexGuard;
+
+use crate::chat::{self, ChatId};
+use crate::constants::{Blocked, Viewtype};
+use crate::contact::{Contact, Origin};
+use crate::context::Context;
+use crate::events::EventType;
+use crate::headerdef::HeaderDef;
+use crate::key::{DcKey, SignedPublicKey};
+use crate::message::Message;
+use crate::mimeparser::{MimeMessage, SystemMessage};
+use crate::param::Param;
+
+use super::qrinvite::QrInvite;
+use super::{
+    encrypted_and_signed, fingerprint_equals_sender, mark_peer_as_verified, JoinError, SendMsgError,
+};
+
+/// The stage of the [`BobState`] securejoin handshake protocol state machine.
+///
+/// This does not concern itself with user interactions, only represents what happened to
+/// the protocol state machine from handling this message.
+#[derive(Clone, Copy, Debug, Display)]
+pub enum BobHandshakeStage {
+    /// Step 2 completed: (vc|vg)-request message sent.
+    ///
+    /// Note that this is only ever returned by [`BobState::start_protocol`] and never by
+    /// [`BobState::handle_message`].
+    RequestSent,
+    /// Step 4 completed: (vc|vg)-request-with-auth message sent.
+    RequestWithAuthSent,
+    /// The protocol completed successfully.
+    Completed,
+    /// The protocol prematurely terminated with given reason.
+    Terminated(&'static str),
+}
+
+/// A handle to work with the [`BobState`] of Bob's securejoin protocol.
+///
+/// This handle can only be created for when an underlying [`BobState`] exists.  It keeps
+/// open a lock which guarantees unique access to the state and this struct must be dropped
+/// to return the lock.
+pub struct BobStateHandle<'a> {
+    guard: MutexGuard<'a, Option<BobState>>,
+    bobstate: BobState,
+    clear_state_on_drop: bool,
+}
+
+impl<'a> BobStateHandle<'a> {
+    /// Creates a new instance, upholding the guarantee that [`BobState`] must exist.
+    pub fn from_guard(mut guard: MutexGuard<'a, Option<BobState>>) -> Option<Self> {
+        match guard.take() {
+            Some(bobstate) => Some(Self {
+                guard,
+                bobstate,
+                clear_state_on_drop: false,
+            }),
+            None => None,
+        }
+    }
+
+    /// Returns the [`ChatId`] of the 1:1 chat with the inviter (Alice).
+    pub fn chat_id(&self) -> ChatId {
+        self.bobstate.chat_id
+    }
+
+    /// Returns a reference to the [`QrInvite`] of the joiner process.
+    pub fn invite(&self) -> &QrInvite {
+        &self.bobstate.invite
+    }
+
+    /// Handles the given message for the securejoin handshake for Bob.
+    ///
+    /// This proxies to [`BobState::handle_message`] and makes sure to clear the state when
+    /// the protocol state is terminal.  It returns `Some` if the message successfully
+    /// advanced the state of the protocol state machine, `None` otherwise.
+    pub async fn handle_message(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Option<BobHandshakeStage> {
+        info!(context, "Handling securejoin message for BobStateHandle");
+        match self.bobstate.handle_message(context, mime_message).await {
+            Ok(Some(stage)) => {
+                if matches!(
+                    stage,
+                    BobHandshakeStage::Completed | BobHandshakeStage::Terminated(_)
+                ) {
+                    self.finish_protocol(context).await;
+                }
+                Some(stage)
+            }
+            Ok(None) => None,
+            Err(err) => {
+                warn!(
+                    context,
+                    "Error handling handshake message, aborting handshake: {}", err
+                );
+                self.finish_protocol(context).await;
+                None
+            }
+        }
+    }
+
+    /// Marks the bob handshake as finished.
+    ///
+    /// This will clear the state on [`InnerContext::bob`] once this handle is dropped,
+    /// allowing a new handshake to be started from [`Bob`].
+    ///
+    /// Note that the state is only cleared on Drop since otherwise the invariant that the
+    /// state is always consistent is violated.  However the "ongoing" process is released
+    /// here a little bit earlier as this requires access to the Context, which we do not
+    /// have on Drop (Drop can not run asynchronous code).  Stopping the "ongoing" process
+    /// will release [`securejoin`](super::securejoin) which in turn will finally free the
+    /// ongoing process using [`Context::free_ongoing`].
+    ///
+    /// [`InnerContext::bob`]: crate::context::InnerContext::bob
+    /// [`Bob`]: super::Bob
+    async fn finish_protocol(&mut self, context: &Context) {
+        info!(context, "Finishing securejoin handshake protocol for Bob");
+        self.clear_state_on_drop = true;
+        if let QrInvite::Group { .. } = self.bobstate.invite {
+            context.stop_ongoing().await;
+        }
+    }
+}
+
+impl<'a> Drop for BobStateHandle<'a> {
+    fn drop(&mut self) {
+        if self.clear_state_on_drop {
+            // The Option should already be empty because we take it out in the ctor,
+            // however the typesystem doesn't guarantee this so do it again anyway.
+            self.guard.take();
+        } else {
+            // Make sure to put back the BobState into the Option of the Mutex, it was taken
+            // out by the constructor.
+            self.guard.replace(self.bobstate.clone());
+        }
+    }
+}
+
+/// The securejoin state kept in-memory while Bob is joining.
+///
+/// This is currently stored in [`Bob`] which is stored on the [`Context`], thus Bob can
+/// only run one securejoin joiner protocol at a time.
+///
+/// This purposefully has nothing optional, the state is always fully valid.  See
+/// [`Bob::state`] to get access to this state.
+///
+/// # Conducting the securejoin handshake
+///
+/// The methods on this struct allow you to interact with the state and thus conduct the
+/// securejoin handshake for Bob.  The methods only concern themselves with the protocol
+/// state and explicitly avoid performing any user interactions required by securejoin.
+/// This simplifies the concerns and logic required in both the callers and in the state
+/// management.  The return values can be used to understand what user interactions need to
+/// happen.
+///
+/// [`Bob`]: super::Bob
+/// [`Bob::state`]: super::Bob::state
+#[derive(Debug, Clone)]
+pub struct BobState {
+    /// The QR Invite code.
+    invite: QrInvite,
+    /// The next expected message from Alice.
+    next: SecureJoinStep,
+    /// The [`ChatId`] of the 1:1 chat with Alice, matching [`QrInvite::contact_id`].
+    chat_id: ChatId,
+}
+
+impl BobState {
+    /// Starts the securejoin protocol and creates a new [`BobState`].
+    ///
+    /// # Bob - the joiner's side
+    /// ## Step 2 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    pub async fn start_protocol(
+        context: &Context,
+        invite: QrInvite,
+    ) -> Result<(Self, BobHandshakeStage), JoinError> {
+        let chat_id = chat::create_by_contact_id(context, invite.contact_id())
+            .await
+            .map_err(JoinError::UnknownContact)?;
+        if fingerprint_equals_sender(context, invite.fingerprint(), chat_id).await {
+            // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
+            info!(context, "Taking securejoin protocol shortcut");
+            let state = Self {
+                invite,
+                next: SecureJoinStep::ContactConfirm,
+                chat_id,
+            };
+            state
+                .send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
+                .await?;
+            Ok((state, BobHandshakeStage::RequestWithAuthSent))
+        } else {
+            let state = Self {
+                invite,
+                next: SecureJoinStep::AuthRequired,
+                chat_id,
+            };
+            state
+                .send_handshake_message(context, BobHandshakeMsg::Request)
+                .await?;
+            Ok((state, BobHandshakeStage::RequestSent))
+        }
+    }
+
+    /// Returns the [`QrInvite`] used to create this [`BobState`].
+    pub fn invite(&self) -> &QrInvite {
+        &self.invite
+    }
+
+    /// Handles the given message for the securejoin handshake for Bob.
+    ///
+    /// If the message was not used for this handshake `None` is returned, otherwise the new
+    /// stage is returned.  Once [`BobHandshakeStage::Completed`] or
+    /// [`BobHandshakeStage::Terminated`] are reached this [`BobState`] should be destroyed,
+    /// further calling it will just result in the messages being unused by this handshake.
+    ///
+    /// # Errors
+    ///
+    /// Under normal operation this should never return an error, regardless of what kind of
+    /// message it is called with.  Any errors therefore should be treated as fatal internal
+    /// errors and this entire [`BobState`] should be thrown away as the state machine can
+    /// no longer be considered consistent.
+    async fn handle_message(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Result<Option<BobHandshakeStage>> {
+        let step = match mime_message.get(HeaderDef::SecureJoin) {
+            Some(step) => step,
+            None => {
+                warn!(
+                    context,
+                    "Message has no Secure-Join header: {}",
+                    mime_message.get_rfc724_mid().unwrap_or_default()
+                );
+                return Ok(None);
+            }
+        };
+        if !self.is_msg_expected(context, step.as_str()) {
+            info!(context, "{} message out of sync for BobState", step);
+            return Ok(None);
+        }
+        match step.as_str() {
+            "vg-auth-required" | "vc-auth-required" => {
+                self.step_auth_required(context, mime_message).await
+            }
+            "vg-member-added" | "vc-contact-confirm" => {
+                self.step_contact_confirm(context, mime_message).await
+            }
+            _ => {
+                warn!(context, "Invalid step for BobState: {}", step);
+                Ok(None)
+            }
+        }
+    }
+
+    /// Returns `true` if the message is expected according to the protocol.
+    fn is_msg_expected(&self, context: &Context, step: &str) -> bool {
+        let variant_matches = match self.invite {
+            QrInvite::Contact { .. } => step.starts_with("vc-"),
+            QrInvite::Group { .. } => step.starts_with("vg-"),
+        };
+        let step_matches = self.next.matches(context, step);
+        variant_matches && step_matches
+    }
+
+    /// Handles a *vc-auth-required* or *vg-auth-required* message.
+    ///
+    /// # Bob - the joiner's side
+    /// ## Step 4 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    async fn step_auth_required(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Result<Option<BobHandshakeStage>> {
+        info!(
+            context,
+            "Bob Step 4 - handling vc-auth-require/vg-auth-required message"
+        );
+        if !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint())) {
+            let reason = if mime_message.was_encrypted() {
+                "Valid signature missing"
+            } else {
+                "Required encryption missing"
+            };
+            self.next = SecureJoinStep::Terminated;
+            return Ok(Some(BobHandshakeStage::Terminated(reason)));
+        }
+        if !fingerprint_equals_sender(context, self.invite.fingerprint(), self.chat_id).await {
+            self.next = SecureJoinStep::Terminated;
+            return Ok(Some(BobHandshakeStage::Terminated("Fingerprint mismatch")));
+        }
+        info!(context, "Fingerprint verified.",);
+        self.next = SecureJoinStep::ContactConfirm;
+        self.send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
+            .await?;
+        Ok(Some(BobHandshakeStage::RequestWithAuthSent))
+    }
+
+    /// Handles a *vc-contact-confirm* or *vg-member-added* message.
+    ///
+    /// # Bob - the joiner's side
+    /// ## Step 7 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    ///
+    /// This deviates from the protocol by also sending a confirmation message in response
+    /// to the *vc-contact-confirm* message.  This has no specific value to the protocol and
+    /// is only done out of symmerty with *vg-member-added* handling.
+    async fn step_contact_confirm(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Result<Option<BobHandshakeStage>> {
+        info!(
+            context,
+            "Bob Step 7 - handling vc-contact-confirm/vg-member-added message"
+        );
+        let vg_expect_encrypted = match self.invite {
+            QrInvite::Contact { .. } => {
+                // setup-contact is always encrypted
+                true
+            }
+            QrInvite::Group { ref grpid, .. } => {
+                // This is buggy, is_verified_group will always be
+                // false since the group is created by receive_imf for
+                // the very handshake message we're handling now.  But
+                // only after we have returned.  It does not impact
+                // the security invariants of secure-join however.
+                let (_, is_verified_group, _) = chat::get_chat_id_by_grpid(context, grpid)
+                    .await
+                    .unwrap_or((ChatId::new(0), false, Blocked::Not));
+                // when joining a non-verified group
+                // the vg-member-added message may be unencrypted
+                // when not all group members have keys or prefer encryption.
+                // So only expect encryption if this is a verified group
+                is_verified_group
+            }
+        };
+        if vg_expect_encrypted
+            && !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint()))
+        {
+            self.next = SecureJoinStep::Terminated;
+            return Ok(Some(BobHandshakeStage::Terminated(
+                "Contact confirm message not encrypted",
+            )));
+        }
+        mark_peer_as_verified(context, self.invite.fingerprint()).await?;
+        Contact::scaleup_origin_by_id(context, self.invite.contact_id(), Origin::SecurejoinJoined)
+            .await;
+        emit_event!(context, EventType::ContactsChanged(None));
+
+        if let QrInvite::Group { .. } = self.invite {
+            let member_added = mime_message
+                .get(HeaderDef::ChatGroupMemberAdded)
+                .map(|s| s.as_str())
+                .ok_or_else(|| Error::msg("Missing Chat-Group-Member-Added header"))?;
+            if !context.is_self_addr(member_added).await? {
+                info!(context, "Message belongs to a different handshake (scaled up contact anyway to allow creation of group).");
+                return Ok(None);
+            }
+        }
+
+        self.send_handshake_message(context, BobHandshakeMsg::ContactConfirmReceived)
+            .await
+            .map_err(|_| {
+                warn!(
+                    context,
+                    "Failed to send vc-contact-confirm-received/vg-member-added-received"
+                );
+            })
+            // This is not an error affecting the protocol outcome.
+            .ok();
+
+        self.next = SecureJoinStep::Completed;
+        Ok(Some(BobHandshakeStage::Completed))
+    }
+
+    /// Sends the requested handshake message to Alice.
+    ///
+    /// This takes care of adding the required headers for the step.
+    async fn send_handshake_message(
+        &self,
+        context: &Context,
+        step: BobHandshakeMsg,
+    ) -> Result<(), SendMsgError> {
+        let mut msg = Message {
+            viewtype: Viewtype::Text,
+            text: Some(step.body_text(&self.invite)),
+            hidden: true,
+            ..Default::default()
+        };
+        msg.param.set_cmd(SystemMessage::SecurejoinMessage);
+
+        // Sends the step in Secure-Join header.
+        msg.param
+            .set(Param::Arg, step.securejoin_header(&self.invite));
+
+        match step {
+            BobHandshakeMsg::Request => {
+                // Sends the Secure-Join-Invitenumber header in mimefactory.rs.
+                msg.param.set(Param::Arg2, self.invite.invitenumber());
+                msg.param.set_int(Param::ForcePlaintext, 1);
+            }
+            BobHandshakeMsg::RequestWithAuth => {
+                // Sends the Secure-Join-Auth header in mimefactory.rs.
+                msg.param.set(Param::Arg2, self.invite.authcode());
+                msg.param.set_int(Param::GuaranteeE2ee, 1);
+            }
+            BobHandshakeMsg::ContactConfirmReceived => {
+                msg.param.set_int(Param::GuaranteeE2ee, 1);
+            }
+        };
+
+        // Sends our own fingerprint in the Secure-Join-Fingerprint header.
+        let bob_fp = SignedPublicKey::load_self(context).await?.fingerprint();
+        msg.param.set(Param::Arg3, bob_fp.hex());
+
+        // Sends the grpid in the Secure-Join-Group header.
+        if let QrInvite::Group { ref grpid, .. } = self.invite {
+            msg.param.set(Param::Arg4, grpid);
+        }
+
+        chat::send_msg(context, self.chat_id, &mut msg).await?;
+        Ok(())
+    }
+}
+
+/// Identifies the SecureJoin handshake messages Bob can send.
+enum BobHandshakeMsg {
+    /// vc-request or vg-request
+    Request,
+    /// vc-request-with-auth or vg-request-with-auth
+    RequestWithAuth,
+    /// vc-contact-confirm-received or vg-member-added-received
+    ContactConfirmReceived,
+}
+
+impl BobHandshakeMsg {
+    /// Returns the text to send in the body of the handshake message.
+    ///
+    /// This text has no significance to the protocol, but would be visible if users see
+    /// this email message directly, e.g. when accessing their email without using
+    /// DeltaChat.
+    fn body_text(&self, invite: &QrInvite) -> String {
+        format!("Secure-Join: {}", self.securejoin_header(invite))
+    }
+
+    /// Returns the `Secure-Join` header value.
+    ///
+    /// This identifies the step this message is sending information about.  Most protocol
+    /// steps include additional information into other headers, see
+    /// [`BobState::send_handshake_message`] for these.
+    fn securejoin_header(&self, invite: &QrInvite) -> &'static str {
+        match self {
+            Self::Request => match invite {
+                QrInvite::Contact { .. } => "vc-request",
+                QrInvite::Group { .. } => "vg-request",
+            },
+            Self::RequestWithAuth => match invite {
+                QrInvite::Contact { .. } => "vc-request-with-auth",
+                QrInvite::Group { .. } => "vg-request-with-auth",
+            },
+            Self::ContactConfirmReceived => match invite {
+                QrInvite::Contact { .. } => "vc-contact-confirm-received",
+                QrInvite::Group { .. } => "vg-member-added-received",
+            },
+        }
+    }
+}
+
+/// The next message expected by [`BobState`] in the setup-contact/secure-join protocol.
+#[derive(Debug, Clone, PartialEq)]
+enum SecureJoinStep {
+    /// Expecting the auth-required message.
+    ///
+    /// This corresponds to the `vc-auth-required` or `vg-auth-required` message of step 3d.
+    AuthRequired,
+    /// Expecting the contact-confirm message.
+    ///
+    /// This corresponds to the `vc-contact-confirm` or `vg-member-added` message of step
+    /// 6b.
+    ContactConfirm,
+    /// The protocol terminated because of an error.
+    ///
+    /// The securejoin protocol terminated, this exists to ensure [`BobState`] can detect
+    /// when it earlier signalled that is should be terminated.  It is an error to call with
+    /// this state.
+    Terminated,
+    /// The protocol completed.
+    ///
+    /// This exists to ensure [`BobState`] can detect when it earlier signalled that it is
+    /// complete.  It is an error to call with this state.
+    Completed,
+}
+
+impl SecureJoinStep {
+    /// Compares the legacy string representation of a step to a [`SecureJoinStep`] variant.
+    fn matches(&self, context: &Context, step: &str) -> bool {
+        match self {
+            Self::AuthRequired => step == "vc-auth-required" || step == "vg-auth-required",
+            Self::ContactConfirm => step == "vc-contact-confirm" || step == "vg-member-added",
+            SecureJoinStep::Terminated => {
+                warn!(context, "Terminated state for next securejoin step");
+                false
+            }
+            SecureJoinStep::Completed => {
+                warn!(context, "Complted state for next securejoin step");
+                false
+            }
+        }
+    }
+}

src/securejoin/qrinvite.rs

@@ -0,0 +1,120 @@
+//! Supporting code for the QR-code invite.
+//!
+//! QR-codes are decoded into a more general-purpose [`Lot`] struct normally, this struct is
+//! so general it is not even specific to QR-codes.  This makes working with it rather hard,
+//! so here we have a wrapper type that specifically deals with Secure-Join QR-codes so
+//! that the Secure-Join code can have many more guarantees when dealing with this.
+
+use std::convert::TryFrom;
+
+use anyhow::Result;
+
+use crate::key::{Fingerprint, FingerprintError};
+use crate::lot::{Lot, LotState};
+
+/// Represents the data from a QR-code scan.
+///
+/// There are methods to conveniently access fields present in both variants.
+#[derive(Debug, Clone)]
+pub enum QrInvite {
+    Contact {
+        contact_id: u32,
+        fingerprint: Fingerprint,
+        invitenumber: String,
+        authcode: String,
+    },
+    Group {
+        contact_id: u32,
+        fingerprint: Fingerprint,
+        name: String,
+        grpid: String,
+        invitenumber: String,
+        authcode: String,
+    },
+}
+
+impl QrInvite {
+    /// The contact ID of the inviter.
+    ///
+    /// The actual QR-code contains a URL-encoded email address, but upon scanning this is
+    /// translated to a contact ID.
+    pub fn contact_id(&self) -> u32 {
+        match self {
+            Self::Contact { contact_id, .. } | Self::Group { contact_id, .. } => *contact_id,
+        }
+    }
+
+    /// The fingerprint of the inviter.
+    pub fn fingerprint(&self) -> &Fingerprint {
+        match self {
+            Self::Contact { fingerprint, .. } | Self::Group { fingerprint, .. } => fingerprint,
+        }
+    }
+
+    /// The `INVITENUMBER` of the setup-contact/secure-join protocol.
+    pub fn invitenumber(&self) -> &str {
+        match self {
+            Self::Contact { invitenumber, .. } | Self::Group { invitenumber, .. } => invitenumber,
+        }
+    }
+
+    /// The `AUTH` code of the setup-contact/secure-join protocol.
+    pub fn authcode(&self) -> &str {
+        match self {
+            Self::Contact { authcode, .. } | Self::Group { authcode, .. } => authcode,
+        }
+    }
+}
+
+impl TryFrom<Lot> for QrInvite {
+    type Error = QrError;
+
+    fn try_from(lot: Lot) -> Result<Self, Self::Error> {
+        if lot.state != LotState::QrAskVerifyContact && lot.state != LotState::QrAskVerifyGroup {
+            return Err(QrError::UnsupportedProtocol);
+        }
+        if lot.id == 0 {
+            return Err(QrError::MissingContactId);
+        }
+        let fingerprint = lot.fingerprint.ok_or(QrError::MissingFingerprint)?;
+        let invitenumber = lot.invitenumber.ok_or(QrError::MissingInviteNumber)?;
+        let authcode = lot.auth.ok_or(QrError::MissingAuthCode)?;
+        match lot.state {
+            LotState::QrAskVerifyContact => Ok(QrInvite::Contact {
+                contact_id: lot.id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            }),
+            LotState::QrAskVerifyGroup => Ok(QrInvite::Group {
+                contact_id: lot.id,
+                fingerprint,
+                name: lot.text1.ok_or(QrError::MissingGroupName)?,
+                grpid: lot.text2.ok_or(QrError::MissingGroupId)?,
+                invitenumber,
+                authcode,
+            }),
+            _ => Err(QrError::UnsupportedProtocol),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum QrError {
+    #[error("Unsupported protocol in QR-code")]
+    UnsupportedProtocol,
+    #[error("Failed to read fingerprint")]
+    InvalidFingerprint(#[from] FingerprintError),
+    #[error("Missing fingerprint")]
+    MissingFingerprint,
+    #[error("Missing invitenumber")]
+    MissingInviteNumber,
+    #[error("Missing auth code")]
+    MissingAuthCode,
+    #[error("Missing group name")]
+    MissingGroupName,
+    #[error("Missing group id")]
+    MissingGroupId,
+    #[error("Missing contact id")]
+    MissingContactId,
+}

src/simplify.rs

@@ -17,18 +17,23 @@ pub fn escape_message_footer_marks(text: &str) -> String {
 }
 
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
+/// Returns `(lines, footer_lines)` tuple;
+/// `footer_lines` is set to `Some` if the footer was actually removed from `lines`
+/// (which is equal to the input array otherwise).
 #[allow(clippy::indexing_slicing)]
-fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
+fn remove_message_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<&'a [&'a str]>) {
     let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
         match line {
             // some providers encode `-- ` to `-- =20` which results in `--  `
-            "-- " | "--  " => return &lines[..ix],
+            "-- " | "--  " => return (&lines[..ix], lines.get(ix + 1..)),
             // some providers encode `-- ` to `=2D-` which results in only `--`;
             // use that only when no other footer is found
             // and if the line before is empty and the line after is not empty
             "--" => {
-                if (ix == 0 || lines[ix - 1] == "") && ix != lines.len() - 1 && lines[ix + 1] != ""
+                if (ix == 0 || lines[ix - 1].is_empty())
+                    && ix != lines.len() - 1
+                    && !lines[ix + 1].is_empty()
                 {
                     nearly_standard_footer = Some(ix);
                 }
@@ -37,13 +42,15 @@ fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
         }
     }
     if let Some(ix) = nearly_standard_footer {
-        return &lines[..ix];
+        return (&lines[..ix], lines.get(ix + 1..));
     }
-    lines
+    (lines, None)
 }
 
-/// Remove nonstandard footer and a boolean indicating whether such
-/// footer was removed.
+/// Remove nonstandard footer and a boolean indicating whether such footer was removed.
+/// Returns `(lines, is_footer_removed)` tuple;
+/// `is_footer_removed` is set to `true` if the footer was actually removed from `lines`
+/// (which is equal to the input array otherwise).
 #[allow(clippy::indexing_slicing)]
 fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     for (ix, &line) in lines.iter().enumerate() {
@@ -60,34 +67,58 @@ fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     (lines, false)
 }
 
-fn split_lines(buf: &str) -> Vec<&str> {
+pub(crate) fn split_lines(buf: &str) -> Vec<&str> {
     buf.split('\n').collect()
 }
 
 /// Simplify message text for chat display.
 /// Remove quotes, signatures, trailing empty lines etc.
-pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool, Option<String>) {
+/// Returns `(text, is_forwarded, is_cut, quote, footer)` tuple,
+/// returning the simplified text and some additional information gained from the input.
+pub fn simplify(
+    mut input: String,
+    is_chat_message: bool,
+) -> (String, bool, bool, Option<String>, Option<String>) {
+    let mut is_cut = false;
+
     input.retain(|c| c != '\r');
     let lines = split_lines(&input);
     let (lines, is_forwarded) = skip_forward_header(&lines);
 
-    let (lines, top_quote) = remove_top_quote(lines);
+    let (lines, mut top_quote) = remove_top_quote(lines);
     let original_lines = &lines;
-    let lines = remove_message_footer(lines);
+    let (lines, footer_lines) = remove_message_footer(lines);
+    let footer = footer_lines.map(|footer_lines| render_message(footer_lines, false));
+    is_cut = is_cut || footer.is_some();
 
     let text = if is_chat_message {
         render_message(lines, false)
     } else {
         let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
-        let (lines, has_bottom_quote) = remove_bottom_quote(lines);
+        let (lines, mut bottom_quote) = remove_bottom_quote(lines);
+
+        if top_quote.is_none() && bottom_quote.is_some() {
+            std::mem::swap(&mut top_quote, &mut bottom_quote);
+        }
 
         if lines.iter().all(|it| it.trim().is_empty()) {
             render_message(original_lines, false)
         } else {
-            render_message(lines, has_nonstandard_footer || has_bottom_quote)
+            is_cut = is_cut || has_nonstandard_footer || bottom_quote.is_some();
+            render_message(lines, has_nonstandard_footer || bottom_quote.is_some())
         }
     };
-    (text, is_forwarded, top_quote)
+
+    if !is_chat_message {
+        top_quote = top_quote.map(|quote| {
+            let quote_lines = split_lines(&quote);
+            let (quote_lines, quote_footer_lines) = remove_message_footer(&quote_lines);
+            is_cut = is_cut || quote_footer_lines.is_some();
+
+            render_message(quote_lines, false)
+        });
+    }
+    (text, is_forwarded, is_cut, top_quote, footer)
 }
 
 /// Skips "forwarded message" header.
@@ -105,16 +136,27 @@ fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
 }
 
 #[allow(clippy::indexing_slicing)]
-fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<String>) {
+    let mut first_quoted_line = lines.len();
     let mut last_quoted_line = None;
     for (l, line) in lines.iter().enumerate().rev() {
         if is_plain_quote(line) {
+            if last_quoted_line.is_none() {
+                first_quoted_line = l + 1;
+            }
             last_quoted_line = Some(l)
         } else if !is_empty_line(line) {
             break;
         }
     }
     if let Some(mut l_last) = last_quoted_line {
+        let quoted_text = lines[l_last..first_quoted_line]
+            .iter()
+            .map(|s| {
+                s.strip_prefix(">")
+                    .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u))
+            })
+            .join("\n");
         if l_last > 1 && is_empty_line(lines[l_last - 1]) {
             l_last -= 1
         }
@@ -124,9 +166,9 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
                 l_last -= 1
             }
         }
-        (&lines[..l_last], true)
+        (&lines[..l_last], Some(quoted_text))
     } else {
-        (lines, false)
+        (lines, None)
     }
 }
 
@@ -203,19 +245,10 @@ fn render_message(lines: &[&str], is_cut_at_end: bool) -> String {
  * Tools
  */
 fn is_empty_line(buf: &str) -> bool {
-    // XXX: can it be simplified to buf.chars().all(|c| c.is_whitespace())?
-    //
-    // Strictly speaking, it is not equivalent (^A is not whitespace, but less than ' '),
-    // but having control sequences in email body?!
-    //
-    // See discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
-    for c in buf.chars() {
-        if c > ' ' {
-            return false;
-        }
-    }
-
-    true
+    buf.chars().all(char::is_whitespace)
+    // for some time, this checked for `char <= ' '`,
+    // see discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
+    // and https://github.com/deltachat/deltachat-core-rust/pull/2104/files#r538973613
 }
 
 fn is_quoted_headline(buf: &str) -> bool {
@@ -240,7 +273,7 @@ mod tests {
         #[test]
         // proptest does not support [[:graphical:][:space:]] regex.
         fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let (output, _is_forwarded, _) = simplify(input, true);
+            let (output, _is_forwarded, _, _, _) = simplify(input, true);
             assert!(output.split('\n').all(|s| s != "-- "));
         }
     }
@@ -248,38 +281,47 @@ mod tests {
     #[test]
     fn test_dont_remove_whole_message() {
         let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
-        let (plain, is_forwarded, _) = simplify(input, false);
+        let (plain, is_forwarded, is_cut, _, _) = simplify(input, false);
         assert_eq!(
             plain,
             "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
         );
         assert!(!is_forwarded);
+        assert!(!is_cut);
     }
 
     #[test]
     fn test_chat_message() {
         let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
-        let (plain, is_forwarded, _) = simplify(input, true);
+        let (plain, is_forwarded, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
         assert!(!is_forwarded);
+        assert!(is_cut);
+        assert_eq!(
+            footer.unwrap(),
+            "Sent with my Delta Chat Messenger: https://delta.chat"
+        );
     }
 
     #[test]
     fn test_simplify_trim() {
         let input = "line1\n\r\r\rline2".to_string();
-        let (plain, is_forwarded, _) = simplify(input, false);
+        let (plain, is_forwarded, is_cut, _, _) = simplify(input, false);
 
         assert_eq!(plain, "line1\nline2");
         assert!(!is_forwarded);
+        assert!(!is_cut);
     }
 
     #[test]
     fn test_simplify_forwarded_message() {
         let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string();
-        let (plain, is_forwarded, _) = simplify(input, false);
+        let (plain, is_forwarded, is_cut, _, footer) = simplify(input, false);
 
         assert_eq!(plain, "Forwarded message");
         assert!(is_forwarded);
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "Signature goes here");
     }
 
     #[test]
@@ -321,41 +363,60 @@ mod tests {
     #[test]
     fn test_remove_message_footer() {
         let input = "text\n--\nno footer".to_string();
-        let (plain, _, _) = simplify(input, true);
+        let (plain, _, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n--\nno footer");
+        assert_eq!(footer, None);
+        assert!(!is_cut);
 
         let input = "text\n\n--\n\nno footer".to_string();
-        let (plain, _, _) = simplify(input, true);
+        let (plain, _, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\n\nno footer");
+        assert_eq!(footer, None);
+        assert!(!is_cut);
 
         let input = "text\n\n-- no footer\n\n".to_string();
-        let (plain, _, _) = simplify(input, true);
+        let (plain, _, _, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n\n-- no footer");
+        assert_eq!(footer, None);
 
         let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
-        let (plain, _, _) = simplify(input, true);
+        let (plain, _, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\nno footer");
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "footer");
 
         let input = "text\n\n--\ntreated as footer when unescaped".to_string();
-        let (plain, _, _) = simplify(input.clone(), true);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), true);
         assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "treated as footer when unescaped");
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _, _) = simplify(escaped, true);
+        let (plain, _, is_cut, _, footer) = simplify(escaped, true);
         assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
+        assert!(!is_cut);
+        assert_eq!(footer, None);
 
         // Nonstandard footer sent by https://siju.es/
         let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
-        let (plain, _, _) = simplify(input.clone(), false);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), false);
         assert_eq!(plain, "Message text here [...]");
-        let (plain, _, _) = simplify(input.clone(), true);
+        assert!(is_cut);
+        assert_eq!(footer, None);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), true);
         assert_eq!(plain, input);
+        assert!(!is_cut);
+        assert_eq!(footer, None);
 
         let input = "--\ntreated as footer when unescaped".to_string();
-        let (plain, _, _) = simplify(input.clone(), true);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), true);
         assert_eq!(plain, ""); // see remove_message_footer() for some explanations
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "treated as footer when unescaped");
 
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _, _) = simplify(escaped, true);
+        let (plain, _, is_cut, _, footer) = simplify(escaped, true);
         assert_eq!(plain, "--\ntreated as footer when unescaped");
+        assert!(!is_cut);
+        assert_eq!(footer, None);
     }
 }

src/smtp/mod.rs

@@ -4,16 +4,16 @@ pub mod send;
 
 use std::time::{Duration, SystemTime};
 
-use async_smtp::smtp::client::net::*;
-use async_smtp::*;
+use async_smtp::smtp::client::net::ClientTlsParameters;
+use async_smtp::{error, smtp, EmailAddress};
 
-use crate::constants::*;
+use crate::constants::DC_LP_AUTH_OAUTH2;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam};
-use crate::oauth2::*;
-use crate::provider::{get_provider_info, Socket};
-use crate::stock::StockMessage;
+use crate::oauth2::dc_get_oauth2_access_token;
+use crate::provider::Socket;
+use crate::stock_str;
 
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
@@ -107,16 +107,16 @@ impl Smtp {
                 &lp.smtp,
                 &lp.addr,
                 lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+                lp.provider.map_or(false, |provider| provider.strict_tls),
             )
             .await;
         if let Err(ref err) = res {
-            let message = context
-                .stock_string_repl_str2(
-                    StockMessage::ServerResponse,
-                    format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
-                    err.to_string(),
-                )
-                .await;
+            let message = stock_str::server_response(
+                context,
+                format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
+                err.to_string(),
+            )
+            .await;
 
             context.emit_event(EventType::ErrorNetwork(message));
         };
@@ -130,6 +130,7 @@ impl Smtp {
         lp: &ServerLoginParam,
         addr: &str,
         oauth2: bool,
+        provider_strict_tls: bool,
     ) -> Result<()> {
         if self.is_connected().await {
             warn!(context, "SMTP already connected.");
@@ -151,9 +152,8 @@ impl Smtp {
         let domain = &lp.server;
         let port = lp.port;
 
-        let provider = get_provider_info(addr);
         let strict_tls = match lp.certificate_checks {
-            CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
+            CertificateChecks::Automatic => provider_strict_tls,
             CertificateChecks::Strict => true,
             CertificateChecks::AcceptInvalidCertificates
             | CertificateChecks::AcceptInvalidCertificates2 => false,
@@ -192,7 +192,8 @@ impl Smtp {
         };
 
         let security = match lp.security {
-            Socket::STARTTLS | Socket::Plain => smtp::ClientSecurity::Opportunistic(tls_parameters),
+            Socket::Plain => smtp::ClientSecurity::None,
+            Socket::STARTTLS => smtp::ClientSecurity::Required(tls_parameters),
             _ => smtp::ClientSecurity::Wrapper(tls_parameters),
         };
 

src/smtp/send.rs

@@ -1,8 +1,9 @@
 //! # SMTP message sending
 
 use super::Smtp;
-use async_smtp::*;
+use async_smtp::{EmailAddress, Envelope, SendableEmail, Transport};
 
+use crate::constants::DEFAULT_MAX_SMTP_RCPT_TO;
 use crate::context::Context;
 use crate::events::EventType;
 use itertools::Itertools;
@@ -34,37 +35,46 @@ impl Smtp {
     ) -> Result<()> {
         let message_len_bytes = message.len();
 
-        let recipients_display = recipients.iter().map(|x| x.to_string()).join(",");
-
-        let envelope =
-            Envelope::new(self.from.clone(), recipients).map_err(Error::EnvelopeError)?;
-        let mail = SendableEmail::new(
-            envelope,
-            format!("{}", job_id), // only used for internal logging
-            message,
-        );
-
-        if let Some(ref mut transport) = self.transport {
-            // The timeout is 1min + 3min per MB.
-            let timeout = 60 + (180 * message_len_bytes / 1_000_000) as u64;
-            transport
-                .send_with_timeout(mail, Some(&Duration::from_secs(timeout)))
-                .await
-                .map_err(Error::SendError)?;
-
-            context.emit_event(EventType::SmtpMessageSent(format!(
-                "Message len={} was smtp-sent to {}",
-                message_len_bytes, recipients_display
-            )));
-            self.last_success = Some(std::time::SystemTime::now());
-
-            Ok(())
-        } else {
-            warn!(
-                context,
-                "uh? SMTP has no transport, failed to send to {}", recipients_display
-            );
-            Err(Error::NoTransport)
+        let mut chunk_size = DEFAULT_MAX_SMTP_RCPT_TO;
+        if let Some(provider) = context.get_configured_provider().await {
+            if let Some(max_smtp_rcpt_to) = provider.max_smtp_rcpt_to {
+                chunk_size = max_smtp_rcpt_to as usize;
+            }
         }
+
+        for recipients_chunk in recipients.chunks(chunk_size).into_iter() {
+            let recipients = recipients_chunk.to_vec();
+            let recipients_display = recipients.iter().map(|x| x.to_string()).join(",");
+
+            let envelope =
+                Envelope::new(self.from.clone(), recipients).map_err(Error::EnvelopeError)?;
+            let mail = SendableEmail::new(
+                envelope,
+                format!("{}", job_id), // only used for internal logging
+                &message,
+            );
+
+            if let Some(ref mut transport) = self.transport {
+                // The timeout is 1min + 3min per MB.
+                let timeout = 60 + (180 * message_len_bytes / 1_000_000) as u64;
+                transport
+                    .send_with_timeout(mail, Some(&Duration::from_secs(timeout)))
+                    .await
+                    .map_err(Error::SendError)?;
+
+                context.emit_event(EventType::SmtpMessageSent(format!(
+                    "Message len={} was smtp-sent to {}",
+                    message_len_bytes, recipients_display
+                )));
+                self.last_success = Some(std::time::SystemTime::now());
+            } else {
+                warn!(
+                    context,
+                    "uh? SMTP has no transport, failed to send to {}", recipients_display
+                );
+                return Err(Error::NoTransport);
+            }
+        }
+        Ok(())
     }
 }

src/sql.rs

@@ -7,16 +7,23 @@ use std::collections::HashSet;
 use std::path::Path;
 use std::time::Duration;
 
+use anyhow::format_err;
+use anyhow::Context as _;
 use rusqlite::{Connection, Error as SqlError, OpenFlags};
 
-use crate::chat::{update_device_icon, update_saved_messages_icon};
-use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH};
+use crate::chat::{add_device_msg, update_device_icon, update_saved_messages_icon};
+use crate::config::Config;
+use crate::config::Config::DeleteServerAfter;
+use crate::constants::{ShowEmails, Viewtype, DC_CHAT_ID_TRASH};
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_delete_file, time, EmailAddress};
 use crate::ephemeral::start_ephemeral_timers;
-use crate::error::format_err;
-use crate::param::*;
-use crate::peerstate::*;
+use crate::imap;
+use crate::message::Message;
+use crate::param::{Param, Params};
+use crate::peerstate::Peerstate;
+use crate::provider::get_provider_by_domain;
+use crate::stock_str;
 
 #[macro_export]
 macro_rules! paramsv {
@@ -45,7 +52,7 @@ pub enum Error {
     #[error("{0:?}")]
     BlobError(#[from] crate::blob::BlobError),
     #[error("{0}")]
-    Other(#[from] crate::error::Error),
+    Other(#[from] anyhow::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -83,7 +90,7 @@ impl Sql {
         context: &Context,
         dbfile: T,
         readonly: bool,
-    ) -> crate::error::Result<()> {
+    ) -> anyhow::Result<()> {
         let res = open(context, self, &dbfile, readonly).await;
         if let Err(err) = &res {
             match err.downcast_ref::<Error>() {
@@ -220,6 +227,31 @@ impl Sql {
         .await
     }
 
+    /// Check if a column exists in a given table.
+    pub async fn col_exists(
+        &self,
+        table_name: impl AsRef<str>,
+        col_name: impl AsRef<str>,
+    ) -> Result<bool> {
+        let table_name = table_name.as_ref().to_string();
+        let col_name = col_name.as_ref().to_string();
+        self.with_conn(move |conn| {
+            let mut exists = false;
+            // `PRAGMA table_info` returns one row per column,
+            // each row containing 0=cid, 1=name, 2=type, 3=notnull, 4=dflt_value
+            conn.pragma(None, "table_info", &table_name, |row| {
+                let curr_name: String = row.get(1)?;
+                if col_name == curr_name {
+                    exists = true;
+                }
+                Ok(())
+            })?;
+
+            Ok(exists)
+        })
+        .await
+    }
+
     /// Execute a query which is expected to return zero or one row.
     pub async fn query_row_optional<T, F>(
         &self,
@@ -293,7 +325,7 @@ impl Sql {
         }
 
         let key = key.as_ref();
-        let res = if let Some(ref value) = value {
+        let res = if let Some(value) = value {
             let exists = self
                 .exists("SELECT value FROM config WHERE keyname=?;", paramsv![key])
                 .await?;
@@ -464,7 +496,11 @@ pub fn get_rowid2(
     )
 }
 
-pub async fn housekeeping(context: &Context) {
+pub async fn housekeeping(context: &Context) -> anyhow::Result<()> {
+    if let Err(err) = crate::ephemeral::delete_expired_messages(context).await {
+        warn!(context, "Failed to delete expired messages: {}", err);
+    }
+
     let mut files_in_use = HashSet::new();
     let mut unreferenced_count = 0;
 
@@ -475,28 +511,28 @@ pub async fn housekeeping(context: &Context) {
         "SELECT param FROM msgs  WHERE chat_id!=3   AND type!=10;",
         Param::File,
     )
-    .await;
+    .await?;
     maybe_add_from_param(
         context,
         &mut files_in_use,
         "SELECT param FROM jobs;",
         Param::File,
     )
-    .await;
+    .await?;
     maybe_add_from_param(
         context,
         &mut files_in_use,
         "SELECT param FROM chats;",
         Param::ProfileImage,
     )
-    .await;
+    .await?;
     maybe_add_from_param(
         context,
         &mut files_in_use,
         "SELECT param FROM contacts;",
         Param::ProfileImage,
     )
-    .await;
+    .await?;
 
     context
         .sql
@@ -512,9 +548,7 @@ pub async fn housekeeping(context: &Context) {
             },
         )
         .await
-        .unwrap_or_else(|err| {
-            warn!(context, "sql: failed query: {}", err);
-        });
+        .context("housekeeping: failed to SELECT value FROM config")?;
 
     info!(context, "{} files in use.", files_in_use.len(),);
     /* go through directory and delete unused files */
@@ -595,7 +629,14 @@ pub async fn housekeeping(context: &Context) {
         );
     }
 
-    info!(context, "Housekeeping done.",);
+    if let Err(e) = context
+        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
+        .await
+    {
+        warn!(context, "Can't set config: {}", e);
+    }
+    info!(context, "Housekeeping done.");
+    Ok(())
 }
 
 #[allow(clippy::indexing_slicing)]
@@ -624,7 +665,7 @@ async fn maybe_add_from_param(
     files_in_use: &mut HashSet<String>,
     query: &str,
     param_id: Param,
-) {
+) -> anyhow::Result<()> {
     context
         .sql
         .query_map(
@@ -642,9 +683,7 @@ async fn maybe_add_from_param(
             },
         )
         .await
-        .unwrap_or_else(|err| {
-            warn!(context, "sql: failed to add_from_param: {}", err);
-        });
+        .context(format!("housekeeping: failed to add_from_param {}", query))
 }
 
 #[allow(clippy::cognitive_complexity)]
@@ -653,7 +692,7 @@ async fn open(
     sql: &Sql,
     dbfile: impl AsRef<Path>,
     readonly: bool,
-) -> crate::error::Result<()> {
+) -> anyhow::Result<()> {
     if sql.is_open().await {
         error!(
             context,
@@ -926,6 +965,7 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
         let mut dbversion = dbversion_before_update;
         let mut recalc_fingerprints = false;
         let mut update_icons = !exists_before_update;
+        let mut disable_server_delete = false;
 
         if dbversion < 1 {
             info!(context, "[migration] v1");
@@ -1382,6 +1422,97 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
             .await?;
             sql.set_raw_config_int(context, "dbversion", 69).await?;
         }
+        if dbversion < 71 {
+            info!(context, "[migration] v71");
+            if let Some(addr) = context.get_config(Config::ConfiguredAddr).await {
+                if let Ok(domain) = addr.parse::<EmailAddress>().map(|email| email.domain) {
+                    context
+                        .set_config(
+                            Config::ConfiguredProvider,
+                            get_provider_by_domain(&domain).map(|provider| provider.id),
+                        )
+                        .await?;
+                } else {
+                    warn!(context, "Can't parse configured address: {:?}", addr);
+                }
+            }
+
+            sql.set_raw_config_int(context, "dbversion", 71).await?;
+        }
+        if dbversion < 72 {
+            info!(context, "[migration] v72");
+            if !sql.col_exists("msgs", "mime_modified").await? {
+                sql.execute(
+                    "ALTER TABLE msgs ADD COLUMN mime_modified INTEGER DEFAULT 0;",
+                    paramsv![],
+                )
+                .await?;
+            }
+            sql.set_raw_config_int(context, "dbversion", 72).await?;
+        }
+        if dbversion < 73 {
+            use Config::*;
+            info!(context, "[migration] v73");
+            sql.execute(
+                "CREATE TABLE imap_sync (folder TEXT PRIMARY KEY, uidvalidity INTEGER DEFAULT 0, uid_next INTEGER DEFAULT 0);",
+                paramsv![],
+            )
+            .await?;
+            for c in &[
+                ConfiguredInboxFolder,
+                ConfiguredSentboxFolder,
+                ConfiguredMvboxFolder,
+            ] {
+                if let Some(folder) = context.get_config(*c).await {
+                    let (uid_validity, last_seen_uid) =
+                        imap::get_config_last_seen_uid(context, &folder).await;
+                    if last_seen_uid > 0 {
+                        imap::set_uid_next(context, &folder, last_seen_uid + 1).await?;
+                        imap::set_uidvalidity(context, &folder, uid_validity).await?;
+                    }
+                }
+            }
+            if exists_before_update {
+                disable_server_delete = true;
+
+                // Don't disable server delete if it was on by default (Nauta):
+                if let Some(provider) = context.get_configured_provider().await {
+                    if let Some(defaults) = &provider.config_defaults {
+                        if defaults.iter().any(|d| d.key == Config::DeleteServerAfter) {
+                            disable_server_delete = false;
+                        }
+                    }
+                }
+            }
+            sql.set_raw_config_int(context, "dbversion", 73).await?;
+        }
+        if dbversion < 74 {
+            info!(context, "[migration] v74");
+            sql.execute(
+                "UPDATE contacts SET name='' WHERE name=authname",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 74).await?;
+        }
+        if dbversion < 75 {
+            info!(context, "[migration] v75");
+            sql.execute(
+                "ALTER TABLE contacts ADD COLUMN status TEXT DEFAULT '';",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 75).await?;
+        }
+        if dbversion < 76 {
+            info!(context, "[migration] v76");
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN subject TEXT DEFAULT '';",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 76).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)
@@ -1412,6 +1543,16 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
             update_saved_messages_icon(context).await?;
             update_device_icon(context).await?;
         }
+        if disable_server_delete {
+            // We now always watch all folders and delete messages there if delete_server is enabled.
+            // So, for people who have delete_server enabled, disable it and add a hint to the devicechat:
+            if context.get_config_delete_server_after().await.is_some() {
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text = Some(stock_str::delete_server_turned_off(context).await);
+                add_device_msg(context, None, Some(&mut msg)).await?;
+                context.set_config(DeleteServerAfter, Some("0")).await?;
+            }
+        }
     }
 
     info!(context, "Opened {:?}.", dbfile.as_ref(),);
@@ -1436,7 +1577,10 @@ async fn prune_tombstones(context: &Context) -> Result<()> {
 
 #[cfg(test)]
 mod test {
+    use async_std::fs::File;
+
     use super::*;
+    use crate::{test_utils::TestContext, Event, EventType};
 
     #[test]
     fn test_maybe_add_file() {
@@ -1463,4 +1607,59 @@ mod test {
         assert!(!is_file_in_use(&files, Some(".txt"), "hello"));
         assert!(is_file_in_use(&files, Some("-suffix"), "world.txt-suffix"));
     }
+
+    #[async_std::test]
+    async fn test_table_exists() {
+        let t = TestContext::new().await;
+        assert!(t.ctx.sql.table_exists("msgs").await.unwrap());
+        assert!(!t.ctx.sql.table_exists("foobar").await.unwrap());
+    }
+
+    #[async_std::test]
+    async fn test_col_exists() {
+        let t = TestContext::new().await;
+        assert!(t.ctx.sql.col_exists("msgs", "mime_modified").await.unwrap());
+        assert!(!t.ctx.sql.col_exists("msgs", "foobar").await.unwrap());
+        assert!(!t.ctx.sql.col_exists("foobar", "foobar").await.unwrap());
+    }
+
+    #[async_std::test]
+    async fn test_housekeeping_db_closed() {
+        let t = TestContext::new().await;
+
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+
+        t.add_event_sink(move |event: Event| async move {
+            match event.typ {
+                EventType::Info(s) => assert!(
+                    !s.contains("Keeping new unreferenced file"),
+                    "File {} was almost deleted, only reason it was kept is that it was created recently (as the tests don't run for a long time)",
+                    s
+                ),
+                EventType::Error(s) => panic!(s),
+                _ => {}
+            }
+        })
+        .await;
+
+        let a = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_bytes, &async_std::fs::read(&a).await.unwrap()[..]);
+
+        t.sql.close().await;
+        housekeeping(&t).await.unwrap_err(); // housekeeping should fail as the db is closed
+        t.sql.open(&t, &t.get_dbfile(), false).await.unwrap();
+
+        let a = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_bytes, &async_std::fs::read(&a).await.unwrap()[..]);
+    }
 }

src/stock.rs

@@ -1,667 +0,0 @@
-//! Module to work with translatable stock strings
-
-use std::borrow::Cow;
-
-use strum::EnumProperty;
-use strum_macros::EnumProperty;
-
-use crate::chat;
-use crate::chat::ProtectionStatus;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
-use crate::contact::*;
-use crate::context::Context;
-use crate::error::{bail, Error};
-use crate::message::Message;
-use crate::param::Param;
-use crate::stock::StockMessage::{DeviceMessagesHint, WelcomeMessage};
-use crate::{blob::BlobObject, config::Config};
-
-/// Stock strings
-///
-/// These identify the string to return in [Context.stock_str].  The
-/// numbers must stay in sync with `deltachat.h` `DC_STR_*` constants.
-///
-/// See the `stock_*` methods on [Context] to use these.
-///
-/// [Context]: crate::context::Context
-#[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, EnumProperty)]
-#[repr(u32)]
-pub enum StockMessage {
-    #[strum(props(fallback = "No messages."))]
-    NoMessages = 1,
-
-    #[strum(props(fallback = "Me"))]
-    SelfMsg = 2,
-
-    #[strum(props(fallback = "Draft"))]
-    Draft = 3,
-
-    #[strum(props(fallback = "Voice message"))]
-    VoiceMessage = 7,
-
-    #[strum(props(fallback = "Contact requests"))]
-    DeadDrop = 8,
-
-    #[strum(props(fallback = "Image"))]
-    Image = 9,
-
-    #[strum(props(fallback = "Video"))]
-    Video = 10,
-
-    #[strum(props(fallback = "Audio"))]
-    Audio = 11,
-
-    #[strum(props(fallback = "File"))]
-    File = 12,
-
-    #[strum(props(fallback = "Sent with my Delta Chat Messenger: https://delta.chat"))]
-    StatusLine = 13,
-
-    #[strum(props(fallback = "Hello, I\'ve just created the group \"%1$s\" for us."))]
-    NewGroupDraft = 14,
-
-    #[strum(props(fallback = "Group name changed from \"%1$s\" to \"%2$s\"."))]
-    MsgGrpName = 15,
-
-    #[strum(props(fallback = "Group image changed."))]
-    MsgGrpImgChanged = 16,
-
-    #[strum(props(fallback = "Member %1$s added."))]
-    MsgAddMember = 17,
-
-    #[strum(props(fallback = "Member %1$s removed."))]
-    MsgDelMember = 18,
-
-    #[strum(props(fallback = "Group left."))]
-    MsgGroupLeft = 19,
-
-    #[strum(props(fallback = "GIF"))]
-    Gif = 23,
-
-    #[strum(props(fallback = "Encrypted message"))]
-    EncryptedMsg = 24,
-
-    #[strum(props(fallback = "End-to-end encryption available."))]
-    E2eAvailable = 25,
-
-    #[strum(props(fallback = "Transport-encryption."))]
-    EncrTransp = 27,
-
-    #[strum(props(fallback = "No encryption."))]
-    EncrNone = 28,
-
-    #[strum(props(fallback = "This message was encrypted for another setup."))]
-    CantDecryptMsgBody = 29,
-
-    #[strum(props(fallback = "Fingerprints"))]
-    FingerPrints = 30,
-
-    #[strum(props(fallback = "Return receipt"))]
-    ReadRcpt = 31,
-
-    #[strum(props(fallback = "This is a return receipt for the message \"%1$s\"."))]
-    ReadRcptMailBody = 32,
-
-    #[strum(props(fallback = "Group image deleted."))]
-    MsgGrpImgDeleted = 33,
-
-    #[strum(props(fallback = "End-to-end encryption preferred."))]
-    E2ePreferred = 34,
-
-    #[strum(props(fallback = "%1$s verified."))]
-    ContactVerified = 35,
-
-    #[strum(props(fallback = "Cannot verify %1$s"))]
-    ContactNotVerified = 36,
-
-    #[strum(props(fallback = "Changed setup for %1$s"))]
-    ContactSetupChanged = 37,
-
-    #[strum(props(fallback = "Archived chats"))]
-    ArchivedChats = 40,
-
-    #[strum(props(fallback = "Autocrypt Setup Message"))]
-    AcSetupMsgSubject = 42,
-
-    #[strum(props(
-        fallback = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\nTo decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device."
-    ))]
-    AcSetupMsgBody = 43,
-
-    #[strum(props(
-        fallback = "Cannot login as \"%1$s\". Please check if the email address and the password are correct."
-    ))]
-    CannotLogin = 60,
-
-    #[strum(props(fallback = "Could not connect to %1$s: %2$s"))]
-    ServerResponse = 61,
-
-    #[strum(props(fallback = "%1$s by %2$s."))]
-    MsgActionByUser = 62,
-
-    #[strum(props(fallback = "%1$s by me."))]
-    MsgActionByMe = 63,
-
-    #[strum(props(fallback = "Location streaming enabled."))]
-    MsgLocationEnabled = 64,
-
-    #[strum(props(fallback = "Location streaming disabled."))]
-    MsgLocationDisabled = 65,
-
-    #[strum(props(fallback = "Location"))]
-    Location = 66,
-
-    #[strum(props(fallback = "Sticker"))]
-    Sticker = 67,
-
-    #[strum(props(fallback = "Device messages"))]
-    DeviceMessages = 68,
-
-    #[strum(props(fallback = "Saved messages"))]
-    SavedMessages = 69,
-
-    #[strum(props(
-        fallback = "Messages in this chat are generated locally by your Delta Chat app. \
-                    Its makers use it to inform about app updates and problems during usage."
-    ))]
-    DeviceMessagesHint = 70,
-
-    #[strum(props(fallback = "Welcome to Delta Chat! – \
-                    Delta Chat looks and feels like other popular messenger apps, \
-                    but does not involve centralized control, \
-                    tracking or selling you, friends, colleagues or family out to large organizations.\n\n\
-                    Technically, Delta Chat is an email application with a modern chat interface. \
-                    Email in a new dress if you will 👻\n\n\
-                    Use Delta Chat with anyone out of billions of people: just use their e-mail address. \
-                    Recipients don't need to install Delta Chat, visit websites or sign up anywhere - \
-                    however, of course, if they like, you may point them to 👉 https://get.delta.chat"))]
-    WelcomeMessage = 71,
-
-    #[strum(props(fallback = "Unknown sender for this chat. See 'info' for more details."))]
-    UnknownSenderForChat = 72,
-
-    #[strum(props(fallback = "Message from %1$s"))]
-    SubjectForNewContact = 73,
-
-    #[strum(props(fallback = "Failed to send message to %1$s."))]
-    FailedSendingTo = 74,
-
-    #[strum(props(fallback = "Message deletion timer is disabled."))]
-    MsgEphemeralTimerDisabled = 75,
-
-    // A fallback message for unknown timer values.
-    // "s" stands for "second" SI unit here.
-    #[strum(props(fallback = "Message deletion timer is set to %1$s s."))]
-    MsgEphemeralTimerEnabled = 76,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 minute."))]
-    MsgEphemeralTimerMinute = 77,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 hour."))]
-    MsgEphemeralTimerHour = 78,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 day."))]
-    MsgEphemeralTimerDay = 79,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 week."))]
-    MsgEphemeralTimerWeek = 80,
-
-    #[strum(props(fallback = "Message deletion timer is set to 4 weeks."))]
-    MsgEphemeralTimerFourWeeks = 81,
-
-    #[strum(props(fallback = "Video chat invitation"))]
-    VideochatInvitation = 82,
-
-    #[strum(props(fallback = "You are invited to a video chat, click %1$s to join."))]
-    VideochatInviteMsgBody = 83,
-
-    #[strum(props(fallback = "Error:\n\n“%1$s”"))]
-    ConfigurationFailed = 84,
-
-    #[strum(props(
-        fallback = "⚠️ Date or time of your device seem to be inaccurate (%1$s).\n\n\
-                    Adjust your clock ⏰🔧 to ensure your messages are received correctly."
-    ))]
-    BadTimeMsgBody = 85,
-
-    #[strum(props(fallback = "⚠️ Your Delta Chat version might be outdated.\n\n\
-                    This may cause problems because your chat partners use newer versions - \
-                    and you are missing the latest features 😳\n\
-                    Please check https://get.delta.chat or your app store for updates."))]
-    UpdateReminderMsgBody = 86,
-
-    #[strum(props(
-        fallback = "Could not find your mail server.\n\nPlease check your internet connection."
-    ))]
-    ErrorNoNetwork = 87,
-
-    #[strum(props(fallback = "Chat protection enabled."))]
-    ProtectionEnabled = 88,
-
-    #[strum(props(fallback = "Chat protection disabled."))]
-    ProtectionDisabled = 89,
-
-    // used in summaries, a noun, not a verb (not: "to reply")
-    #[strum(props(fallback = "Reply"))]
-    ReplyNoun = 90,
-}
-
-/*
-"
-*/
-
-impl StockMessage {
-    /// Default untranslated strings for stock messages.
-    ///
-    /// These could be used in logging calls, so no logging here.
-    fn fallback(self) -> &'static str {
-        self.get_str("fallback").unwrap_or_default()
-    }
-}
-
-impl Context {
-    /// Set the stock string for the [StockMessage].
-    ///
-    pub async fn set_stock_translation(
-        &self,
-        id: StockMessage,
-        stockstring: String,
-    ) -> Result<(), Error> {
-        if stockstring.contains("%1") && !id.fallback().contains("%1") {
-            bail!(
-                "translation {} contains invalid %1 placeholder, default is {}",
-                stockstring,
-                id.fallback()
-            );
-        }
-        if stockstring.contains("%2") && !id.fallback().contains("%2") {
-            bail!(
-                "translation {} contains invalid %2 placeholder, default is {}",
-                stockstring,
-                id.fallback()
-            );
-        }
-        self.translated_stockstrings
-            .write()
-            .await
-            .insert(id as usize, stockstring);
-        Ok(())
-    }
-
-    /// Return the stock string for the [StockMessage].
-    ///
-    /// Return a translation (if it was set with set_stock_translation before)
-    /// or a default (English) string.
-    pub async fn stock_str(&self, id: StockMessage) -> Cow<'_, str> {
-        match self
-            .translated_stockstrings
-            .read()
-            .await
-            .get(&(id as usize))
-        {
-            Some(ref x) => Cow::Owned((*x).to_string()),
-            None => Cow::Borrowed(id.fallback()),
-        }
-    }
-
-    /// Return stock string, replacing placeholders with provided string.
-    ///
-    /// This replaces both the *first* `%1$s`, `%1$d` and `%1$@`
-    /// placeholders with the provided string.
-    /// (the `%1$@` variant is used on iOS, the other are used on Android and Desktop)
-    pub async fn stock_string_repl_str(&self, id: StockMessage, insert: impl AsRef<str>) -> String {
-        self.stock_str(id)
-            .await
-            .replacen("%1$s", insert.as_ref(), 1)
-            .replacen("%1$d", insert.as_ref(), 1)
-            .replacen("%1$@", insert.as_ref(), 1)
-    }
-
-    /// Return stock string, replacing placeholders with provided int.
-    ///
-    /// Like [Context::stock_string_repl_str] but substitute the placeholders
-    /// with an integer.
-    pub async fn stock_string_repl_int(&self, id: StockMessage, insert: i32) -> String {
-        self.stock_string_repl_str(id, format!("{}", insert).as_str())
-            .await
-    }
-
-    /// Return stock string, replacing 2 placeholders with provided string.
-    ///
-    /// This replaces both the *first* `%1$s`, `%1$d` and `%1$@`
-    /// placeholders with the string in `insert` and does the same for
-    /// `%2$s`, `%2$d` and `%2$@` for `insert2`.
-    /// (the `%1$@` variant is used on iOS, the other are used on Android and Desktop)
-    pub async fn stock_string_repl_str2(
-        &self,
-        id: StockMessage,
-        insert: impl AsRef<str>,
-        insert2: impl AsRef<str>,
-    ) -> String {
-        self.stock_str(id)
-            .await
-            .replacen("%1$s", insert.as_ref(), 1)
-            .replacen("%1$d", insert.as_ref(), 1)
-            .replacen("%1$@", insert.as_ref(), 1)
-            .replacen("%2$s", insert2.as_ref(), 1)
-            .replacen("%2$d", insert2.as_ref(), 1)
-            .replacen("%2$@", insert2.as_ref(), 1)
-    }
-
-    /// Return some kind of stock message
-    ///
-    /// If the `id` is [StockMessage::MsgAddMember] or
-    /// [StockMessage::MsgDelMember] then `param1` is considered to be the
-    /// contact address and will be replaced by that contact's display
-    /// name.
-    ///
-    /// If `from_id` is not `0`, any trailing dot is removed from the
-    /// first stock string created so far.  If the `from_id` contact is
-    /// the user itself, i.e. `DC_CONTACT_ID_SELF` the string is used
-    /// itself as param to the [StockMessage::MsgActionByMe] stock string
-    /// resulting in a string like "Member Alice added by me." (for
-    /// [StockMessage::MsgAddMember] as `id`).  If the `from_id` contact
-    /// is any other user than the contact's display name is looked up and
-    /// used as the second parameter to [StockMessage::MsgActionByUser] with
-    /// again the original stock string being used as the first parameter,
-    /// resulting in a string like "Member Alice added by Bob.".
-    pub async fn stock_system_msg(
-        &self,
-        id: StockMessage,
-        param1: impl AsRef<str>,
-        param2: impl AsRef<str>,
-        from_id: u32,
-    ) -> String {
-        let insert1 = if id == StockMessage::MsgAddMember || id == StockMessage::MsgDelMember {
-            let contact_id =
-                Contact::lookup_id_by_addr(self, param1.as_ref(), Origin::Unknown).await;
-            if contact_id != 0 {
-                Contact::get_by_id(self, contact_id)
-                    .await
-                    .map(|contact| contact.get_name_n_addr())
-                    .unwrap_or_default()
-            } else {
-                param1.as_ref().to_string()
-            }
-        } else {
-            param1.as_ref().to_string()
-        };
-
-        let action = self
-            .stock_string_repl_str2(id, insert1, param2.as_ref().to_string())
-            .await;
-        let action1 = action.trim_end_matches('.');
-        match from_id {
-            0 => action,
-            DC_CONTACT_ID_SELF => {
-                self.stock_string_repl_str(StockMessage::MsgActionByMe, action1)
-                    .await
-            }
-            _ => {
-                let displayname = Contact::get_by_id(self, from_id)
-                    .await
-                    .map(|contact| contact.get_name_n_addr())
-                    .unwrap_or_default();
-
-                self.stock_string_repl_str2(StockMessage::MsgActionByUser, action1, &displayname)
-                    .await
-            }
-        }
-    }
-
-    /// Returns a stock message saying that protection status has changed.
-    pub async fn stock_protection_msg(&self, protect: ProtectionStatus, from_id: u32) -> String {
-        self.stock_system_msg(
-            match protect {
-                ProtectionStatus::Protected => StockMessage::ProtectionEnabled,
-                ProtectionStatus::Unprotected => StockMessage::ProtectionDisabled,
-            },
-            "",
-            "",
-            from_id,
-        )
-        .await
-    }
-
-    pub(crate) async fn update_device_chats(&self) -> Result<(), Error> {
-        if self.get_config_bool(Config::Bot).await {
-            return Ok(());
-        }
-
-        // create saved-messages chat; we do this only once, if the user has deleted the chat,
-        // he can recreate it manually (make sure we do not re-add it when configure() was called a second time)
-        if !self.sql.get_raw_config_bool(&self, "self-chat-added").await {
-            self.sql
-                .set_raw_config_bool(&self, "self-chat-added", true)
-                .await?;
-            chat::create_by_contact_id(&self, DC_CONTACT_ID_SELF).await?;
-        }
-
-        // add welcome-messages. by the label, this is done only once,
-        // if the user has deleted the message or the chat, it is not added again.
-        let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(self.stock_str(DeviceMessagesHint).await.to_string());
-        chat::add_device_msg(&self, Some("core-about-device-chat"), Some(&mut msg)).await?;
-
-        let image = include_bytes!("../assets/welcome-image.jpg");
-        let blob = BlobObject::create(&self, "welcome-image.jpg".to_string(), image).await?;
-        let mut msg = Message::new(Viewtype::Image);
-        msg.param.set(Param::File, blob.as_name());
-        chat::add_device_msg(&self, Some("core-welcome-image"), Some(&mut msg)).await?;
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(self.stock_str(WelcomeMessage).await.to_string());
-        chat::add_device_msg(&self, Some("core-welcome"), Some(&mut msg)).await?;
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::test_utils::*;
-
-    use crate::constants::DC_CONTACT_ID_SELF;
-
-    use crate::chatlist::Chatlist;
-    use num_traits::ToPrimitive;
-
-    #[test]
-    fn test_enum_mapping() {
-        assert_eq!(StockMessage::NoMessages.to_usize().unwrap(), 1);
-        assert_eq!(StockMessage::SelfMsg.to_usize().unwrap(), 2);
-    }
-
-    #[test]
-    fn test_fallback() {
-        assert_eq!(StockMessage::NoMessages.fallback(), "No messages.");
-    }
-
-    #[async_std::test]
-    async fn test_set_stock_translation() {
-        let t = TestContext::new().await;
-        t.ctx
-            .set_stock_translation(StockMessage::NoMessages, "xyz".to_string())
-            .await
-            .unwrap();
-        assert_eq!(t.ctx.stock_str(StockMessage::NoMessages).await, "xyz")
-    }
-
-    #[async_std::test]
-    async fn test_set_stock_translation_wrong_replacements() {
-        let t = TestContext::new().await;
-        assert!(t
-            .ctx
-            .set_stock_translation(StockMessage::NoMessages, "xyz %1$s ".to_string())
-            .await
-            .is_err());
-        assert!(t
-            .ctx
-            .set_stock_translation(StockMessage::NoMessages, "xyz %2$s ".to_string())
-            .await
-            .is_err());
-    }
-
-    #[async_std::test]
-    async fn test_stock_str() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx.stock_str(StockMessage::NoMessages).await,
-            "No messages."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_string_repl_str() {
-        let t = TestContext::new().await;
-        // uses %1$s substitution
-        assert_eq!(
-            t.ctx
-                .stock_string_repl_str(StockMessage::MsgAddMember, "Foo")
-                .await,
-            "Member Foo added."
-        );
-        // We have no string using %1$d to test...
-    }
-
-    #[async_std::test]
-    async fn test_stock_string_repl_int() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_string_repl_int(StockMessage::MsgAddMember, 42)
-                .await,
-            "Member 42 added."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_string_repl_str2() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_string_repl_str2(StockMessage::ServerResponse, "foo", "bar")
-                .await,
-            "Could not connect to foo: bar"
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_simple() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
-                .await,
-            "Location streaming enabled."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_add_member_by_me() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgAddMember,
-                    "alice@example.com",
-                    "",
-                    DC_CONTACT_ID_SELF
-                )
-                .await,
-            "Member alice@example.com added by me."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_add_member_by_me_with_displayname() {
-        let t = TestContext::new().await;
-        Contact::create(&t.ctx, "Alice", "alice@example.com")
-            .await
-            .expect("failed to create contact");
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgAddMember,
-                    "alice@example.com",
-                    "",
-                    DC_CONTACT_ID_SELF
-                )
-                .await,
-            "Member Alice (alice@example.com) added by me."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_add_member_by_other_with_displayname() {
-        let t = TestContext::new().await;
-        let contact_id = {
-            Contact::create(&t.ctx, "Alice", "alice@example.com")
-                .await
-                .expect("Failed to create contact Alice");
-            Contact::create(&t.ctx, "Bob", "bob@example.com")
-                .await
-                .expect("failed to create bob")
-        };
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgAddMember,
-                    "alice@example.com",
-                    "",
-                    contact_id,
-                )
-                .await,
-            "Member Alice (alice@example.com) added by Bob (bob@example.com)."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_grp_name() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgGrpName,
-                    "Some chat",
-                    "Other chat",
-                    DC_CONTACT_ID_SELF
-                )
-                .await,
-            "Group name changed from \"Some chat\" to \"Other chat\" by me."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_grp_name_other() {
-        let t = TestContext::new().await;
-        let id = Contact::create(&t.ctx, "Alice", "alice@example.com")
-            .await
-            .expect("failed to create contact");
-
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(StockMessage::MsgGrpName, "Some chat", "Other chat", id)
-                .await,
-            "Group name changed from \"Some chat\" to \"Other chat\" by Alice (alice@example.com)."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_update_device_chats() {
-        let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.ok();
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 2);
-
-        chats.get_chat_id(0).delete(&t.ctx).await.ok();
-        chats.get_chat_id(1).delete(&t.ctx).await.ok();
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 0);
-
-        // a subsequent call to update_device_chats() must not re-add manally deleted messages or chats
-        t.ctx.update_device_chats().await.ok();
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 0);
-    }
-}

src/test_utils.rs

@@ -1,40 +1,73 @@
 //! Utilities to help writing tests.
 //!
-//! This module is only compiled for test runs.
+//! This private module is only compiled for test runs.
 
+use std::ops::Deref;
 use std::str::FromStr;
 use std::time::{Duration, Instant};
+use std::{collections::BTreeMap, panic};
+use std::{fmt, thread};
 
+use ansi_term::Color;
 use async_std::path::PathBuf;
-use async_std::sync::RwLock;
+use async_std::sync::{Arc, RwLock};
+use async_std::{channel, pin::Pin};
+use async_std::{future::Future, task};
+use chat::ChatItem;
+use once_cell::sync::Lazy;
 use tempfile::{tempdir, TempDir};
 
-use crate::chat;
-use crate::chat::{ChatId, ChatItem};
+use crate::chat::{self, Chat, ChatId};
+use crate::chatlist::Chatlist;
 use crate::config::Config;
+use crate::constants::Chattype;
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF, DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
+use crate::contact::{Contact, Origin};
 use crate::context::Context;
 use crate::dc_receive_imf::dc_receive_imf;
 use crate::dc_tools::EmailAddress;
+use crate::events::{Event, EventType};
 use crate::job::Action;
 use crate::key::{self, DcKey};
-use crate::message::Message;
+use crate::message::{update_msg_state, Message, MessageState, MsgId};
 use crate::mimeparser::MimeMessage;
 use crate::param::{Param, Params};
 
+type EventSink =
+    dyn Fn(Event) -> Pin<Box<dyn Future<Output = ()> + Send + 'static>> + Send + Sync + 'static;
+
+/// Map of [`Context::id`] to names for [`TestContext`]s.
+static CONTEXT_NAMES: Lazy<std::sync::RwLock<BTreeMap<u32, String>>> =
+    Lazy::new(|| std::sync::RwLock::new(BTreeMap::new()));
+
 /// A Context and temporary directory.
 ///
 /// The temporary directory can be used to store the SQLite database,
 /// see e.g. [test_context] which does this.
-#[derive(Debug)]
 pub(crate) struct TestContext {
     pub ctx: Context,
     pub dir: TempDir,
     /// Counter for fake IMAP UIDs in [recv_msg], for private use in that function only.
     recv_idx: RwLock<u32>,
+    /// Functions to call for events received.
+    event_sinks: Arc<RwLock<Vec<Box<EventSink>>>>,
+    /// Receives panics from sinks ("sink" means "event handler" here)
+    poison_receiver: channel::Receiver<String>,
+}
+
+impl fmt::Debug for TestContext {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("TestContext")
+            .field("ctx", &self.ctx)
+            .field("dir", &self.dir)
+            .field("recv_idx", &self.recv_idx)
+            .field("event_sinks", &String::from("Vec<EventSink>"))
+            .finish()
+    }
 }
 
 impl TestContext {
-    /// Create a new [TestContext].
+    /// Creates a new [`TestContext`].
     ///
     /// The [Context] will be created and have an SQLite database named "db.sqlite" in the
     /// [TestContext.dir] directory.  This directory is cleaned up when the [TestContext] is
@@ -42,44 +75,111 @@ impl TestContext {
     ///
     /// [Context]: crate::context::Context
     pub async fn new() -> Self {
+        Self::new_named(None).await
+    }
+
+    /// Creates a new [`TestContext`] with a set name used in event logging.
+    pub async fn with_name(name: impl Into<String>) -> Self {
+        Self::new_named(Some(name.into())).await
+    }
+
+    async fn new_named(name: Option<String>) -> Self {
         use rand::Rng;
 
         let dir = tempdir().unwrap();
         let dbfile = dir.path().join("db.sqlite");
         let id = rand::thread_rng().gen();
+        if let Some(name) = name {
+            let mut context_names = CONTEXT_NAMES.write().unwrap();
+            context_names.insert(id, name);
+        }
         let ctx = Context::new("FakeOS".into(), dbfile.into(), id)
             .await
             .unwrap();
+
+        let events = ctx.get_event_emitter();
+        let event_sinks: Arc<RwLock<Vec<Box<EventSink>>>> = Arc::new(RwLock::new(Vec::new()));
+        let sinks = Arc::clone(&event_sinks);
+        let (poison_sender, poison_receiver) = channel::bounded(1);
+        async_std::task::spawn(async move {
+            // Make sure that the test fails if there is a panic on this thread here:
+            let current_id = task::current().id();
+            let orig_hook = panic::take_hook();
+            panic::set_hook(Box::new(move |panic_info| {
+                if task::current().id() == current_id {
+                    poison_sender.try_send(panic_info.to_string()).ok();
+                }
+                orig_hook(panic_info);
+            }));
+
+            while let Some(event) = events.recv().await {
+                {
+                    let sinks = sinks.read().await;
+                    for sink in sinks.iter() {
+                        sink(event.clone()).await;
+                    }
+                }
+                receive_event(event);
+            }
+        });
+
         Self {
             ctx,
             dir,
             recv_idx: RwLock::new(0),
+            event_sinks,
+            poison_receiver,
         }
     }
 
-    /// Create a new configured [TestContext].
+    /// Creates a new configured [`TestContext`].
     ///
-    /// This is a shortcut which automatically calls [TestContext::configure_alice] after
+    /// This is a shortcut which automatically calls [`TestContext::configure_alice`] after
     /// creating the context.
     pub async fn new_alice() -> Self {
-        let t = Self::new().await;
+        let t = Self::with_name("alice").await;
         t.configure_alice().await;
         t
     }
 
-    /// Create a new configured [TestContext].
+    /// Creates a new configured [`TestContext`].
     ///
     /// This is a shortcut which configures bob@example.net with a fixed key.
     pub async fn new_bob() -> Self {
-        let t = Self::new().await;
+        let t = Self::with_name("bob").await;
         let keypair = bob_keypair();
         t.configure_addr(&keypair.addr.to_string()).await;
-        key::store_self_keypair(&t.ctx, &keypair, key::KeyPairUse::Default)
+        key::store_self_keypair(&t, &keypair, key::KeyPairUse::Default)
             .await
             .expect("Failed to save Bob's key");
         t
     }
 
+    /// Sets a name for this [`TestContext`] if one isn't yet set.
+    ///
+    /// This will show up in events logged in the test output.
+    pub fn set_name(&self, name: impl Into<String>) {
+        let mut context_names = CONTEXT_NAMES.write().unwrap();
+        context_names
+            .entry(self.ctx.get_id())
+            .or_insert_with(|| name.into());
+    }
+
+    /// Add a new callback which will receive events.
+    ///
+    /// The test context runs an async task receiving all events from the [`Context`], which
+    /// are logged to stdout.  This allows you to register additional callbacks which will
+    /// receive all events in case your tests need to watch for a specific event.
+    pub async fn add_event_sink<F, R>(&self, sink: F)
+    where
+        // Aka `F: EventSink` but type aliases are not allowed.
+        F: Fn(Event) -> R + Send + Sync + 'static,
+        R: Future<Output = ()> + Send + 'static,
+    {
+        let mut sinks = self.event_sinks.write().await;
+        sinks.push(Box::new(move |evt| Box::pin(sink(evt))));
+    }
+
     /// Configure with alice@example.com.
     ///
     /// The context will be fake-configured as the alice user, with a pre-generated secret
@@ -107,9 +207,12 @@ impl TestContext {
             .set_config(Config::Configured, Some("1"))
             .await
             .unwrap();
+        if let Some(name) = addr.split('@').next() {
+            self.set_name(name);
+        }
     }
 
-    /// Retrieve a sent message from the jobs table.
+    /// Retrieves a sent message from the jobs table.
     ///
     /// This retrieves and removes a message which has been scheduled to send from the jobs
     /// table.  Messages are returned in the order they have been sent.
@@ -126,7 +229,7 @@ impl TestContext {
                     SELECT id, foreign_id, param
                       FROM jobs
                      WHERE action=?
-                  ORDER BY desired_timestamp;
+                  ORDER BY desired_timestamp DESC;
                 "#,
                     paramsv![Action::SendMsgToSmtp],
                     |row| {
@@ -146,7 +249,7 @@ impl TestContext {
                 panic!("no sent message found in jobs table");
             }
         };
-        let id = ChatId::new(foreign_id as u32);
+        let id = MsgId::new(foreign_id as u32);
         let params = Params::from_str(&raw_params).unwrap();
         let blob_path = params
             .get_blob(Param::File, &self.ctx, false)
@@ -159,14 +262,15 @@ impl TestContext {
             .execute("DELETE FROM jobs WHERE id=?;", paramsv![rowid])
             .await
             .expect("failed to remove job");
+        update_msg_state(&self.ctx, id, MessageState::OutDelivered).await;
         SentMessage {
-            id,
             params,
             blob_path,
+            sender_msg_id: id,
         }
     }
 
-    /// Parse a message.
+    /// Parses a message.
     ///
     /// Parsing a message does not run the entire receive pipeline, but is not without
     /// side-effects either.  E.g. if the message includes autocrypt headers the relevant
@@ -185,15 +289,19 @@ impl TestContext {
     pub async fn recv_msg(&self, msg: &SentMessage) {
         let mut idx = self.recv_idx.write().await;
         *idx += 1;
-        dc_receive_imf(&self.ctx, msg.payload().as_bytes(), "INBOX", *idx, false)
+        let received_msg =
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n"
+                .to_owned()
+                + &msg.payload();
+        dc_receive_imf(&self.ctx, received_msg.as_bytes(), "INBOX", *idx, false)
             .await
             .unwrap();
     }
 
-    /// Get the most recent message of a chat.
+    /// Gets the most recent message of a chat.
     ///
     /// Panics on errors or if the most recent message is a marker.
-    pub async fn get_last_msg(&self, chat_id: ChatId) -> Message {
+    pub async fn get_last_msg_in(&self, chat_id: ChatId) -> Message {
         let msgs = chat::get_chat_msgs(&self.ctx, chat_id, 0, None).await;
         let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() {
             msg_id
@@ -202,6 +310,181 @@ impl TestContext {
         };
         Message::load_from_db(&self.ctx, *msg_id).await.unwrap()
     }
+
+    /// Gets the most recent message over all chats.
+    pub async fn get_last_msg(&self) -> Message {
+        let chats = Chatlist::try_load(&self.ctx, 0, None, None).await.unwrap();
+        // 0 is correct in the next line (as opposed to `chats.len() - 1`, which would be the last element):
+        // The chatlist describes what you see when you open DC, a list of chats and in each of them
+        // the first words of the last message. To get the last message overall, we look at the chat at the top of the
+        // list, which has the index 0.
+        let msg_id = chats.get_msg_id(0).unwrap();
+        Message::load_from_db(&self.ctx, msg_id).await.unwrap()
+    }
+
+    /// Creates or returns an existing 1:1 [`Chat`] with another account.
+    ///
+    /// This first creates a contact using the configured details on the other account, then
+    /// creates a 1:1 chat with this contact.
+    pub async fn create_chat(&self, other: &TestContext) -> Chat {
+        let (contact_id, _modified) = Contact::add_or_lookup(
+            self,
+            other
+                .ctx
+                .get_config(Config::Displayname)
+                .await
+                .unwrap_or_default(),
+            other.ctx.get_config(Config::ConfiguredAddr).await.unwrap(),
+            Origin::ManuallyCreated,
+        )
+        .await
+        .unwrap();
+
+        let chat_id = chat::create_by_contact_id(self, contact_id).await.unwrap();
+        Chat::load_from_db(self, chat_id).await.unwrap()
+    }
+
+    /// Creates or returns an existing [`Contact`] and 1:1 [`Chat`] with another email.
+    ///
+    /// This first creates a contact from the `name` and `addr` and then creates a 1:1 chat
+    /// with this contact.
+    pub async fn create_chat_with_contact(&self, name: &str, addr: &str) -> Chat {
+        let contact = Contact::create(self, name, addr)
+            .await
+            .expect("failed to create contact");
+        let chat_id = chat::create_by_contact_id(self, contact).await.unwrap();
+        Chat::load_from_db(self, chat_id).await.unwrap()
+    }
+
+    /// Retrieves the "self" chat.
+    pub async fn get_self_chat(&self) -> Chat {
+        let chat_id = chat::create_by_contact_id(self, DC_CONTACT_ID_SELF)
+            .await
+            .unwrap();
+        Chat::load_from_db(self, chat_id).await.unwrap()
+    }
+
+    /// Sends out the text message.
+    ///
+    /// This is not hooked up to any SMTP-IMAP pipeline, so the other account must call
+    /// [`TestContext::recv_msg`] with the returned [`SentMessage`] if it wants to receive
+    /// the message.
+    pub async fn send_text(&self, chat_id: ChatId, txt: &str) -> SentMessage {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some(txt.to_string()));
+        self.send_msg(chat_id, &mut msg).await
+    }
+
+    /// Sends out the message to the specified chat.
+    ///
+    /// This is not hooked up to any SMTP-IMAP pipeline, so the other account must call
+    /// [`TestContext::recv_msg`] with the returned [`SentMessage`] if it wants to receive
+    /// the message.
+    pub async fn send_msg(&self, chat_id: ChatId, msg: &mut Message) -> SentMessage {
+        chat::prepare_msg(self, chat_id, msg).await.unwrap();
+        chat::send_msg(self, chat_id, msg).await.unwrap();
+        self.pop_sent_msg().await
+    }
+
+    /// Prints out the entire chat to stdout.
+    ///
+    /// You can use this to debug your test by printing the entire chat conversation.
+    // This code is mainly the same as `log_msglist` in `cmdline.rs`, so one day, we could
+    // merge them to a public function in the `deltachat` crate.
+    #[allow(dead_code)]
+    #[allow(clippy::clippy::indexing_slicing)]
+    pub async fn print_chat(&self, chat_id: ChatId) {
+        let msglist = chat::get_chat_msgs(self, chat_id, 0x1, None).await;
+        let msglist: Vec<MsgId> = msglist
+            .into_iter()
+            .map(|x| match x {
+                ChatItem::Message { msg_id } => msg_id,
+                ChatItem::Marker1 => MsgId::new(DC_MSG_ID_MARKER1),
+                ChatItem::DayMarker { .. } => MsgId::new(DC_MSG_ID_DAYMARKER),
+            })
+            .collect();
+
+        let sel_chat = Chat::load_from_db(self, chat_id).await.unwrap();
+        let members = chat::get_chat_contacts(self, sel_chat.id).await;
+        let subtitle = if sel_chat.is_device_talk() {
+            "device-talk".to_string()
+        } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
+            let contact = Contact::get_by_id(self, members[0]).await.unwrap();
+            contact.get_addr().to_string()
+        } else if sel_chat.get_type() == Chattype::Mailinglist && !members.is_empty() {
+            "mailinglist".to_string()
+        } else {
+            format!("{} member(s)", members.len())
+        };
+        println!(
+            "{}#{}: {} [{}]{}{}{} {}",
+            sel_chat.typ,
+            sel_chat.get_id(),
+            sel_chat.get_name(),
+            subtitle,
+            if sel_chat.is_muted() { "🔇" } else { "" },
+            if sel_chat.is_sending_locations() {
+                "📍"
+            } else {
+                ""
+            },
+            match sel_chat.get_profile_image(self).await {
+                Some(icon) => match icon.to_str() {
+                    Some(icon) => format!(" Icon: {}", icon),
+                    _ => " Icon: Err".to_string(),
+                },
+                _ => "".to_string(),
+            },
+            if sel_chat.is_protected() {
+                "🛡️"
+            } else {
+                ""
+            },
+        );
+
+        let mut lines_out = 0;
+        for msg_id in msglist {
+            if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
+                println!(
+                "--------------------------------------------------------------------------------"
+            );
+
+                lines_out += 1
+            } else if !msg_id.is_special() {
+                if lines_out == 0 {
+                    println!(
+                    "--------------------------------------------------------------------------------",
+                );
+                    lines_out += 1
+                }
+                let msg = Message::load_from_db(self, msg_id).await.unwrap();
+                log_msg(self, "", &msg).await;
+            }
+        }
+        if lines_out > 0 {
+            println!(
+                "--------------------------------------------------------------------------------"
+            );
+        }
+    }
+}
+
+impl Deref for TestContext {
+    type Target = Context;
+
+    fn deref(&self) -> &Context {
+        &self.ctx
+    }
+}
+
+impl Drop for TestContext {
+    fn drop(&mut self) {
+        if !thread::panicking() {
+            if let Ok(p) = self.poison_receiver.try_recv() {
+                panic!(p);
+            }
+        }
+    }
 }
 
 /// A raw message as it was scheduled to be sent.
@@ -210,17 +493,12 @@ impl TestContext {
 /// passed through a SMTP-IMAP pipeline.
 #[derive(Debug, Clone)]
 pub struct SentMessage {
-    id: ChatId,
     params: Params,
     blob_path: PathBuf,
+    pub sender_msg_id: MsgId,
 }
 
 impl SentMessage {
-    /// The ChatId the message belonged to.
-    pub fn id(&self) -> ChatId {
-        self.id
-    }
-
     /// A recipient the message was destined for.
     ///
     /// If there are multiple recipients this is just a random one, so is not very useful.
@@ -244,7 +522,7 @@ impl SentMessage {
 /// This saves CPU cycles by avoiding having to generate a key.
 ///
 /// The keypair was created using the crate::key::tests::gen_key test.
-pub(crate) fn alice_keypair() -> key::KeyPair {
+pub fn alice_keypair() -> key::KeyPair {
     let addr = EmailAddress::new("alice@example.com").unwrap();
     let public =
         key::SignedPublicKey::from_base64(include_str!("../test-data/key/alice-public.asc"))
@@ -262,7 +540,7 @@ pub(crate) fn alice_keypair() -> key::KeyPair {
 /// Load a pre-generated keypair for bob@example.net from disk.
 ///
 /// Like [alice_keypair] but a different key and identity.
-pub(crate) fn bob_keypair() -> key::KeyPair {
+pub fn bob_keypair() -> key::KeyPair {
     let addr = EmailAddress::new("bob@example.net").unwrap();
     let public =
         key::SignedPublicKey::from_base64(include_str!("../test-data/key/bob-public.asc")).unwrap();
@@ -274,3 +552,147 @@ pub(crate) fn bob_keypair() -> key::KeyPair {
         secret,
     }
 }
+
+/// Gets a specific message from a chat and asserts that the chat has a specific length.
+///
+/// Panics if the length of the chat is not `asserted_msgs_count` or if the chat item at `index` is not a Message.
+#[allow(clippy::indexing_slicing)]
+pub(crate) async fn get_chat_msg(
+    t: &TestContext,
+    chat_id: ChatId,
+    index: usize,
+    asserted_msgs_count: usize,
+) -> Message {
+    let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None).await;
+    assert_eq!(msgs.len(), asserted_msgs_count);
+    let msg_id = if let ChatItem::Message { msg_id } = msgs[index] {
+        msg_id
+    } else {
+        panic!("Wrong item type");
+    };
+    Message::load_from_db(&t.ctx, msg_id).await.unwrap()
+}
+
+/// Pretty-print an event to stdout
+///
+/// Done during tests this is captured by `cargo test` and associated with the test itself.
+fn receive_event(event: Event) {
+    let green = Color::Green.normal();
+    let yellow = Color::Yellow.normal();
+    let red = Color::Red.normal();
+
+    let msg = match event.typ {
+        EventType::Info(msg) => format!("INFO: {}", msg),
+        EventType::SmtpConnected(msg) => format!("[SMTP_CONNECTED] {}", msg),
+        EventType::ImapConnected(msg) => format!("[IMAP_CONNECTED] {}", msg),
+        EventType::SmtpMessageSent(msg) => format!("[SMTP_MESSAGE_SENT] {}", msg),
+        EventType::Warning(msg) => format!("WARN: {}", yellow.paint(msg)),
+        EventType::Error(msg) => format!("ERROR: {}", red.paint(msg)),
+        EventType::ErrorNetwork(msg) => format!("{}", red.paint(format!("[NETWORK] msg={}", msg))),
+        EventType::ErrorSelfNotInGroup(msg) => {
+            format!("{}", red.paint(format!("[SELF_NOT_IN_GROUP] {}", msg)))
+        }
+        EventType::MsgsChanged { chat_id, msg_id } => format!(
+            "{}",
+            green.paint(format!(
+                "Received MSGS_CHANGED(chat_id={}, msg_id={})",
+                chat_id, msg_id,
+            ))
+        ),
+        EventType::ContactsChanged(_) => format!("{}", green.paint("Received CONTACTS_CHANGED()")),
+        EventType::LocationChanged(contact) => format!(
+            "{}",
+            green.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
+        ),
+        EventType::ConfigureProgress { progress, comment } => {
+            if let Some(comment) = comment {
+                format!(
+                    "{}",
+                    green.paint(format!(
+                        "Received CONFIGURE_PROGRESS({} ‰, {})",
+                        progress, comment
+                    ))
+                )
+            } else {
+                format!(
+                    "{}",
+                    green.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+                )
+            }
+        }
+        EventType::ImexProgress(progress) => format!(
+            "{}",
+            green.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
+        ),
+        EventType::ImexFileWritten(file) => format!(
+            "{}",
+            green.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
+        ),
+        EventType::ChatModified(chat) => format!(
+            "{}",
+            green.paint(format!("Received CHAT_MODIFIED({})", chat))
+        ),
+        _ => format!("Received {:?}", event),
+    };
+    let context_names = CONTEXT_NAMES.read().unwrap();
+    match context_names.get(&event.id) {
+        Some(name) => println!("{} {}", name, msg),
+        None => println!("{} {}", event.id, msg),
+    }
+}
+
+/// Logs and individual message to stdout.
+///
+/// This includes a bunch of the message meta-data as well.
+async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
+    let contact = Contact::get_by_id(context, msg.get_from_id())
+        .await
+        .expect("invalid contact");
+
+    let contact_name = contact.get_name();
+    let contact_id = contact.get_id();
+
+    let statestr = match msg.get_state() {
+        MessageState::OutPending => " o",
+        MessageState::OutDelivered => " √",
+        MessageState::OutMdnRcvd => " √√",
+        MessageState::OutFailed => " !!",
+        _ => "",
+    };
+    let msgtext = msg.get_text();
+    println!(
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}",
+        prefix.as_ref(),
+        msg.get_id(),
+        if msg.get_showpadlock() { "🔒" } else { "" },
+        if msg.has_location() { "📍" } else { "" },
+        &contact_name,
+        contact_id,
+        msgtext.unwrap_or_default(),
+        if msg.get_from_id() == 1u32 {
+            ""
+        } else if msg.get_state() == MessageState::InSeen {
+            "[SEEN]"
+        } else if msg.get_state() == MessageState::InNoticed {
+            "[NOTICED]"
+        } else {
+            "[FRESH]"
+        },
+        if msg.is_info() { "[INFO]" } else { "" },
+        if msg.get_viewtype() == Viewtype::VideochatInvitation {
+            format!(
+                "[VIDEOCHAT-INVITATION: {}, type={}]",
+                msg.get_videochat_url().unwrap_or_default(),
+                msg.get_videochat_type().unwrap_or_default()
+            )
+        } else {
+            "".to_string()
+        },
+        if msg.is_forwarded() {
+            "[FORWARDED]"
+        } else {
+            ""
+        },
+        statestr,
+    );
+}

src/token.rs

@@ -4,14 +4,16 @@
 //!
 //! Tokens are used in countermitm verification protocols.
 
-use deltachat_derive::*;
+use deltachat_derive::{FromSql, ToSql};
 
 use crate::chat::ChatId;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_create_id, time};
 
 /// Token namespace
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 #[repr(i32)]
 pub enum Namespace {
     Unknown = 0,
@@ -26,38 +28,71 @@ impl Default for Namespace {
 }
 
 /// Creates a new token and saves it into the database.
+///
 /// Returns created token.
-pub async fn save(context: &Context, namespace: Namespace, foreign_id: ChatId) -> String {
-    // foreign_id may be 0
+pub async fn save(context: &Context, namespace: Namespace, chat: Option<ChatId>) -> String {
     let token = dc_create_id();
-    context
-        .sql
-        .execute(
-            "INSERT INTO tokens (namespc, foreign_id, token, timestamp) VALUES (?, ?, ?, ?);",
-            paramsv![namespace, foreign_id, token, time()],
-        )
-        .await
-        .ok();
+    match chat {
+        Some(chat_id) => context
+            .sql
+            .execute(
+                "INSERT INTO tokens (namespc, foreign_id, token, timestamp) VALUES (?, ?, ?, ?);",
+                paramsv![namespace, chat_id, token, time()],
+            )
+            .await
+            .ok(),
+        None => context
+            .sql
+            .execute(
+                "INSERT INTO tokens (namespc, token, timestamp) VALUES (?, ?, ?);",
+                paramsv![namespace, token, time()],
+            )
+            .await
+            .ok(),
+    };
     token
 }
 
-pub async fn lookup(context: &Context, namespace: Namespace, foreign_id: ChatId) -> Option<String> {
-    context
-        .sql
-        .query_get_value::<String>(
-            context,
-            "SELECT token FROM tokens WHERE namespc=? AND foreign_id=?;",
-            paramsv![namespace, foreign_id],
-        )
-        .await
+pub async fn lookup(
+    context: &Context,
+    namespace: Namespace,
+    chat: Option<ChatId>,
+) -> Option<String> {
+    match chat {
+        Some(chat_id) => {
+            context
+                .sql
+                .query_get_value::<String>(
+                    context,
+                    "SELECT token FROM tokens WHERE namespc=? AND foreign_id=?;",
+                    paramsv![namespace, chat_id],
+                )
+                .await
+        }
+        // foreign_id is declared as `INTEGER DEFAULT 0` in the schema.
+        None => {
+            context
+                .sql
+                .query_get_value::<String>(
+                    context,
+                    "SELECT token FROM tokens WHERE namespc=? AND foreign_id=0;",
+                    paramsv![namespace],
+                )
+                .await
+        }
+    }
 }
 
-pub async fn lookup_or_new(context: &Context, namespace: Namespace, foreign_id: ChatId) -> String {
-    if let Some(token) = lookup(context, namespace, foreign_id).await {
+pub async fn lookup_or_new(
+    context: &Context,
+    namespace: Namespace,
+    chat: Option<ChatId>,
+) -> String {
+    if let Some(token) = lookup(context, namespace, chat).await {
         return token;
     }
 
-    save(context, namespace, foreign_id).await
+    save(context, namespace, chat).await
 }
 
 pub async fn exists(context: &Context, namespace: Namespace, token: &str) -> bool {

standards.md

@@ -6,6 +6,8 @@ Tasks                            | Standards
 ---------------------------------|---------------------------------------------
 Transport                        | IMAP v4 ([RFC 3501](https://tools.ietf.org/html/rfc3501)), SMTP ([RFC 5321](https://tools.ietf.org/html/rfc5321)) and Internet Message Format (IMF, [RFC 5322](https://tools.ietf.org/html/rfc5322))
 Embedded media                   | MIME Document Series ([RFC 2045](https://tools.ietf.org/html/rfc2045), [RFC 2046](https://tools.ietf.org/html/rfc2046)), Content-Disposition Header ([RFC 2183](https://tools.ietf.org/html/rfc2183)), Multipart/Related ([RFC 2387](https://tools.ietf.org/html/rfc2387))
+Text and Quote encoding          | Fixed, Flowed ([RFC 3676](https://tools.ietf.org/html/rfc3676))
+Filename encoding                | Encoded Words ([RFC 2047](https://tools.ietf.org/html/rfc2047)), Encoded Word Extensions ([RFC 2231](https://tools.ietf.org/html/rfc2231))
 Identify server folders          | IMAP LIST Extension ([RFC 6154](https://tools.ietf.org/html/rfc6154))
 Push                             | IMAP IDLE ([RFC 2177](https://tools.ietf.org/html/rfc2177))
 Authorization                    | OAuth2 ([RFC 6749](https://tools.ietf.org/html/rfc6749))
@@ -13,6 +15,7 @@ End-to-end encryption            | [Autocrypt Level 1](https://autocrypt.org/lev
 Configuration assistance         | [Autoconfigure](https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover](https://technet.microsoft.com/library/bb124251(v=exchg.150).aspx)
 Messenger functions              | [Chat-over-Email](https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#chat-over-email-specification)
 Detect mailing list              | List-Id ([RFC 2919](https://tools.ietf.org/html/rfc2919)) and Precedence ([RFC 3834](https://tools.ietf.org/html/rfc3834))
+User and chat colors             | [XEP-0392](https://xmpp.org/extensions/xep-0392.html): Consistent Color Generation
 Send and receive system messages | Multipart/Report Media Type ([RFC 6522](https://tools.ietf.org/html/rfc6522))
 Return receipts                  | Message Disposition Notification (MDN, [RFC 8098](https://tools.ietf.org/html/rfc8098), [RFC 3503](https://tools.ietf.org/html/rfc3503)) using the Chat-Disposition-Notification-To header
 Locations                        | KML ([Open Geospatial Consortium](http://www.opengeospatial.org/standards/kml/), [Google Dev](https://developers.google.com/kml/))

test-data/message/AutocryptSetupMessage.eml

@@ -0,0 +1,77 @@
+Return-Path: <alice@example.com>
+Delivered-To: alice@example.com
+Received: from hq5.merlinux.eu
+	by hq5.merlinux.eu with LMTP
+	id gNKpOrrTvF+tVAAAPzvFDg
+	(envelope-from <alice@example.com>)
+	for <alice@example.com>; Tue, 24 Nov 2020 10:34:50 +0100
+Subject: Autocrypt Setup Message
+DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=testrun.org;
+	s=testrun; t=1606210490;
+	bh=MXqLqHFK1xC48pxx2TS1GUdxKSi4tdejRRSV4EAN5Tc=;
+	h=Subject:Date:To:From:From;
+	b=DRajftyu+Ycfhaxy0jXAIKCihQRMI0rxbo9+EBu6y5jhtZx13emW3odgZnvyhU6uD
+	 IKfMXaqlmc/2HNV1/mloJVIRsIp5ORncSPX9tLykNApJVyPHg3NKdMo3Ib4NGIJ1Qo
+	 binmLtL5qqL3bYCL68WUgieH1rcgCaf9cwck9GvwZ79pexGuWz4ItgtNWqYfapG8Zc
+	 9eD5maiTMNkV7UwgtOzhbBd39uKgKCoGdLAq63hoJF6dhdBBRVRyRMusAooGUZMgwm
+	 QVuTZ76z9G8w3rDgZuHmoiICWsLsar4CDl4zAgicE6bHwtw3a7YuMiHoCtceq0RjQP
+	 BHVaXT7B75BoA==
+MIME-Version: 1.0
+Date: Tue, 24 Nov 2020 09:34:48 +0000
+Chat-Version: 1.0
+Autocrypt-Setup-Message: v1
+Message-ID: <Mr._G2UTiTkgfk.HWf5RnFC2xy@testrun.org>
+To: <alice@example.com>
+From:  <alice@example.com>
+Content-Type: multipart/mixed; boundary="dKhu3bbmBniQsT8W8w58YRCCiBK2YY"
+
+
+--dKhu3bbmBniQsT8W8w58YRCCiBK2YY
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+This is the Autocrypt Setup Message used to transfer your end-to-end setup 
+between clients.
+
+To decrypt and use your setup, open the message in an Autocrypt-compliant 
+client and enter the setup code presented on the generating device.
+
+-- 
+Sent with my Delta Chat Messenger: https://delta.chat
+
+--dKhu3bbmBniQsT8W8w58YRCCiBK2YY
+Content-Type: application/autocrypt-setup
+Content-Disposition: attachment; filename="autocrypt-setup-message.html"
+Content-Transfer-Encoding: base64
+
+PCFET0NUWVBFIGh0bWw+DQo8aHRtbD4NCiAgPGhlYWQ+DQogICAgPHRpdGxlPkF1dG9jcnlwdCBTZX
+R1cCBNZXNzYWdlPC90aXRsZT4NCiAgPC9oZWFkPg0KICA8Ym9keT4NCiAgICA8aDE+QXV0b2NyeXB0
+IFNldHVwIE1lc3NhZ2U8L2gxPg0KICAgIDxwPlRoaXMgaXMgdGhlIEF1dG9jcnlwdCBTZXR1cCBNZX
+NzYWdlIHVzZWQgdG8gdHJhbnNmZXIgeW91ciBlbmQtdG8tZW5kIHNldHVwIGJldHdlZW4gY2xpZW50
+cy48YnI+PGJyPlRvIGRlY3J5cHQgYW5kIHVzZSB5b3VyIHNldHVwLCBvcGVuIHRoZSBtZXNzYWdlIG
+luIGFuIEF1dG9jcnlwdC1jb21wbGlhbnQgY2xpZW50IGFuZCBlbnRlciB0aGUgc2V0dXAgY29kZSBw
+cmVzZW50ZWQgb24gdGhlIGdlbmVyYXRpbmcgZGV2aWNlLjwvcD4NCiAgICA8cHJlPg0KLS0tLS1CRU
+dJTiBQR1AgTUVTU0FHRS0tLS0tDQpQYXNzcGhyYXNlLUZvcm1hdDogbnVtZXJpYzl4NA0KUGFzc3Bo
+cmFzZS1CZWdpbjogNjIKCnd4NEVCd01JWEUzNCs4RGhtSC9nRDNNY21JTjhCSUorbmhpbDMrOFE3bF
+hTd21JQnhDSnhBU2VhQUJlTGdFOTIKTi9WaER5MHlrUHFBQkp0S0xvSG9pQmxTQWZJajFRemdPeVlV
+Wjl3czRtSng5OVREUE1lSnNmNHJaemJhUHZFSApQcEIrTTgyTjVhUitvV0dTcWRtUUZNQUplNWNtWX
+hwM3p4eE5aTEc2cXVnRzUzOFNxNUV1TzBDSGduaXlFeEwyCkJya2hFOWVFVE1oSkNRQ3dCZDc5alhN
+U2Mwcm5xYjFHbS9Kd21jbXFqVFNHMlBLTWNNcFlaV1QwQkNMaDE2TmwKTkNNbmRQWGt2cTlHd1crNX
+pEMHc4cElyOERNRHk1SWVBcG83amNZR1U5UWNUR1lMWmltR2QxK1RYYlgvdGxqRQplMnNZd0hZeU5D
+R1N5bHVsYi9XYnNkS2FrYXVodHJ6cUVHOXNYSkJkMnF5ZjNJajRULzdCd1pJVk42OXF1T21sCnlmMm
+9PTmtYY1pCcFBJUE9ZQzdhMnJ5aFh0Q0NhbWhIVEw0czdzclg2NzJXMTVXS3VqNGVBK25URlNocFBC
+cXoKb05EY3QzbG95V0hNSUluSzRha1VJeTFZak42TDFSbGwwRVhudlVQS0lkT0FpY0swbFBPaDVUZU
+t6ZFMvTklyMQpQc2x6c2RyWTRZd0diMWNTdk95OXJQRFpaS3Y4d0dzbFczcFpFOCs3NnJWckllbkNY
+dTdvOUZ6OFhQcVlxTGRrCkpCZGRHUGZnY0l6Um5nZjZqb0lmT0RsU2NiajR0VlgyK3htVVN5RlVhSD
+RQcDFzZDgwVjhDN2xhREJ2WTc0TlAKQW9ydEVhL2xGbzQzcHNOdlhrc0JUUEVRNHFoTVZneVdQWW9V
+ZGV2aUFZOGVDMmJjT0dMSFVURk5zaHZCaDFGRgozVGpIZEVRVk5zZVlqaWtZRWtkUU9Mb3B5VWdqbj
+lSTUJnV2xIZTNKL1VRcmtFUkNYWi9BSVRXeGdYdmE0NHBPCkkzUHllcnF2T1lpVlJLam9JSTVIZGU4
+UFdkTnZwb2J5ZCsrTHlqN3Jxd0kyNFRwbVRwYWtIZ1RJNEJvYWtLSUcKWm1JWDhsQm4xMnQ5dlcvcD
+lrbDluYWluS3Z1VFBoTk4xZmkrTE1YYTRDK1hqRXVPUnQwMFMzc01MdVo3RnBPaQprcXdGWk12RUtw
+bHA3dmRLSnJNbmVzZ2dKLzBLeWc1RTJ4dVd2VFdkZUFBOE1saEJqSGlsK3JVK0dSZzdaTmxsCkxUej
+RKeGpWUVl5TGpFbkhqdGU4bUVnZlNIZEE3ZDErVnV1RTZSZjlYMzRPeXhkL3NocllJSU8xY3FVdnQw
+V3MKNGIwQURIN0lkbjkveTdDRjVrbWFONkMyQURBRkhFRzNIRWFZaDVNNmIwVzVJSW55WkhUQ0QxdC
+tmUFdQYndxUQo0TzFRMEROZ01QT1FCRVJ0ODNXR3g5YW5GQU9YCj05dTUrCi0tLS0tRU5EIFBHUCBN
+RVNTQUdFLS0tLS0KDQo8L3ByZT4NCiAgPC9ib2R5Pg0KPC9odG1sPg0K
+
+--dKhu3bbmBniQsT8W8w58YRCCiBK2YY--
+

test-data/message/apple_cid_jpg.eml

@@ -0,0 +1,53 @@
+From: =?utf-8?Q?Bj=C3=B6rn_Petersen?= <somewhere-apple@me.com>
+Content-Type: multipart/alternative;
+	boundary="Apple-Mail=_19251BCB-E12B-423A-9553-5A68560C2AFD"
+Mime-Version: 1.0 (Mac OS X Mail 13.4 \(3608.120.23.2.4\))
+Subject: a jpeg
+Message-Id: <BC47DA72-6C78-443A-8EBF-2CD199ABAD09@me.com>
+Date: Sat, 9 Jan 2021 00:36:11 +0100
+To: somewhere-nonapple@testrun.org
+X-Mailer: Apple Mail (2.3608.120.23.2.4)
+
+
+--Apple-Mail=_19251BCB-E12B-423A-9553-5A68560C2AFD
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain;
+	charset=us-ascii
+
+
+a jpeg
+--Apple-Mail=_19251BCB-E12B-423A-9553-5A68560C2AFD
+Content-Type: multipart/related;
+	type="text/html";
+	boundary="Apple-Mail=_4C3710FD-D75D-47FB-8D41-983220390856"
+
+
+--Apple-Mail=_4C3710FD-D75D-47FB-8D41-983220390856
+Content-Transfer-Encoding: 7bit
+Content-Type: text/html;
+	charset=us-ascii
+
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=us-ascii"><base></head><body style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class=""><base class=""><div class="Apple-Mail-URLShareUserContentTopClass"><br class=""></div><div class="Apple-Mail-URLShareWrapperClass"><blockquote type="cite" style="border-left-style: none; color: inherit; padding: inherit; margin: inherit;" class="">a jpeg
+	<img apple-inline="yes" id="118F6150-5EF5-4DE8-917F-1851EC94FB7C" src="cid:8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box" class=""></blockquote></div></body></html>
+--Apple-Mail=_4C3710FD-D75D-47FB-8D41-983220390856
+Content-Transfer-Encoding: base64
+Content-Disposition: inline;
+	filename=small.jpg
+Content-Type: image/jpeg;
+	x-unix-mode=0666;
+	name="small.jpg"
+Content-Id: <8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box>
+
+/9j/4AAQSkZJRgABAQEBLAEsAAD/2wBDABELDA8MChEPDg8TEhEUGSobGRcXGTMkJh4qPDU/Pjs1
+OjlDS2BRQ0daSDk6U3FUWmNma2xrQFB2fnRofWBpa2f/2wBDARITExkWGTEbGzFnRTpFZ2dnZ2dn
+Z2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2f/wgARCAAIAAgDAREA
+AhEBAxEB/8QAFAABAAAAAAAAAAAAAAAAAAAABP/EABUBAQEAAAAAAAAAAAAAAAAAAAMF/9oADAMB
+AAIQAxAAAAF8s//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAQUCf//EABQRAQAAAAAAAAAA
+AAAAAAAAAAD/2gAIAQMBAT8Bf//EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQIBAT8Bf//EABQQ
+AQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEABj8Cf//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEA
+AT8hf//aAAwDAQACAAMAAAAQ3//EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQMBAT8Qf//EABQR
+AQAAAAAAAAAAAAAAAAAAAAD/2gAIAQIBAT8Qf//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEA
+AT8Qf//Z
+--Apple-Mail=_4C3710FD-D75D-47FB-8D41-983220390856--
+
+--Apple-Mail=_19251BCB-E12B-423A-9553-5A68560C2AFD--

test-data/message/attach_filename_apostrophed.eml

@@ -0,0 +1,24 @@
+Subject: Test apostrophed filenames
+Message-ID: 12345@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+X-Mailer: Kopano 8.7.16
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+apostrophed filenames as of
+https://tools.ietf.org/html/rfc2231
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment;
+ filename*=utf-8''Ma%C3%9Fnahmen%20Okt.%202021.html
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--

test-data/message/attach_filename_apostrophed_cont.eml

@@ -0,0 +1,34 @@
+Subject: Test apostrophed filenames
+Message-ID: 12345@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+X-Mailer: Kopano 8.7.16
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+apostrophed filenames as of
+https://tools.ietf.org/html/rfc2231,
+span over several header lines.
+
+note, that, in contrast to encoded-words,
+the character-set is not repeated.
+
+as a side-effect,
+this tests unquoted header attributes in filename*1*
+and lower-case-urlencoded utf-8
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment;
+ filename*0*="utf-8''Ma%C3%9Fna";
+ filename*1*=hm;
+ filename*2*="en%20M%c3%a4rz%202022.html";
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--

test-data/message/attach_filename_apostrophed_cp1252.eml

@@ -0,0 +1,23 @@
+Subject: Test apostrophed filenames
+Message-ID: 12345@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+X-Mailer: Kopano 8.7.16
+To: recp@testrun.org
+From: sender@testrun.org
+Content-Type: multipart/mixed; boundary="==BREAK=="
+
+
+--==BREAK==
+Content-Type: text/plain; charset=utf-8
+
+testing cp1252 aka ANSI aka Windows-1252
+
+--==BREAK==
+Content-Type: text/html
+Content-Disposition: attachment;
+ filename*=Cp1252''Auftragsbest%E4tigung.pdf;
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
+
+--==BREAK==--