add missing members receiving non-dc messages

* Save subject for sent-out messages

* Test that subject is saved (outgoing)

* Correctly set subject when forwarding messages, add test for this

.circleci/config.yml

@@ -7,9 +7,6 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
-  python:
-    docker:
-      - image: 3.7.7-stretch
             
 
 restore-workspace: &restore-workspace
@@ -156,8 +153,6 @@ jobs:
       - checkout
       - attach_workspace:
           at: workspace
-      - run: pyenv versions
-      - run: pyenv global 3.5.2
       - run: ls -laR workspace
       - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 
@@ -189,13 +184,11 @@ workflows:
               only: /.*/
 
       - remote_python_packaging:
-          requires: 
-             - remote_tests_python
           filters:
             branches:
               only: master
-            tags:
-              only: /.*/
+            #tags:
+            #  only: /.*/
 
       - upload_docs_wheels:
           requires:

.gitattributes

@@ -12,3 +12,6 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
+*.py diff=python
+*.rs diff=rust
+*.md diff=markdown

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.45.0
+          toolchain: 1.50.0
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: 1.45.0
+            toolchain: 1.50.0
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
@@ -40,6 +40,28 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --workspace --tests --examples
 
+  docs:
+    name: Rust doc comments
+    runs-on: ubuntu-latest
+    env:
+      RUSTDOCFLAGS: -Dwarnings
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - name: Install rust stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          components: rust-docs
+          override: true
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v1
+      - name: Rustdoc
+        uses: actions-rs/cargo@v1
+        with:
+          command: doc
+          args: --document-private-items --no-deps
 
   build_and_test:
     name: Build and test
@@ -47,8 +69,10 @@ jobs:
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [1.45.0]
+        # macOS disabled due to random failures related to caching
+        #os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest]
+        rust: [1.50.0]
         experimental: [false]
 #        include:
 #          - os: ubuntu-latest

CHANGELOG.md

@@ -1,5 +1,249 @@
 # Changelog 
 
+## UNRELEASED
+
+- breaking change: You have to call dc_stop_io()/dc_start_io() before/after EXPORT_BACKUP:
+  fix race condition and db corruption when a message was received during backup #2253
+
+- save subject for messages:
+  new api `dc_msg_get_subject()`,
+  when quoting, use the subject of the quoted message as the new subject, instead of the
+  last subject in the chat
+
+- new apis to get full or html message,
+  `dc_msg_has_html()` and `dc_get_msg_html()` #2125 #2151
+
+- new chat type and apis for the new mailing list support,
+  `DC_CHAT_TYPE_MAILINGLIST`, `dc_msg_get_real_chat_id()`,
+  `dc_msg_get_override_sender_name()` #1964 #2181 #2185 #2195 #2211 #2210 #2240
+  #2243
+
+- new api `dc_decide_on_contact_request()`,
+  deprecated `dc_create_chat_by_msg_id()` and `dc_marknoticed_contact()` #1964
+
+- new flag `DC_GCM_INFO_ONLY` for api `dc_get_chat_msgs()` #2132
+
+- new api `dc_get_chat_encrinfo()` #2186
+
+- new api `dc_contact_get_status()`, returning the recent footer #2218
+
+- improve contact name update rules,
+  add api `dc_contact_get_auth_name()` #2206 #2212 #2225
+
+- new api for bots: `dc_msg_set_html()` #2153
+
+- new api for bots: `dc_msg_set_override_sender_name()` #2231
+
+- api removed: `dc_is_io_running()` #2139
+
+- api removed: `dc_contact_get_first_name()` #2165 #2171
+
+- implement Consistent Color Generation (XEP-0392),
+  that results in contact colors be be changed #2228 #2229 #2239
+
+- fetch recent existing messages
+  and create corresponding chats after configure #2106
+
+- improve e-mail compatibility
+  by scanning all folders from time to time #2067 #2152 #2158 #2184 #2215 #2224
+
+- better support videochat-services not supporting random rooms #2191
+
+- export backups as .tar files #2023
+
+- scale avatars based on media_quality, fix avatar rotation #2063
+
+- compare ephemeral timer to parent message to deal with reordering better #2100
+
+- better ephemeral system messages #2183
+
+- read quotes out of html messages #2104
+
+- prepend subject to messages with attachments, if needed #2111
+
+- run housekeeping at least once a day #2114
+
+- resolve MX domain only once per OAuth2 provider #2122
+
+- configure provider based on MX record #2123 #2134
+
+- make transient bad destination address error permanent
+  after n tries #2126 #2202
+
+- enable strict TLS for known providers by default #2121
+
+- improve and harden secure join #2154 #2161
+
+- update `dc_get_info()` to return more information #2156
+
+- prefer In-Reply-To/References
+  over group-id stored in Message-ID #2164 #2172 #2173
+
+- apply gossiped encryption preference to new peerstates #2174
+
+- fix: do not return quoted messages from the trash chat #2221
+
+- fix: allow emojis for location markers #2177
+
+- fix encoding of Chat-Group-Name-Changed messages that could even lead to
+  messages not being delivered #2141
+
+- fix error when no temporary directory is available #1929
+
+- fix marking read receipts as seen #2117
+
+- fix read-notification for mixed-case addresses #2103
+
+- fix decoding of attachment filenames #2080 #2094 #2102
+
+- fix downloading ranges of message #2061
+
+- fix parsing quoted encoded words in From: header #2193 #2204
+
+- fix ci #2217 #2226
+
+- try harder on backup opening #2148
+
+- switch to rust 1.50, update toolchains, deps #2150 #2155 #2165 #2107 #2262 #2271
+
+- improve python bindings #2113 #2115 #2133 #2214
+
+- improve documentation #2143 #2160 #2175 #2146
+
+- refactorings #2110 #2136 #2135 #2168 #2178 #2189 #2190 #2198 #2197 #2201 #2196
+  #2200 #2230
+
+
+## 1.50.0
+
+- do not fetch emails in between inbox_watch disabled and enabled again #2087
+
+- fix: do not fetch from INBOX if inbox_watch is disabled #2085
+
+- fix: do not use STARTTLS when PLAIN connection is requested
+  and do not allow downgrade if STARTTLS is not available #2071
+
+
+## 1.49.0
+
+- add timestamps to image and video filenames #2068
+
+- forbid quoting messages from another context #2069
+
+- fix: preserve quotes in messages with attachments #2070
+
+
+## 1.48.0
+
+- `fetch_existing` renamed to `fetch_existing_msgs` and disabled by default
+  #2035 #2042
+
+- skip fetch existing messages/contacts if config-option `bot` set #2017
+
+- always log why a message is sorted to trash #2045
+
+- display a quote if top posting is detected #2047
+
+- add ephemeral task cancellation to `dc_stop_io()`;
+  before, there was no way to quickly terminate pending ephemeral tasks #2051
+
+- when saved-messages chat is deleted,
+  a device-message about recreation is added #2050
+
+- use `max_smtp_rcpt_to` from provider-db,
+  sending messages to many recipients in configurable chunks #2056
+
+- fix handling of empty autoconfigure files #2027
+
+- fix adding saved messages to wrong chats on multi-device #2034 #2039
+
+- fix hang on android4.4 and other systems
+  by adding a workaround to executer-blocking-handling bug #2040
+
+- fix secret key export/import roundtrip #2048
+
+- fix mistakenly unarchived chats #2057
+
+- fix outdated-reminder test that fails only 7 days a year,
+  including halloween :) #2059
+
+- improve python bindings #2021 #2036 #2038
+
+- update provider-database #2037
+
+
+## 1.47.0
+
+- breaking change: `dc_update_device_chats()` removed;
+  this is now done automatically during configure
+  unless the new config-option `bot` is set #1957
+
+- breaking change: split `DC_EVENT_MSGS_NOTICED` off `DC_EVENT_MSGS_CHANGED`
+  and remove `dc_marknoticed_all_chats()` #1942 #1981
+
+- breaking change: remove unused starring options #1965
+
+- breaking change: `DC_CHAT_TYPE_VERIFIED_GROUP` replaced by
+  `dc_chat_is_protected()`; also single-chats may be protected now, this may
+  happen over the wire even if the UI do not offer an option for that #1968
+
+- breaking change: split quotes off message text,
+  UIs should use at least `dc_msg_get_quoted_text()` to show quotes now #1975
+
+- new api for quote handling: `dc_msg_set_quote()`, `dc_msg_get_quoted_text()`,
+  `dc_msg_get_quoted_msg()` #1975 #1984 #1985 #1987 #1989 #2004
+
+- require quorum to enable encryption #1946
+
+- speed up and clean up account creation #1912 #1927 #1960 #1961
+
+- configure now collects recent contacts and fetches last messages
+  unless disabled by `fetch_existing` config-option #1913 #2003
+  EDIT: `fetch_existing` renamed to `fetch_existing_msgs` in 1.48.0 #2042
+
+- emit `DC_EVENT_CHAT_MODIFIED` on contact rename
+  and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
+
+- add `dc_set_chat_protection()`; the `protect` parameter in
+  `dc_create_group_chat()` will be removed in an upcoming release;
+  up to then, UIs using the "verified group" paradigm
+  should not use `dc_set_chat_protection()` #1968 #2014 #2001 #2012 #2007
+
+- remove unneeded `DC_STR_COUNT` #1991
+
+- mark all failed messages as failed when receiving an NDN #1993
+
+- check some easy cases for bad system clock and outdated app #1901
+
+- fix import temporary directory usage #1929
+
+- fix forcing encryption for reset peers #1998
+
+- fix: do not allow to save drafts in non-writeable chats #1997
+
+- fix: do not show HTML if there is no content and there is an attachment #1988
+
+- fix recovering offline/lost connections, fixes background receive bug #1983
+
+- fix ordering of accounts returned by `dc_accounts_get_all()` #1909
+
+- fix whitespace for summaries #1938
+
+- fix: improve sentbox name guessing #1941
+
+- fix: avoid manual poll impl for accounts events #1944
+
+- fix encoding newlines in param as a preparation for storing quotes #1945
+
+- fix: internal and ffi error handling #1967 #1966 #1959 #1911 #1916 #1917 #1915
+
+- fix ci #1928 #1931 #1932 #1933 #1934 #1943
+
+- update provider-database #1940 #2005 #2006
+
+- update dependencies #1919 #1908 #1950 #1963 #1996 #2010 #2013
+
+
 ## 1.46.0
 
 - breaking change: `dc_configure()` report errors in
@@ -745,4 +989,3 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
-

CMakeLists.txt

@@ -0,0 +1,28 @@
+cmake_minimum_required(VERSION 3.18)
+project (deltachat)
+
+find_program(CARGO cargo)
+
+add_custom_command(
+	OUTPUT
+	"target/release/libdeltachat.a"
+	"target/release/libdeltachat.so"
+	"target/release/pkgconfig/deltachat.pc"
+	COMMAND PREFIX=${CMAKE_INSTALL_PREFIX} ${CARGO} build --package deltachat_ffi --release
+	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
+add_custom_target(
+	lib_deltachat
+	ALL
+	DEPENDS
+	"target/release/libdeltachat.a"
+	"target/release/libdeltachat.so"
+	"target/release/pkgconfig/deltachat.pc"
+)
+
+include(GNUInstallDirs)
+install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,10 +1,13 @@
 [package]
 name = "deltachat"
-version = "1.46.0"
+version = "1.51.0-alpha.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
+[profile.dev]
+debug = 0
+
 [profile.release]
 lto = true
 
@@ -12,8 +15,9 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.6.0", default-features = false } 
+pgp = { version = "0.7.0", default-features = false } 
 hex = "0.4.0"
+sha-1 = "0.9.3"
 sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
@@ -25,7 +29,7 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.4.0"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.1", features = ["unstable"] }
+async-std = { version = "~1.8.0", features = ["unstable"] }
 base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -34,22 +38,22 @@ serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
 kamadak-exif = "0.5"
-lazy_static = "1.4.0"
+once_cell = "1.4.1"
 regex = "1.1.6"
-rusqlite = { version = "0.23", features = ["bundled"] }
-r2d2_sqlite = "0.16.0"
+rusqlite = { version = "0.24", features = ["bundled"] }
+r2d2_sqlite = "0.17.0"
 r2d2 = "0.8.5"
-strum = "0.18.0"
-strum_macros = "0.18.0"
+strum = "0.19.0"
+strum_macros = "0.19.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.9.0"
 quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-sanitize-filename = "0.2.1"
+sanitize-filename = "0.3.0"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.1"
+mailparse = "0.13.0"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
@@ -61,6 +65,7 @@ url = "2.1.1"
 async-std-resolver = "0.19.5"
 async-tar = "0.3.0"
 uuid = { version = "0.8", features = ["serde", "v4"] }
+rust-hsluv = "0.1.4"
 
 pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
@@ -75,8 +80,10 @@ tempfile = "3.0"
 pretty_assertions = "0.6.1"
 pretty_env_logger = "0.4.0"
 proptest = "0.10"
-async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
-smol = "0.1.10"
+async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
+futures-lite = "1.7.0"
+criterion = "0.3"
+ansi_term = "0.12.0"
 
 [workspace]
 members = [
@@ -95,6 +102,10 @@ path = "examples/repl/main.rs"
 required-features = ["repl"]
 
 
+[[bench]]
+name = "create_account"
+harness = false
+
 [features]
 default = []
 internals = []

README.md

@@ -123,7 +123,7 @@ Language bindings are available for:
 - [C](https://c.delta.chat)
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
-- [Go](https://github.com/hugot/go-deltachat/)
+- [Go](https://github.com/deltachat/go-deltachat/)
 - [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 

benches/create_account.rs

@@ -0,0 +1,26 @@
+use async_std::path::PathBuf;
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::accounts::Accounts;
+use tempfile::tempdir;
+
+async fn create_accounts(n: u32) {
+    let dir = tempdir().unwrap();
+    let p: PathBuf = dir.path().join("accounts").into();
+
+    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+    for expected_id in 2..n {
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, expected_id);
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("create 1 account", |b| {
+        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

ci_scripts/ci_upload.sh

@@ -10,9 +10,13 @@ set -xe
 PYDOCDIR=${1:?directory with python docs}
 WHEELHOUSEDIR=${2:?directory with pre-built wheels}
 DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+SSHTARGET=ci@b1.delta.chat
 
+    
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
+
 
 # python docs to py.delta.chat
 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
@@ -35,23 +39,36 @@ echo upload wheels
 echo -----------------------
 
 # Bundle external shared libraries into the wheels
-pushd $WHEELHOUSEDIR
 
-pip3 install -U pip setuptools
-pip3 install devpi-client
-devpi use https://m.devpi.net
-devpi login dc --password $DEVPI_LOGIN
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ci_scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  $WHEELHOUSEDIR \
+  $SSHTARGET:$BUILDDIR 
 
-N_BRANCH=${BRANCH//[\/]}
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
 
-devpi use dc/$N_BRANCH || {
-    devpi index -c $N_BRANCH 
-    devpi use dc/$N_BRANCH
-}
-devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*
+    # we rely on the "venv" virtualenv on the remote account to exist 
+    source venv/bin/activate
+    cd $BUILDDIR
 
-popd
+    devpi use https://m.devpi.net
+    devpi login dc --password $DEVPI_LOGIN
 
-# remove devpi non-master dc indices if thy are too old
-python ci_scripts/cleanup_devpi_indices.py
+    N_BRANCH=${BRANCH//[\/]}
+
+    devpi use dc/\$N_BRANCH || {
+        devpi index -c \$N_BRANCH 
+        devpi use dc/\$N_BRANCH
+    }
+    devpi index \$N_BRANCH bases=/root/pypi
+    devpi upload wheelhouse/deltachat*
+
+    # remove devpi non-master dc indices if thy are too old
+    # this script was copied above 
+    python cleanup_devpi_indices.py
+_HERE

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.50.0-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share
+rm -rf /root/.rustup/toolchains/1.50.0-x86_64-unknown-linux-gnu/share

ci_scripts/remote_python_packaging.sh

@@ -1,11 +1,9 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"

ci_scripts/remote_tests_python.sh

@@ -1,11 +1,9 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
@@ -30,9 +28,6 @@ ssh $SSHTARGET <<_HERE
 
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL

ci_scripts/remote_tests_rust.sh

@@ -1,11 +1,9 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 set -e
@@ -24,9 +22,6 @@ ssh $SSHTARGET <<_HERE
     shopt -s huponexit
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=x86_64-unknown-linux-gnu
     export RUSTC_WRAPPER=sccache
 

ci_scripts/run-python-test.sh

@@ -4,6 +4,7 @@
 # and tox/pytest. 
 
 set -e -x
+shopt -s huponexit
 
 # for core-building and python install step
 export DCC_RS_TARGET=debug

ci_scripts/run-rust-test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -ex
+shopt -s huponexit
 
 #export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.46.0"
+version = "1.51.0-alpha.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/Doxyfile

@@ -236,12 +236,6 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
-# This tag can be used to specify a number of word-keyword mappings (TCL only).
-# A mapping has the form "name=value". For example adding "class=itcl::class"
-# will allow you to use the command class in the itcl::class meaning.
-
-TCL_SUBST              =
-
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all

deltachat-ffi/DoxygenLayout.xml

@@ -0,0 +1,38 @@
+<doxygenlayout version="1.0">
+  <!-- Generated by doxygen 1.8.20 -->
+  <!-- Navigation index tabs for HTML output -->
+  <navindex>
+    <tab type="mainpage" visible="yes" title=""/>
+    <tab type="classes" visible="yes" title="">
+      <tab type="classlist" visible="no" title="" intro=""/>
+      <tab type="classindex" visible="no" title=""/>
+      <tab type="hierarchy" visible="no" title="" intro=""/>
+      <tab type="classmembers" visible="no" title="" intro=""/>
+    </tab>
+    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
+    <tab type="pages" visible="yes" title="" intro=""/>
+    <tab type="namespaces" visible="yes" title="">
+      <tab type="namespacelist" visible="yes" title="" intro=""/>
+      <tab type="namespacemembers" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="interfaces" visible="yes" title="">
+      <tab type="interfacelist" visible="yes" title="" intro=""/>
+      <tab type="interfaceindex" visible="$ALPHABETICAL_INDEX" title=""/>
+      <tab type="interfacehierarchy" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="structs" visible="yes" title="">
+      <tab type="structlist" visible="yes" title="" intro=""/>
+      <tab type="structindex" visible="$ALPHABETICAL_INDEX" title=""/>
+    </tab>
+    <tab type="exceptions" visible="yes" title="">
+      <tab type="exceptionlist" visible="yes" title="" intro=""/>
+      <tab type="exceptionindex" visible="$ALPHABETICAL_INDEX" title=""/>
+      <tab type="exceptionhierarchy" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="files" visible="yes" title="">
+      <tab type="filelist" visible="yes" title="" intro=""/>
+      <tab type="globals" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="examples" visible="yes" title="" intro=""/>
+  </navindex>
+</doxygenlayout>

deltachat-ffi/src/lib.rs

@@ -24,16 +24,16 @@ use std::time::{Duration, SystemTime};
 use async_std::task::{block_on, spawn};
 use num_traits::{FromPrimitive, ToPrimitive};
 
-use deltachat::accounts::Accounts;
-use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
+use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, Origin};
 use deltachat::context::Context;
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
-use deltachat::stock::StockMessage;
+use deltachat::stock_str::StockMessage;
 use deltachat::*;
+use deltachat::{accounts::Accounts, log::LogExt};
 
 mod dc_array;
 
@@ -218,7 +218,7 @@ pub unsafe extern "C" fn dc_set_config_from_qr(
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c_char {
+pub unsafe extern "C" fn dc_get_info(context: *const dc_context_t) -> *mut libc::c_char {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_get_info()");
         return "".strdup();
@@ -296,16 +296,6 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     block_on(ctx.start_io())
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_is_io_running(context: *mut dc_context_t) -> libc::c_int {
-    if context.is_null() {
-        return 0;
-    }
-    let ctx = &*context;
-
-    block_on(ctx.is_io_running()) as libc::c_int
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_id(context: *mut dc_context_t) -> libc::c_int {
     if context.is_null() {
@@ -316,7 +306,6 @@ pub unsafe extern "C" fn dc_get_id(context: *mut dc_context_t) -> libc::c_int {
     ctx.get_id() as libc::c_int
 }
 
-#[no_mangle]
 pub type dc_event_t = Event;
 
 #[no_mangle]
@@ -363,6 +352,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ErrorSelfNotInGroup(_) => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
+        | EventType::MsgsNoticed(chat_id)
         | EventType::MsgDelivered { chat_id, .. }
         | EventType::MsgFailed { chat_id, .. }
         | EventType::MsgRead { chat_id, .. }
@@ -408,6 +398,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ConfigureProgress { .. }
         | EventType::ImexProgress(_)
         | EventType::ImexFileWritten(_)
+        | EventType::MsgsNoticed(_)
         | EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
@@ -447,6 +438,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         }
         EventType::MsgsChanged { .. }
         | EventType::IncomingMsg { .. }
+        | EventType::MsgsNoticed(_)
         | EventType::MsgDelivered { .. }
         | EventType::MsgFailed { .. }
         | EventType::MsgRead { .. }
@@ -481,7 +473,6 @@ pub unsafe extern "C" fn dc_event_get_account_id(event: *mut dc_event_t) -> u32
     (*event).id
 }
 
-#[no_mangle]
 pub type dc_event_emitter_t = EventEmitter;
 
 #[no_mangle]
@@ -499,7 +490,7 @@ pub unsafe extern "C" fn dc_get_event_emitter(
 #[no_mangle]
 pub unsafe extern "C" fn dc_event_emitter_unref(emitter: *mut dc_event_emitter_t) {
     if emitter.is_null() {
-        eprintln!("ignoring careless call to dc_event_mitter_unref()");
+        eprintln!("ignoring careless call to dc_event_emitter_unref()");
         return;
     }
 
@@ -509,6 +500,7 @@ pub unsafe extern "C" fn dc_event_emitter_unref(emitter: *mut dc_event_emitter_t
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *mut dc_event_t {
     if events.is_null() {
+        eprintln!("ignoring careless call to dc_get_next_event()");
         return ptr::null_mut();
     }
     let events = &*events;
@@ -805,21 +797,6 @@ pub unsafe extern "C" fn dc_add_device_msg(
     .to_u32()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_update_device_chats(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_update_device_chats()");
-        return;
-    }
-    let ctx = &mut *context;
-
-    block_on(async move {
-        ctx.update_device_chats()
-            .await
-            .unwrap_or_log_default(&ctx, "Failed to add device message")
-    })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_was_device_msg_ever_added(
     context: *mut dc_context_t,
@@ -948,6 +925,8 @@ pub unsafe extern "C" fn dc_get_fresh_msgs(
         let arr = dc_array_t::from(
             ctx.get_fresh_msgs()
                 .await
+                .log_err(ctx, "Failed to get fresh messages")
+                .unwrap_or_default()
                 .iter()
                 .map(|msg_id| msg_id.to_u32())
                 .collect::<Vec<u32>>(),
@@ -972,22 +951,6 @@ pub unsafe extern "C" fn dc_marknoticed_chat(context: *mut dc_context_t, chat_id
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_marknoticed_all_chats(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_marknoticed_all_chats()");
-        return;
-    }
-    let ctx = &*context;
-
-    block_on(async move {
-        chat::marknoticed_all_chats(&ctx)
-            .await
-            .log_err(ctx, "Failed marknoticed all chats")
-            .unwrap_or(())
-    })
-}
-
 fn from_prim<S, T>(s: S) -> Option<T>
 where
     T: FromPrimitive,
@@ -1071,6 +1034,32 @@ pub unsafe extern "C" fn dc_get_next_media(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_set_chat_protection(
+    context: *mut dc_context_t,
+    chat_id: u32,
+    protect: libc::c_int,
+) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_set_chat_protection()");
+        return 0;
+    }
+    let ctx = &*context;
+    let protect = if let Some(s) = ProtectionStatus::from_i32(protect) {
+        s
+    } else {
+        warn!(ctx, "bad protect-value for dc_set_chat_protection()");
+        return 0;
+    };
+
+    block_on(async move {
+        match ChatId::new(chat_id).set_protection(&ctx, protect).await {
+            Ok(()) => 1,
+            Err(_) => 0,
+        }
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_chat_visibility(
     context: *mut dc_context_t,
@@ -1149,10 +1138,15 @@ pub unsafe extern "C" fn dc_search_msgs(
         return ptr::null_mut();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
     block_on(async move {
         let arr = dc_array_t::from(
-            ctx.search_msgs(ChatId::new(chat_id), to_string_lossy(query))
+            ctx.search_msgs(chat_id, to_string_lossy(query))
                 .await
                 .iter()
                 .map(|msg_id| msg_id.to_u32())
@@ -1184,7 +1178,7 @@ pub unsafe extern "C" fn dc_get_chat(context: *mut dc_context_t, chat_id: u32) -
 #[no_mangle]
 pub unsafe extern "C" fn dc_create_group_chat(
     context: *mut dc_context_t,
-    verified: libc::c_int,
+    protect: libc::c_int,
     name: *const libc::c_char,
 ) -> u32 {
     if context.is_null() || name.is_null() {
@@ -1192,14 +1186,15 @@ pub unsafe extern "C" fn dc_create_group_chat(
         return 0;
     }
     let ctx = &*context;
-    let verified = if let Some(s) = contact::VerifiedStatus::from_i32(verified) {
+    let protect = if let Some(s) = ProtectionStatus::from_i32(protect) {
         s
     } else {
+        warn!(ctx, "bad protect-value for dc_create_group_chat()");
         return 0;
     };
 
     block_on(async move {
-        chat::create_group_chat(&ctx, verified, to_string_lossy(name))
+        chat::create_group_chat(&ctx, protect, to_string_lossy(name))
             .await
             .log_err(ctx, "Failed to create group chat")
             .map(|id| id.to_u32())
@@ -1334,6 +1329,29 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_chat_encrinfo(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_chat_encrinfo()");
+        return "".strdup();
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        ChatId::new(chat_id)
+            .get_encryption_info(&ctx)
+            .await
+            .map(|s| s.strdup())
+            .unwrap_or_else(|e| {
+                error!(&ctx, "{}", e);
+                ptr::null_mut()
+            })
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_chat_ephemeral_timer(
     context: *mut dc_context_t,
@@ -1391,6 +1409,20 @@ pub unsafe extern "C" fn dc_get_msg_info(
     block_on(message::get_msg_info(&ctx, MsgId::new(msg_id))).strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_msg_html(
+    context: *mut dc_context_t,
+    msg_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_msg_html()");
+        return ptr::null_mut();
+    }
+    let ctx = &*context;
+
+    block_on(MsgId::new(msg_id).get_html(&ctx)).strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_mime_headers(
     context: *mut dc_context_t,
@@ -1478,23 +1510,6 @@ pub unsafe extern "C" fn dc_markseen_msgs(
     block_on(message::markseen_msgs(&ctx, msg_ids));
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_star_msgs(
-    context: *mut dc_context_t,
-    msg_ids: *const u32,
-    msg_cnt: libc::c_int,
-    star: libc::c_int,
-) {
-    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
-        eprintln!("ignoring careless call to dc_star_msgs()");
-        return;
-    }
-    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
-    let ctx = &*context;
-
-    block_on(message::star_msgs(&ctx, msg_ids, star == 1));
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) -> *mut dc_msg_t {
     if context.is_null() {
@@ -1554,6 +1569,9 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
         to_string_lossy(addr),
         Origin::IncomingReplyTo,
     ))
+    .ok()
+    .flatten()
+    .unwrap_or_default()
 }
 
 #[no_mangle]
@@ -1570,10 +1588,9 @@ pub unsafe extern "C" fn dc_create_contact(
     let name = to_string_lossy(name);
 
     block_on(async move {
-        match Contact::create(&ctx, name, to_string_lossy(addr)).await {
-            Ok(id) => id,
-            Err(_) => 0,
-        }
+        Contact::create(&ctx, name, to_string_lossy(addr))
+            .await
+            .unwrap_or(0)
     })
 }
 
@@ -1625,7 +1642,13 @@ pub unsafe extern "C" fn dc_get_blocked_cnt(context: *mut dc_context_t) -> libc:
     }
     let ctx = &*context;
 
-    block_on(Contact::get_blocked_cnt(&ctx)) as libc::c_int
+    block_on(async move {
+        Contact::get_all_blocked(&ctx)
+            .await
+            .log_err(&ctx, "Can't get blocked count")
+            .unwrap_or_default()
+            .len() as libc::c_int
+    })
 }
 
 #[no_mangle]
@@ -1640,7 +1663,10 @@ pub unsafe extern "C" fn dc_get_blocked_contacts(
 
     block_on(async move {
         Box::into_raw(Box::new(dc_array_t::from(
-            Contact::get_all_blocked(&ctx).await,
+            Contact::get_all_blocked(&ctx)
+                .await
+                .log_err(&ctx, "Can't get blocked contacts")
+                .unwrap_or_default(),
         )))
     })
 }
@@ -1746,13 +1772,15 @@ pub unsafe extern "C" fn dc_imex(
 
     let ctx = &*context;
 
-    let param1 = to_opt_string_lossy(param1);
-
-    spawn(async move {
-        imex::imex(&ctx, what, param1)
-            .await
-            .log_err(ctx, "IMEX failed")
-    });
+    if let Some(param1) = to_opt_string_lossy(param1) {
+        spawn(async move {
+            imex::imex(&ctx, what, &param1)
+                .await
+                .log_err(ctx, "IMEX failed")
+        });
+    } else {
+        eprintln!("dc_imex called without a valid directory");
+    }
 }
 
 #[no_mangle]
@@ -1819,7 +1847,7 @@ pub unsafe extern "C" fn dc_continue_key_transfer(
         {
             Ok(()) => 1,
             Err(err) => {
-                error!(&ctx, "dc_continue_key_transfer: {}", err);
+                warn!(&ctx, "dc_continue_key_transfer: {}", err);
                 0
             }
         }
@@ -1863,9 +1891,14 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
         return "".strdup();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
     block_on(async move {
-        securejoin::dc_get_securejoin_qr(&ctx, ChatId::new(chat_id))
+        securejoin::dc_get_securejoin_qr(&ctx, chat_id)
             .await
             .unwrap_or_else(|| "".to_string())
             .strdup()
@@ -1921,11 +1954,13 @@ pub unsafe extern "C" fn dc_is_sending_locations_to_chat(
         return 0;
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
-    block_on(location::is_sending_locations_to_chat(
-        &ctx,
-        ChatId::new(chat_id),
-    )) as libc::c_int
+    block_on(location::is_sending_locations_to_chat(&ctx, chat_id)) as libc::c_int
 }
 
 #[no_mangle]
@@ -1957,11 +1992,21 @@ pub unsafe extern "C" fn dc_get_locations(
         return ptr::null_mut();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
+    let contact_id = if contact_id == 0 {
+        None
+    } else {
+        Some(contact_id)
+    };
 
     block_on(async move {
         let res = location::get_range(
             &ctx,
-            ChatId::new(chat_id),
+            chat_id,
             contact_id,
             timestamp_begin as i64,
             timestamp_end as i64,
@@ -1989,7 +2034,6 @@ pub unsafe extern "C" fn dc_delete_all_locations(context: *mut dc_context_t) {
 
 // dc_array_t
 
-#[no_mangle]
 pub type dc_array_t = dc_array::dc_array_t;
 
 #[no_mangle]
@@ -2172,7 +2216,6 @@ pub struct ChatlistWrapper {
     list: chatlist::Chatlist,
 }
 
-#[no_mangle]
 pub type dc_chatlist_t = ChatlistWrapper;
 
 #[no_mangle]
@@ -2296,7 +2339,6 @@ pub struct ChatWrapper {
     chat: chat::Chat,
 }
 
-#[no_mangle]
 pub type dc_chat_t = ChatWrapper;
 
 #[no_mangle]
@@ -2423,13 +2465,13 @@ pub unsafe extern "C" fn dc_chat_can_send(chat: *mut dc_chat_t) -> libc::c_int {
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_chat_is_verified(chat: *mut dc_chat_t) -> libc::c_int {
+pub unsafe extern "C" fn dc_chat_is_protected(chat: *mut dc_chat_t) -> libc::c_int {
     if chat.is_null() {
-        eprintln!("ignoring careless call to dc_chat_is_verified()");
+        eprintln!("ignoring careless call to dc_chat_is_protected()");
         return 0;
     }
     let ffi_chat = &*chat;
-    ffi_chat.chat.is_verified() as libc::c_int
+    ffi_chat.chat.is_protected() as libc::c_int
 }
 
 #[no_mangle]
@@ -2522,7 +2564,6 @@ pub struct MessageWrapper {
     message: message::Message,
 }
 
-#[no_mangle]
 pub type dc_msg_t = MessageWrapper;
 
 #[no_mangle]
@@ -2583,6 +2624,16 @@ pub unsafe extern "C" fn dc_msg_get_chat_id(msg: *mut dc_msg_t) -> u32 {
     ffi_msg.message.get_chat_id().to_u32()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_real_chat_id(msg: *mut dc_msg_t) -> u32 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_real_chat_id()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_real_chat_id().to_u32()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_viewtype(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2647,6 +2698,16 @@ pub unsafe extern "C" fn dc_msg_get_text(msg: *mut dc_msg_t) -> *mut libc::c_cha
     ffi_msg.message.get_text().unwrap_or_default().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_subject(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_subject()");
+        return "".strdup();
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_subject().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -2745,7 +2806,7 @@ pub unsafe extern "C" fn dc_msg_get_ephemeral_timer(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_ephemeral_timer()
+    ffi_msg.message.get_ephemeral_timer().to_u32()
 }
 
 #[no_mangle]
@@ -2802,6 +2863,17 @@ pub unsafe extern "C" fn dc_msg_get_summarytext(
     .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_override_sender_name(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_override_sender_name()");
+        return "".strdup();
+    }
+    let ffi_msg = &mut *msg;
+
+    ffi_msg.message.get_override_sender_name().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_has_deviating_timestamp(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2832,16 +2904,6 @@ pub unsafe extern "C" fn dc_msg_is_sent(msg: *mut dc_msg_t) -> libc::c_int {
     ffi_msg.message.is_sent().into()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_is_starred(msg: *mut dc_msg_t) -> libc::c_int {
-    if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_is_starred()");
-        return 0;
-    }
-    let ffi_msg = &*msg;
-    ffi_msg.message.is_starred().into()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_is_forwarded(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2862,6 +2924,16 @@ pub unsafe extern "C" fn dc_msg_is_info(msg: *mut dc_msg_t) -> libc::c_int {
     ffi_msg.message.is_info().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_info_type(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_info_type()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_info_type() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_is_increation(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2882,6 +2954,16 @@ pub unsafe extern "C" fn dc_msg_is_setupmessage(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.is_setupmessage().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_has_html(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_has_html()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.has_html().into()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -2897,6 +2979,32 @@ pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut li
         .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_decide_on_contact_request(
+    context: *mut dc_context_t,
+    msg_id: u32,
+    decision: libc::c_int,
+) -> u32 {
+    if context.is_null() || msg_id <= constants::DC_MSG_ID_LAST_SPECIAL as u32 {
+        eprintln!("ignoring careless call to dc_decide_on_contact_request()");
+    }
+    let ctx = &*context;
+
+    match from_prim(decision) {
+        None => {
+            warn!(ctx, "{} is not a valid decision, ignoring", decision);
+            0
+        }
+        Some(d) => block_on(message::decide_on_contact_request(
+            ctx,
+            MsgId::new(msg_id),
+            d,
+        ))
+        .unwrap_or_default()
+        .to_u32(),
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_videochat_type(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2931,6 +3039,31 @@ pub unsafe extern "C" fn dc_msg_set_text(msg: *mut dc_msg_t, text: *const libc::
     ffi_msg.message.set_text(to_opt_string_lossy(text))
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_html(msg: *mut dc_msg_t, html: *const libc::c_char) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_html()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg.message.set_html(to_opt_string_lossy(html))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_override_sender_name(
+    msg: *mut dc_msg_t,
+    name: *const libc::c_char,
+) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_override_sender_name()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg
+        .message
+        .set_override_sender_name(to_opt_string_lossy(name))
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_set_file(
     msg: *mut dc_msg_t,
@@ -3007,6 +3140,79 @@ pub unsafe extern "C" fn dc_msg_latefiling_mediasize(
     });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_error(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_error()");
+        return ptr::null_mut();
+    }
+    let ffi_msg = &*msg;
+    match ffi_msg.message.error() {
+        Some(error) => error.strdup(),
+        None => ptr::null_mut(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_quote(msg: *mut dc_msg_t, quote: *const dc_msg_t) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_quote()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    let ffi_quote = &*quote;
+
+    if ffi_msg.context != ffi_quote.context {
+        eprintln!("ignoring attempt to quote message from a different context");
+        return;
+    }
+
+    block_on(async move {
+        ffi_msg
+            .message
+            .set_quote(&*ffi_msg.context, &ffi_quote.message)
+            .await
+            .log_err(&*ffi_msg.context, "failed to set quote")
+            .ok();
+    });
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_quoted_text(msg: *const dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_quoted_text()");
+        return ptr::null_mut();
+    }
+    let ffi_msg: &MessageWrapper = &*msg;
+    ffi_msg
+        .message
+        .quoted_text()
+        .map_or_else(ptr::null_mut, |s| s.strdup())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_quoted_msg(msg: *const dc_msg_t) -> *mut dc_msg_t {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_get_quoted_msg()");
+        return ptr::null_mut();
+    }
+    let ffi_msg: &MessageWrapper = &*msg;
+    let context = &*ffi_msg.context;
+    let res = block_on(async move {
+        ffi_msg
+            .message
+            .quoted_message(context)
+            .await
+            .log_err(context, "failed to get quoted message")
+            .unwrap_or(None)
+    });
+
+    match res {
+        Some(message) => Box::into_raw(Box::new(MessageWrapper { context, message })),
+        None => ptr::null_mut(),
+    }
+}
+
 // dc_contact_t
 
 /// FFI struct for [dc_contact_t]
@@ -3021,7 +3227,6 @@ pub struct ContactWrapper {
     contact: contact::Contact,
 }
 
-#[no_mangle]
 pub type dc_contact_t = ContactWrapper;
 
 #[no_mangle]
@@ -3063,6 +3268,16 @@ pub unsafe extern "C" fn dc_contact_get_name(contact: *mut dc_contact_t) -> *mut
     ffi_contact.contact.get_name().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_auth_name(contact: *mut dc_contact_t) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_auth_name()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.get_authname().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_display_name(
     contact: *mut dc_contact_t,
@@ -3087,18 +3302,6 @@ pub unsafe extern "C" fn dc_contact_get_name_n_addr(
     ffi_contact.contact.get_name_n_addr().strdup()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_contact_get_first_name(
-    contact: *mut dc_contact_t,
-) -> *mut libc::c_char {
-    if contact.is_null() {
-        eprintln!("ignoring careless call to dc_contact_get_first_name()");
-        return "".strdup();
-    }
-    let ffi_contact = &*contact;
-    ffi_contact.contact.get_first_name().strdup()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_profile_image(
     contact: *mut dc_contact_t,
@@ -3130,6 +3333,16 @@ pub unsafe extern "C" fn dc_contact_get_color(contact: *mut dc_contact_t) -> u32
     ffi_contact.contact.get_color()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_status(contact: *mut dc_contact_t) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_status()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.get_status().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_is_blocked(contact: *mut dc_contact_t) -> libc::c_int {
     if contact.is_null() {
@@ -3154,7 +3367,6 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
 
 // dc_lot_t
 
-#[no_mangle]
 pub type dc_lot_t = lot::Lot;
 
 #[no_mangle]
@@ -3239,15 +3451,11 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
 }
 
 trait ResultExt<T, E> {
+    /// Like `log_err()`, but:
+    /// - returns the default value instead of an Err value.
+    /// - emits an error instead of a warning for an [Err] result. This means
+    /// that the error will be shown to the user in a small pop-up.
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T;
-
-    /// Log a warning to a [ContextWrapper] for an [Err] result.
-    ///
-    /// Does nothing for an [Ok].
-    ///
-    /// You can do this as soon as the wrapper exists, it does not
-    /// have to be open (which is required for the `warn!()` macro).
-    fn log_err(self, wrapper: &Context, message: &str) -> Result<T, E>;
 }
 
 impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
@@ -3260,13 +3468,6 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
             }
         }
     }
-
-    fn log_err(self, ctx: &Context, message: &str) -> Result<T, E> {
-        self.map_err(|err| {
-            warn!(ctx, "{}: {}", message, err);
-            err
-        })
-    }
 }
 
 trait ResultNullableExt<T> {
@@ -3295,7 +3496,6 @@ fn convert_and_prune_message_ids(msg_ids: *const u32, msg_cnt: libc::c_int) -> V
 
 // dc_provider_t
 
-#[no_mangle]
 pub type dc_provider_t = provider::Provider;
 
 #[no_mangle]
@@ -3308,7 +3508,7 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
         return ptr::null();
     }
     let addr = to_string_lossy(addr);
-    match provider::get_provider_info(addr.as_str()) {
+    match block_on(provider::get_provider_info(addr.as_str())) {
         Some(provider) => provider,
         None => ptr::null_mut(),
     }
@@ -3386,7 +3586,8 @@ pub unsafe extern "C" fn dc_accounts_new(
     match accs {
         Ok(accs) => Box::into_raw(Box::new(accs)),
         Err(err) => {
-            eprintln!("failed to create accounts: {}", err);
+            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+            eprintln!("failed to create accounts: {:#}", err);
             ptr::null_mut()
         }
     }
@@ -3447,7 +3648,7 @@ pub unsafe extern "C" fn dc_accounts_select_account(
     let accounts = &*accounts;
     block_on(accounts.select_account(id))
         .map(|_| 1)
-        .unwrap_or_else(|_| 0)
+        .unwrap_or(0)
 }
 
 #[no_mangle]
@@ -3459,7 +3660,7 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -
 
     let accounts = &*accounts;
 
-    block_on(accounts.add_account()).unwrap_or_else(|_| 0)
+    block_on(accounts.add_account()).unwrap_or(0)
 }
 
 #[no_mangle]
@@ -3561,7 +3762,6 @@ pub unsafe extern "C" fn dc_accounts_maybe_network(accounts: *mut dc_accounts_t)
     block_on(accounts.maybe_network());
 }
 
-#[no_mangle]
 pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
 
 #[no_mangle]
@@ -3595,9 +3795,10 @@ pub unsafe extern "C" fn dc_accounts_get_next_event(
     emitter: *mut dc_accounts_event_emitter_t,
 ) -> *mut dc_event_t {
     if emitter.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_next_event()");
         return ptr::null_mut();
     }
-    let emitter = &*emitter;
+    let emitter = &mut *emitter;
 
     emitter
         .recv_sync()

draft/group-sync.rst

@@ -5,69 +5,95 @@ Problem: missing eventual group consistency
 If group members are concurrently adding new members,
 the new members will miss each other's additions, example:
 
-- Alice and Bob are in a two-member group
+1. Alice and Bob are in a two-member group
 
-- Alice adds Carol, concurrently Bob adds Doris
+2. Then Alice adds Carol, while concurrently Bob adds Doris
 
-- Carol will see a three-member group (Alice, Bob, Carol),
-  Doris will see a different three-member group (Alice, Bob, Doris),
-  and only Alice and Bob will have all four members.
+Right now, the group has inconsistent memberships:
 
-Note that for verified groups any mitigation mechanism likely
-needs to make all clients to know who originally added a member.
+- Alice and Carol see a (Alice, Carol, Bob) group
+
+- Bob and Doris see a (Bob, Doris, Alice)
+
+This then leads to "sender is unknown" messages in the chat,
+for example when Alice receives a message from Doris,
+or when Bob receives a message from Carol.
+
+There are also other sources for group membership inconsistency:
+
+- leaving/deleting/adding in larger groups, while being offline,
+  increases chances for inconsistent group membership
+
+- dropped group-membership messages
+
+- group-membership messages landing in "Spam"
 
 
-solution: memorize+attach (possible encrypted) chat-meta mime messages
-----------------------------------------------------------------------
+Note that all these problems (can) also happen with verified groups,
+then raising "false alarms" which could lure people to ignore such issues.
 
-For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
+IOW, it's clear we need to do something about it to improve overall
+reliability in group-settings.
 
 
-- All Chat-Group-Member-Added/Removed messages are recorded in their
-  full raw (signed and encrypted) mime-format in the DB
 
-- If an incoming member-add/member-delete messages has a member list
-  which is, apart from the added/removed member, not consistent
-  with our own view, broadcast a "Chat-Group-Member-Correction" message to
-  all members, attaching the original added/removed mime-message for all mismatching
-  contacts.  If we have no relevant add/del information, don't send a
-  correction message out.
+Solution: replay group modification messages on inconsistencies
+------------------------------------------------------------------
 
-- Upong receiving added/removed attachments we don't do the
-  check_consistency+correction message dance.
-  This avoids recursion problems and hard-to-reason-about chatter.
+For brevity let's abbreviate "group membership modification" as **GMM**.
 
-Notes:
+Delta chat has explicit GMM messages, typically encrypted to the group members
+as seen by the device that sends the GMM. The `Spec <https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members>`_ details the Mime headers and format.
 
-- mechanism works for both encrypted and unencrypted add/del messages
+If we detect membership inconsistencies we can resend relevant GMM messages
+to the respective chat.  The receiving devices can process those GMM messages
+as if it would be an incoming message. If for example they have already seen
+the Message-ID of the GMM message, they will ignore the message. It's
+probably useful to record GMM message in their original MIME-format and
+not invent a new recording format. Few notes on three aspects:
 
-- we already have a "mime_headers" column in the DB for each incoming message.
-  We could extend it to also include the payload and store mime unconditionally
-  for member-added/removed messages.
+- **group-membership-tracking**: All valid GMM messages are persisted in
+  their full raw (signed/encrypted?) MIME-format in the DB. Note that GMM messages
+  already are in the msgs table, and there is a mime_header column which we could
+  extend to contain the raw Mime GMM message.
 
-- multiple member-added/removed messages can be attached in a single
-  correction message
+- **consistency_checking**: If an incoming GMM has a member list which is
+  not consistent with our own view, broadcast a "Group-Member-Correction"
+  message to all members containing a multipart list of GMMs.
 
-- it is minimal on the number of overall messages to reach group consistency
-  (best-case: no extra messages, the ABCD case above: max two extra messages)
+- **correcting_memberships**: Upon receiving a Group-Member-Correction
+  message we pass the contained GMMs to the "incoming mail pipeline"
+  (without **consistency_checking** them, to avoid recursion issues)
 
-- somewhat backward compatible: older clients will probably ignore
-  messages which are signed by someone who is not the outer From-address.
 
-- the correction-protocol also helps with dropped messages.  If a member
-  did not see a member-added/removed message, the next member add/removed
-  message in the group will likely heal group consistency for this member.
+Alice/Carol and Bob/Doris getting on the same page
+++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Recall that Alice/Carol and Bob/Doris had a differening view of
+group membership. With the proposed solution, when Bob receives
+Alice's "Carol added" message, he will notice that Alice (and thus
+also carol) did not know about Doris.  Bob's device sends a
+"Chat-Group-Member-Correction" message containing his own GMM
+when adding Doris. Therefore, the group's membership is healed
+for everyone in a single broadcast message.
+
+Alice might also send a Group-member-Correction message,
+so there is a second chance that the group gets to know all GMMs.
+
+Note, for example, that if for some reason Bobs and Carols provider
+drop GMM messages between them (spam) that Alice and Doris can heal
+it by resending GMM messages whenever they detect them to be out of sync.
 
-- we can quite easily extend the mechanism to also provide the group-avatar or
-  other meta-information.
 
 Discussions of variants
 ++++++++++++++++++++++++
 
-- instead of acting on MemberAdded/Removed message we could send
-  corrections for any received message that addresses inconsistent group members but
-  a) this would delay group-membership healing
+- instead of acting on GMM messages we could send corrections
+  for any received message that addresses inconsistent group members but
+  a) this could delay group-membership healing
   b) could lead to a lot of members sending corrections
+  c) means we might rely on "To-Addresses" which we also like to strike
+     at least for protected chats.
 
 - instead of broadcasting correction messages we could only send it to
   the sender of the inconsistent member-added/removed message.
@@ -83,44 +109,3 @@ Discussions of variants
   while both being in an offline or bad-connection situation).
 
 
-solution2: repeat member-added/removed messages
----------------------------------------------------
-
-Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
-but keep sending out the old headers until the new protocol is sufficiently deployed.
-
-The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
-which controls repetition of the signed "add/del e-mail address" payload.
-
-Example::
-
-    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
-        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
-        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
-        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
-        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
-        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
-        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
-        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
-        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
-        =P6GG
-
-TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
-Receivers will apply the add/del change to the group-membership,
-decrease the TTL by 1, and if TTL>0 re-sent the header.
-
-The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
-This allows to propagate, in a cryptographically secured way,
-who added a member. This is particularly important for allowing
-to show in verified groups who added a member (planned).
-
-Disadvantage to solution 1:
-
-- requires to specify encoding and precise rules for what/how is signed.
-
-- causes O(N^2) extra messages
-
-- Not easily extendable for other things (without introducing a new
-  header / encoding)
-
-

examples/repl/cmdline.rs

@@ -2,25 +2,29 @@ extern crate dirs;
 
 use std::str::FromStr;
 
-use anyhow::{bail, ensure};
+use anyhow::{bail, ensure, Error};
 use async_std::path::Path;
-use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility};
+use deltachat::chat::{
+    self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
+};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
-use deltachat::error::Error;
 use deltachat::imex::*;
 use deltachat::location;
+use deltachat::log::LogExt;
 use deltachat::lot::LotState;
-use deltachat::message::{self, Message, MessageState, MsgId};
+use deltachat::message::{self, ContactRequestDecision, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
 use deltachat::EventType;
 use deltachat::{config, provider};
+use std::fs;
+use std::time::{Duration, SystemTime};
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
@@ -171,8 +175,11 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let contact = Contact::get_by_id(context, msg.get_from_id())
         .await
         .expect("invalid contact");
-
-    let contact_name = contact.get_name();
+    let contact_name = if let Some(name) = msg.get_override_sender_name() {
+        format!("~{}", name)
+    } else {
+        contact.get_display_name().to_string()
+    };
     let contact_id = contact.get_id();
 
     let statestr = match msg.get_state() {
@@ -193,7 +200,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.is_starred() { "★" } else { "" },
+        if msg.has_html() { "[HAS-HTML]️" } else { "" },
         if msg.get_from_id() == 1 as libc::c_uint {
             ""
         } else if msg.get_state() == MessageState::InSeen {
@@ -252,15 +259,11 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error>
 }
 
 async fn log_contactlist(context: &Context, contacts: &[u32]) {
-    let mut contacts = contacts.to_vec();
-    if !contacts.contains(&1) {
-        contacts.push(1);
-    }
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
-            let name = contact.get_name();
+        if let Ok(contact) = Contact::get_by_id(context, *contact_id).await {
+            let name = contact.get_display_name();
             let addr = contact.get_addr();
             let verified_state = contact.is_verified(context).await;
             let verified_str = if VerifiedStatus::Unverified != verified_state {
@@ -289,14 +292,14 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
             let peerstate = Peerstate::from_addr(context, &addr)
                 .await
                 .expect("peerstate error");
-            if peerstate.is_some() && contact_id != 1 as libc::c_uint {
+            if peerstate.is_some() && *contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
                     peerstate.as_ref().unwrap().prefer_encrypt
                 );
             }
 
-            println!("Contact#{}: {}{}", contact_id, line, line2);
+            println!("Contact#{}: {}{}", *contact_id, line, line2);
         }
     }
 }
@@ -356,9 +359,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listarchived\n\
                  chat [<chat-id>|0]\n\
                  createchat <contact-id>\n\
-                 createchatbymsg <msg-id>\n\
                  creategroup <name>\n\
-                 createverified <name>\n\
+                 createprotected <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
                  groupname <name>\n\
@@ -369,9 +371,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  dellocations\n\
                  getlocations [<contact-id>]\n\
                  send <text>\n\
-                 send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
+                 sendhtml <file for html-part> [<text for plain-part>]\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -380,15 +382,22 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  unarchive <chat-id>\n\
                  pin <chat-id>\n\
                  unpin <chat-id>\n\
+                 mute <chat-id> [<seconds>]\n\
+                 unmute <chat-id>\n\
+                 protect <chat-id>\n\
+                 unprotect <chat-id>\n\
                  delchat <chat-id>\n\
+                 ===========================Contact requests==\n\
+                 decidestartchat <msg-id>\n\
+                 decideblock <msg-id>\n\
+                 decidenotnow <msg-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
+                 html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
-                 star <msg-id>\n\
-                 unstar <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
                  listcontacts [<query>]\n\
@@ -397,10 +406,14 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  contactinfo <contact-id>\n\
                  delcontact <contact-id>\n\
                  cleanupcontacts\n\
+                 block <contact-id>\n\
+                 unblock <contact-id>\n\
+                 listblocked\n\
                  ======================================Misc.==\n\
                  getqr [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
+                 setqr <qr-content>\n\
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
@@ -444,20 +457,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportBackup, Some(&dir)).await?;
+            imex(&context, ImexMode::ExportBackup, &dir).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
+            imex(&context, ImexMode::ImportBackup, arg1).await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?;
+            imex(&context, ImexMode::ExportSelfKeys, &dir).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -501,7 +514,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             context.maybe_network().await;
         }
         "housekeeping" => {
-            sql::housekeeping(&context).await;
+            sql::housekeeping(&context).await.ok_or_log(&context);
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
@@ -526,16 +539,18 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}",
+                        "{}#{}: {} [{} fresh] {}{}{}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
                         chat.get_id().get_fresh_msg_cnt(&context).await,
+                        if chat.is_muted() { "🔇" } else { "" },
                         match chat.visibility {
                             ChatVisibility::Normal => "",
                             ChatVisibility::Archived => "📦",
                             ChatVisibility::Pinned => "📌",
                         },
+                        if chat.is_protected() { "🛡️" } else { "" },
                     );
                     let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
@@ -570,7 +585,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, ChatId::new(0)).await {
+            if location::is_sending_locations_to_chat(&context, None).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
@@ -606,15 +621,18 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
                 let contact = Contact::get_by_id(&context, members[0]).await?;
                 contact.get_addr().to_string()
+            } else if sel_chat.get_type() == Chattype::Mailinglist && !members.is_empty() {
+                "mailinglist".to_string()
             } else {
                 format!("{} member(s)", members.len())
             };
             println!(
-                "{}#{}: {} [{}]{}{}",
+                "{}#{}: {} [{}]{}{}{} {}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
                 sel_chat.get_name(),
                 subtitle,
+                if sel_chat.is_muted() { "🔇" } else { "" },
                 if sel_chat.is_sending_locations() {
                     "📍"
                 } else {
@@ -627,6 +645,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     },
                     _ => "".to_string(),
                 },
+                if sel_chat.is_protected() {
+                    "🛡️"
+                } else {
+                    ""
+                },
             );
             log_msglist(&context, &msglist).await?;
             if let Some(draft) = sel_chat.get_id().get_draft(&context).await? {
@@ -646,26 +669,48 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!("Single#{} created successfully.", chat_id,);
         }
-        "createchatbymsg" => {
+        "decidestartchat" | "createchatbymsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
             let msg_id = MsgId::new(arg1.parse()?);
-            let chat_id = chat::create_by_msg_id(&context, msg_id).await?;
-            let chat = Chat::load_from_db(&context, chat_id).await?;
-
-            println!("{}#{} created successfully.", chat_prefix(&chat), chat_id,);
+            match message::decide_on_contact_request(
+                &context,
+                msg_id,
+                ContactRequestDecision::StartChat,
+            )
+            .await
+            {
+                Some(chat_id) => {
+                    let chat = Chat::load_from_db(&context, chat_id).await?;
+                    println!("{}#{} created successfully.", chat_prefix(&chat), chat_id);
+                }
+                None => println!("Cannot crate chat."),
+            }
+        }
+        "decidenotnow" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::NotNow)
+                .await;
+        }
+        "decideblock" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::Block)
+                .await;
         }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
-                chat::create_group_chat(&context, VerifiedStatus::Unverified, arg1).await?;
+                chat::create_group_chat(&context, ProtectionStatus::Unprotected, arg1).await?;
 
             println!("Group#{} created successfully.", chat_id);
         }
-        "createverified" => {
+        "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id = chat::create_group_chat(&context, VerifiedStatus::Verified, arg1).await?;
+            let chat_id =
+                chat::create_group_chat(&context, ProtectionStatus::Protected, arg1).await?;
 
-            println!("VerifiedGroup#{} created successfully.", chat_id);
+            println!("Group#{} created and protected successfully.", chat_id);
         }
         "addmember" => {
             ensure!(sel_chat.is_some(), "No chat selected");
@@ -726,7 +771,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 contacts.len(),
                 location::is_sending_locations_to_chat(
                     &context,
-                    sel_chat.as_ref().unwrap().get_id()
+                    Some(sel_chat.as_ref().unwrap().get_id())
                 )
                 .await,
             );
@@ -734,10 +779,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "getlocations" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
-            let contact_id = arg1.parse().unwrap_or_default();
+            let contact_id: Option<u32> = arg1.parse().ok();
             let locations = location::get_range(
                 &context,
-                sel_chat.as_ref().unwrap().get_id(),
+                Some(sel_chat.as_ref().unwrap().get_id()),
                 contact_id,
                 0,
                 0,
@@ -825,6 +870,22 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "sendhtml" => {
+            ensure!(sel_chat.is_some(), "No chat selected.");
+            ensure!(!arg1.is_empty(), "No html-file given.");
+            let path: &Path = arg1.as_ref();
+            let html = &*fs::read(&path)?;
+            let html = String::from_utf8_lossy(html);
+
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_html(Some(html.to_string()));
+            msg.set_text(Some(if arg2.is_empty() {
+                path.file_name().unwrap().to_string_lossy().to_string()
+            } else {
+                arg2.to_string()
+            }));
+            chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
+        }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -833,9 +894,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
             let chat = if let Some(ref sel_chat) = sel_chat {
-                sel_chat.get_id()
+                Some(sel_chat.get_id())
             } else {
-                ChatId::new(0)
+                None
             };
 
             let msglist = context.search_msgs(chat, arg1).await;
@@ -875,9 +936,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg.set_text(Some(arg1.to_string()));
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
-        "updatedevicechats" => {
-            context.update_device_chats().await?;
-        }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
@@ -909,7 +967,39 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                         "archive" => ChatVisibility::Archived,
                         "unarchive" | "unpin" => ChatVisibility::Normal,
                         "pin" => ChatVisibility::Pinned,
-                        _ => panic!("Unexpected command (This should never happen)"),
+                        _ => unreachable!("arg0={:?}", arg0),
+                    },
+                )
+                .await?;
+        }
+        "mute" | "unmute" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            let duration = match arg0 {
+                "mute" => {
+                    if arg2.is_empty() {
+                        MuteDuration::Forever
+                    } else {
+                        SystemTime::now()
+                            .checked_add(Duration::from_secs(arg2.parse()?))
+                            .map_or(MuteDuration::Forever, MuteDuration::Until)
+                    }
+                }
+                "unmute" => MuteDuration::NotMuted,
+                _ => unreachable!("arg0={:?}", arg0),
+            };
+            chat::set_muted(&context, chat_id, duration).await?;
+        }
+        "protect" | "unprotect" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id
+                .set_protection(
+                    &context,
+                    match arg0 {
+                        "protect" => ProtectionStatus::Protected,
+                        "unprotect" => ProtectionStatus::Unprotected,
+                        _ => unreachable!("arg0={:?}", arg0),
                     },
                 )
                 .await?;
@@ -925,8 +1015,18 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let res = message::get_msg_info(&context, id).await;
             println!("{}", res);
         }
+        "html" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            let file = dirs::home_dir()
+                .unwrap_or_default()
+                .join(format!("msg-{}.html", id.to_u32()));
+            let html = id.get_html(&context).await.unwrap_or_default();
+            fs::write(&file, html)?;
+            println!("HTML written to: {:#?}", file);
+        }
         "listfresh" => {
-            let msglist = context.get_fresh_msgs().await;
+            let msglist = context.get_fresh_msgs().await?;
 
             log_msglist(&context, &msglist).await?;
             print!("{} fresh messages.", msglist.len());
@@ -948,12 +1048,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             message::markseen_msgs(&context, msg_ids).await;
         }
-        "star" | "unstar" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = vec![MsgId::new(0); 1];
-            msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::star_msgs(&context, msg_ids, arg0 == "star").await;
-        }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
@@ -964,9 +1058,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let contacts = Contact::get_all(
                 &context,
                 if arg0 == "listverified" {
-                    0x1 | 0x2
+                    DC_GCL_VERIFIED_ONLY | DC_GCL_ADD_SELF
                 } else {
-                    0x2
+                    DC_GCL_ADD_SELF
                 },
                 Some(arg1),
             )
@@ -1024,6 +1118,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             Contact::delete(&context, arg1.parse()?).await?;
         }
+        "block" => {
+            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
+            let contact_id = arg1.parse()?;
+            Contact::block(&context, contact_id).await;
+        }
+        "unblock" => {
+            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
+            let contact_id = arg1.parse()?;
+            Contact::unblock(&context, contact_id).await;
+        }
+        "listblocked" => {
+            let contacts = Contact::get_all_blocked(&context).await?;
+            log_contactlist(&context, &contacts).await;
+            println!("{} blocked contacts.", contacts.len());
+        }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
             let res = check_qr(&context, arg1).await;
@@ -1044,7 +1153,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            match provider::get_provider_info(arg1) {
+            match provider::get_provider_info(arg1).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);

examples/repl/main.rs

@@ -168,12 +168,14 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 27] = [
+const CHAT_COMMANDS: [&str; 34] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
-    "createchatbymsg",
+    "decidestartchat",
+    "decideblock",
+    "decidenotnow",
     "creategroup",
     "createverified",
     "addmember",
@@ -188,6 +190,7 @@ const CHAT_COMMANDS: [&str; 27] = [
     "send",
     "sendimage",
     "sendfile",
+    "sendhtml",
     "videochat",
     "draft",
     "listmedia",
@@ -195,25 +198,30 @@ const CHAT_COMMANDS: [&str; 27] = [
     "unarchive",
     "pin",
     "unpin",
+    "mute",
+    "unmute",
+    "protect",
+    "unprotect",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&str; 6] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
-    "star",
-    "unstar",
     "delmsg",
 ];
-const CONTACT_COMMANDS: [&str; 6] = [
+const CONTACT_COMMANDS: [&str; 9] = [
     "listcontacts",
     "listverified",
     "addcontact",
     "contactinfo",
     "delcontact",
     "cleanupcontacts",
+    "block",
+    "unblock",
+    "listblocked",
 ];
 const MISC_COMMANDS: [&str; 10] = [
     "getqr",
@@ -300,7 +308,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::sync::channel(100);
+    let (reader_s, reader_r) = async_std::channel::bounded(100);
     let input_loop = async_std::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
@@ -323,7 +331,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line));
+                    async_std::task::block_on(reader_s.send(line)).unwrap();
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
@@ -400,9 +408,8 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            if let Some(mut qr) =
-                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
-            {
+            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")

python/README.rst

@@ -11,10 +11,10 @@ Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
-yourself <sourceinstall>`_.
+without any "build-from-source" steps.
+Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -19,8 +19,7 @@ if __name__ == "__main__":
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
 
     if target == 'release':
-        extra = " -C lto=on -C embed-bitcode=yes"
-        os.environ["RUSTFLAGS"] = os.environ.get("RUSTFLAGS", "") + extra
+        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
 
     print("running:", " ".join(cmd))

python/src/deltachat/__init__.py

@@ -77,6 +77,7 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
+        ac.set_config("bot", "1")
         configtracker = ac.configure()
         configtracker.wait_finish()
 

python/src/deltachat/_build.py

@@ -145,9 +145,12 @@ def extract_defines(flags):
                 | DC_STR
                 | DC_CONTACT_ID
                 | DC_GCL
+                | DC_GCM
+                | DC_SOCKET
                 | DC_CHAT
                 | DC_PROVIDER
                 | DC_KEY_GEN
+                | DC_IMEX
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains

python/src/deltachat/account.py

@@ -214,6 +214,19 @@ class Account(object):
         :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
+        (name, addr) = self.get_contact_addr_and_name(obj, name)
+        name = as_dc_charpointer(name)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        return Contact(self, contact_id)
+
+    def get_contact(self, obj):
+        if isinstance(obj, Contact):
+            return obj
+        (_, addr) = self.get_contact_addr_and_name(obj)
+        return self.get_contact_by_addr(addr)
+
+    def get_contact_addr_and_name(self, obj, name=None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -229,13 +242,7 @@ class Account(object):
 
         if name is None and displayname:
             name = displayname
-        return self._create_contact(addr, name)
-
-    def _create_contact(self, addr, name):
-        addr = as_dc_charpointer(addr)
-        name = as_dc_charpointer(name)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        return Contact(self, contact_id)
+        return (name, addr)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -263,6 +270,17 @@ class Account(object):
         """
         return Contact(self, contact_id)
 
+    def get_blocked_contacts(self):
+        """ return a list of all blocked contacts.
+
+        :returns: list of :class:`deltachat.contact.Contact` objects.
+        """
+        dc_array = ffi.gc(
+            lib.dc_get_blocked_contacts(self._dc_context),
+            lib.dc_array_unref
+        )
+        return list(iter_array(dc_array, lambda x: Contact(self, x)))
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -336,6 +354,9 @@ class Account(object):
     def get_deaddrop_chat(self):
         return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
+    def get_device_chat(self):
+        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
+
     def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
@@ -391,23 +412,23 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        return self._export(path, imex_cmd=1)
+        return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
 
-        Note that the account does not have to be started.
+        Note that the account has to be stopped; call stop_io() if necessary.
         """
-        export_files = self._export(path, 11)
+        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            self.imex(path, imex_cmd)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -417,7 +438,7 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        self._import(path, imex_cmd=2)
+        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
 
     def import_all(self, path):
         """import delta chat state from the specified backup `path` (a file).
@@ -425,21 +446,22 @@ class Account(object):
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=12)
+        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
 
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            self.imex(path, imex_cmd)
             imex_tracker.wait_finish()
 
+    def imex(self, path, imex_cmd):
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
-        if not self.is_started():
-            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -546,6 +568,9 @@ class Account(object):
         You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
         account is started.
 
+        If you are using this from a test, you may want to call
+        wait_all_initial_fetches() afterwards.
+
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
@@ -555,12 +580,34 @@ class Account(object):
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
-    def configure(self):
+    def maybe_network(self):
+        """This function should be called when there is a hint
+        that the network is available again,
+        e.g. as a response to system event reporting network availability.
+        The library will try to send pending messages out immediately.
+
+        Moreover, to have a reliable state
+        when the app comes to foreground with network available,
+        it may be reasonable to call the function also at that moment.
+
+        It is okay to call the function unconditionally when there is
+        network available, however, calling the function
+        _without_ having network may interfere with the backoff algorithm
+        and will led to let the jobs fail faster, with fewer retries
+        and may avoid messages being sent out.
+
+        Finally, if the context was created by the dc_accounts_t account manager
+        (currently not implemented in the Python bindings),
+        use dc_accounts_maybe_network() instead of this function
+        """
+        lib.dc_maybe_network(self._dc_context)
+
+    def configure(self, reconfigure=False):
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
         """
-        assert not self.is_configured()
+        assert self.is_configured() == reconfigure
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
         configtracker = ConfigureTracker(self)
@@ -568,9 +615,6 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def is_started(self):
-        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
-
     def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
@@ -580,11 +624,8 @@ class Account(object):
         self.log("stop_ongoing")
         self.stop_ongoing()
 
-        if bool(lib.dc_is_io_running(self._dc_context)):
-            self.log("dc_stop_io (stop core IO scheduler)")
-            lib.dc_stop_io(self._dc_context)
-        else:
-            self.log("stop_scheduler called on non-running context")
+        self.log("dc_stop_io (stop core IO scheduler)")
+        lib.dc_stop_io(self._dc_context)
 
     def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove

python/src/deltachat/chat.py

@@ -57,10 +57,7 @@ class Chat(object):
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
-        return lib.dc_chat_get_type(self._dc_chat) in (
-            const.DC_CHAT_TYPE_GROUP,
-            const.DC_CHAT_TYPE_VERIFIED_GROUP
-        )
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
@@ -85,12 +82,20 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def is_verified(self):
-        """ return True if this chat is a verified group.
+    def can_send(self):
+        """Check if messages can be sent to a give chat.
+        This is not true eg. for the deaddrop or for the device-talk
 
-        :returns: True if chat is verified, False otherwise.
+        :returns: True if the chat is writable, False otherwise
         """
-        return lib.dc_chat_is_verified(self._dc_chat)
+        return lib.dc_chat_can_send(self._dc_chat)
+
+    def is_protected(self):
+        """ return True if this chat is a protected chat.
+
+        :returns: True if chat is protected, False otherwise.
+        """
+        return lib.dc_chat_is_protected(self._dc_chat)
 
     def get_name(self):
         """ return name of this chat.
@@ -162,6 +167,13 @@ class Chat(object):
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
+    def get_encryption_info(self):
+        """Return encryption info for this chat.
+
+        :returns: a string with encryption preferences of all chat members"""
+        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
+        return from_dc_charpointer(res)
+
     def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
@@ -366,7 +378,7 @@ class Chat(object):
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.create_contact(obj)
+        contact = self.account.get_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
@@ -490,18 +502,23 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
+                timestamp=datetime.utcfromtimestamp(
+                    lib.dc_array_get_timestamp(dc_array, i)
+                ),
+                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+            )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp):
+    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
         self.timestamp = timestamp
+        self.marker = marker
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__

python/src/deltachat/contact.py

@@ -52,9 +52,17 @@ class Contact(object):
         return lib.dc_contact_is_blocked(self._dc_contact)
 
     def set_blocked(self, block=True):
-        """ Block or unblock a contact. """
+        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
         return lib.dc_block_contact(self.account._dc_context, self.id, block)
 
+    def block(self):
+        """ Block this contact. Message will not be seen/retrieved from this contact. """
+        return lib.dc_block_contact(self.account._dc_context, self.id, True)
+
+    def unblock(self):
+        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
+        return lib.dc_block_contact(self.account._dc_context, self.id, False)
+
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
@@ -69,6 +77,14 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
+    @property
+    def status(self):
+        """Get contact status.
+
+        :returns: contact status, empty string if it doesn't exist.
+        """
+        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
+
     def create_chat(self):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 

python/src/deltachat/direct_imap.py

@@ -9,7 +9,9 @@ import ssl
 import pathlib
 from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
+import imaplib
 import deltachat
+from deltachat import const
 
 
 SEEN = b'\\Seen'
@@ -24,12 +26,29 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    imap = DirectImap(account)
-    if imap.select_config_folder("mvbox"):
-        imap.delete(ALL, expunge=True)
-    assert imap.select_config_folder("inbox")
-    imap.delete(ALL, expunge=True)
-    setattr(account, "direct_imap", imap)
+    try:
+
+        if not hasattr(account, "direct_imap"):
+            imap = DirectImap(account)
+
+            for folder in imap.list_folders():
+                if folder.lower() == "inbox" or folder.lower() == "deltachat":
+                    assert imap.select_folder(folder)
+                    imap.delete(ALL, expunge=True)
+                else:
+                    imap.conn.delete_folder(folder)
+                    # We just deleted the folder, so we have to make DC forget about it, too
+                    if account.get_config("configured_sentbox_folder") == folder:
+                        account.set_config("configured_sentbox_folder", None)
+                    if account.get_config("configured_spam_folder") == folder:
+                        account.set_config("configured_spam_folder", None)
+
+            setattr(account, "direct_imap", imap)
+
+    except Exception as e:
+        # Uncaught exceptions here would lead to a timeout without any note written to the log
+        account.log("=============================== CAN'T RESET ACCOUNT: ===============================")
+        account.log("===================", e, "===================")
 
 
 @deltachat.global_hookimpl
@@ -49,18 +68,31 @@ class DirectImap:
         self.connect()
 
     def connect(self):
-        ssl_context = ssl.create_default_context()
-
-        # don't check if certificate hostname doesn't match target hostname
-        ssl_context.check_hostname = False
-
-        # don't check if the certificate is trusted by a certificate authority
-        ssl_context.verify_mode = ssl.CERT_NONE
-
         host = self.account.get_config("configured_mail_server")
+        port = int(self.account.get_config("configured_mail_port"))
+        security = int(self.account.get_config("configured_mail_security"))
+
         user = self.account.get_config("addr")
         pw = self.account.get_config("mail_pw")
-        self.conn = IMAPClient(host, ssl_context=ssl_context)
+
+        if security == const.DC_SOCKET_PLAIN:
+            ssl_context = None
+        else:
+            ssl_context = ssl.create_default_context()
+
+            # don't check if certificate hostname doesn't match target hostname
+            ssl_context.check_hostname = False
+
+            # don't check if the certificate is trusted by a certificate authority
+            ssl_context.verify_mode = ssl.CERT_NONE
+
+        if security == const.DC_SOCKET_STARTTLS:
+            self.conn = IMAPClient(host, port, ssl=False)
+            self.conn.starttls(ssl_context)
+        elif security == const.DC_SOCKET_PLAIN:
+            self.conn = IMAPClient(host, port, ssl=False)
+        elif security == const.DC_SOCKET_SSL:
+            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
@@ -75,6 +107,12 @@ class DirectImap:
         except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
+    def create_folder(self, foldername):
+        try:
+            self.conn.create_folder(foldername)
+        except imaplib.IMAP4.error as e:
+            print("Can't create", foldername, "probably it already exists:", str(e))
+
     def select_folder(self, foldername):
         assert not self._idling
         return self.conn.select_folder(foldername)
@@ -225,3 +263,9 @@ class DirectImap:
             res = self.conn.idle_done()
             self._idling = False
             return res
+
+    def append(self, folder, msg):
+        if msg.startswith("\n"):
+            msg = msg[1:]
+        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
+        self.conn.append(folder, msg)

python/src/deltachat/events.py

@@ -1,6 +1,7 @@
 import threading
 import time
 import re
+import os
 from queue import Queue, Empty
 
 import deltachat
@@ -48,6 +49,15 @@ class FFIEventLogger:
         if self.logid:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+
+        if os.name == "posix":
+            WARN = '\033[93m'
+            ERROR = '\033[91m'
+            ENDC = '\033[0m'
+            if message.startswith("DC_EVENT_WARNING"):
+                s = WARN + s + ENDC
+            if message.startswith("DC_EVENT_ERROR"):
+                s = ERROR + s + ENDC
         with self._loglock:
             print(s, flush=True)
 
@@ -93,6 +103,14 @@ class FFIEventTracker:
             if rex.search(ev.data2):
                 return ev
 
+    def get_info_regex_groups(self, regex, check_error=True):
+        rex = re.compile(regex)
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
+            m = rex.match(ev.data2)
+            if m is not None:
+                return m.groups()
+
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))
@@ -111,6 +129,15 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
+    def wait_all_initial_fetches(self):
+        """Has to be called after start_io() to wait for fetch_existing_msgs to run
+        so that new messages are not mistaken for old ones:
+        - ac1 and ac2 are created
+        - ac1 sends a message to ac2
+        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
+        - therefore no DC_EVENT_INCOMING_MSG is sent"""
+        self.get_info_contains("Done fetching existing messages")
+
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")

python/src/deltachat/message.py

@@ -175,6 +175,27 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
+    @property
+    def quoted_text(self):
+        """Text inside the quote
+
+        :returns: Quoted text"""
+        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
+
+    @property
+    def quote(self):
+        """Quote getter
+
+        :returns: Quoted message, if found in the database"""
+        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
+        if msg:
+            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
+
+    @quote.setter
+    def quote(self, quoted_message):
+        """Quote setter"""
+        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -191,6 +212,11 @@ class Message(object):
                 return email.message_from_bytes(s)
             return email.message_from_string(s)
 
+    @property
+    def error(self):
+        """Error message"""
+        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
+
     @property
     def chat(self):
         """chat this message was posted in.

python/src/deltachat/testplugin.py

@@ -312,24 +312,31 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io([ac1])
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
             ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io([ac1, ac2])
             return ac1, ac2
 
         def get_many_online_accounts(self, num, move=True):
             accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io(accounts)
             for acc in accounts:
                 acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
+            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
+            direct_imap object of an online account. This simulates the user setting
+            up a new device without importing a backup.
+
+            `pre_generated_key` only means that a key from python/tests/data/key is
+            used in order to speed things up.
+            """
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
@@ -342,19 +349,35 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
+            if hasattr(account, "direct_imap"):
+                # Attach the existing direct_imap. If we did not do this, a new one would be created and
+                # delete existing messages (see dc_account_extra_configure(configure))
+                ac.direct_imap = account.direct_imap
             ac._configtracker = ac.configure()
             return ac
 
-        def wait_configure_and_start_io(self):
-            for acc in self._accounts:
-                if hasattr(acc, "_configtracker"):
-                    acc._configtracker.wait_finish()
-                    del acc._configtracker
-                acc.set_config("bcc_self", "0")
-                if acc.is_configured() and not acc.is_started():
-                    acc.start_io()
-                print("{}: {} account was successfully setup".format(
-                    acc.get_config("displayname"), acc.get_config("addr")))
+        def wait_configure_and_start_io(self, accounts=None):
+            if accounts is None:
+                accounts = self._accounts[:]
+            started_accounts = []
+            for acc in accounts:
+                if acc not in started_accounts:
+                    self.wait_configure(acc)
+                    acc.set_config("bcc_self", "0")
+                    if acc.is_configured():
+                        acc.start_io()
+                        started_accounts.append(acc)
+                    print("{}: {} account was started".format(
+                        acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in started_accounts:
+                acc._evtracker.wait_all_initial_fetches()
+
+        def wait_configure(self, acc):
+            if hasattr(acc, "_configtracker"):
+                acc._configtracker.wait_finish()
+                acc._evtracker.consume_events()
+                acc.get_device_chat().mark_noticed()
+                del acc._configtracker
 
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
@@ -493,6 +516,9 @@ def lp():
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
 
+        def indent(self, msg):
+            print("  " + msg)
+
     return Printer()
 
 

python/src/deltachat/tracker.py

@@ -20,6 +20,16 @@ class ImexTracker:
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
             self._imex_events.put(ffi_event.data2)
 
+    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, int) and ev >= target_progress:
+                assert ev <= progress_upper_limit, \
+                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
+                return ev
+            if ev == 0:
+                return None
+
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
         files_written = []

python/tests/test_account.py

@@ -6,7 +6,10 @@ import queue
 import time
 from deltachat import const, Account
 from deltachat.message import Message
+from deltachat.tracker import ImexTracker
 from deltachat.hookspec import account_hookimpl
+from deltachat.capi import ffi, lib
+from deltachat.cutil import iter_array
 from datetime import datetime, timedelta
 
 
@@ -129,6 +132,20 @@ class TestOfflineContact:
         assert not contact1.is_blocked()
         assert not contact1.is_verified()
 
+    def test_get_blocked(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact1 = ac1.create_contact("some1@example.org", name="some1")
+        contact2 = ac1.create_contact("some2@example.org", name="some2")
+        ac1.create_contact("some3@example.org", name="some3")
+        assert ac1.get_blocked_contacts() == []
+        contact1.block()
+        assert ac1.get_blocked_contacts() == [contact1]
+        contact2.block()
+        blocked = ac1.get_blocked_contacts()
+        assert len(blocked) == 2 and contact1 in blocked and contact2 in blocked
+        contact2.unblock()
+        assert ac1.get_blocked_contacts() == [contact1]
+
     def test_create_self_contact(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact(ac1.get_config("addr"))
@@ -166,6 +183,16 @@ class TestOfflineContact:
         with pytest.raises(ValueError):
             ac1.create_chat(ac3)
 
+    def test_contact_rename(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact = ac1.create_contact("some1@example.com", name="some1")
+        chat = ac1.create_chat(contact)
+        assert chat.get_name() == "some1"
+        ac1.create_contact("some1@example.com", name="renamed")
+        ev = ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED")
+        assert ev.data1 == chat.id
+        assert chat.get_name() == "renamed"
+
 
 class TestOfflineChat:
     @pytest.fixture
@@ -264,6 +291,28 @@ class TestOfflineChat:
         qr = chat.get_join_qr()
         assert ac2.check_qr(qr).is_ask_verifygroup
 
+    def test_removing_blocked_user_from_group(self, ac1, lp):
+        """
+        Test that blocked contact is not unblocked when removed from a group.
+        See https://github.com/deltachat/deltachat-core-rust/issues/2030
+        """
+        lp.sec("Create a group chat with a contact")
+        contact = ac1.create_contact("some1@example.org")
+        group = ac1.create_group_chat("title", contacts=[contact])
+        group.send_text("First group message")
+
+        lp.sec("ac1 blocks contact")
+        contact.block()
+        assert contact.is_blocked()
+
+        lp.sec("ac1 removes contact from their group")
+        group.remove_contact(contact)
+        assert contact.is_blocked()
+
+        lp.sec("ac1 adding blocked contact unblocks it")
+        group.add_contact(contact)
+        assert not contact.is_blocked()
+
     def test_get_set_profile_image_simple(self, ac1, data):
         chat = ac1.create_group_chat(name="title1")
         p = data.get_path("d.png")
@@ -386,11 +435,11 @@ class TestOfflineChat:
         email = "hello <hello@example.org>"
         contact1 = ac1.create_contact(email)
         assert contact1.addr == "hello@example.org"
-        assert contact1.display_name == "hello"
+        assert contact1.name == "hello"
         contact1 = ac1.create_contact(email, name="world")
-        assert contact1.display_name == "world"
+        assert contact1.name == "world"
         contact2 = ac1.create_contact("display1 <x@example.org>", "real")
-        assert contact2.display_name == "real"
+        assert contact2.name == "real"
 
     def test_create_chat_simple(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
@@ -427,6 +476,7 @@ class TestOfflineChat:
         contact = msg.get_sender_contact()
         assert contact == ac1.get_self_contact()
         assert not backupdir.listdir()
+        ac1.stop_io()
         path = ac1.export_all(backupdir.strpath)
         assert os.path.exists(path)
         ac2 = acfactory.get_unconfigured_account()
@@ -441,10 +491,6 @@ class TestOfflineChat:
         assert messages[0].text == "msg1"
         assert os.path.exists(messages[1].filename)
 
-    def test_ac_setup_message_fails(self, ac1):
-        with pytest.raises(RuntimeError):
-            ac1.initiate_key_transfer()
-
     def test_set_get_draft(self, chat1):
         msg = Message.new_empty(chat1.account, "text")
         msg1 = chat1.prepare_message(msg)
@@ -466,6 +512,21 @@ class TestOfflineChat:
         assert not res.is_ask_verifygroup()
         assert res.contact_id == 10
 
+    def test_quote(self, chat1):
+        """Offline quoting test"""
+        msg = Message.new_empty(chat1.account, "text")
+        msg.set_text("Multi\nline\nmessage")
+        assert msg.quoted_text is None
+
+        # Prepare message to assign it a Message-Id.
+        # Messages without Message-Id cannot be quoted.
+        msg = chat1.prepare_message(msg)
+
+        reply_msg = Message.new_empty(chat1.account, "text")
+        reply_msg.set_text("reply")
+        reply_msg.quote = msg
+        assert reply_msg.quoted_text == "Multi\nline\nmessage"
+
     def test_group_chat_many_members_add_remove(self, ac1, lp):
         lp.sec("ac1: creating group chat with 10 other members")
         chat = ac1.create_group_chat(name="title1")
@@ -537,6 +598,28 @@ class TestOfflineChat:
         assert in_list[1][1] == chat
         assert in_list[1][2] == contacts[3]
 
+    def test_audit_log_view_without_daymarker(self, ac1, lp):
+        lp.sec("ac1: test audit log (show only system messages)")
+        chat = ac1.create_group_chat(name="audit log sample data")
+        # promote chat
+        chat.send_text("hello")
+        assert chat.is_promoted()
+
+        lp.sec("create test data")
+        chat.add_contact(ac1.create_contact("some-1@example.org"))
+        chat.set_name("audit log test group")
+        chat.send_text("a message in between")
+
+        lp.sec("check message count of all messages")
+        assert len(chat.get_messages()) == 4
+
+        lp.sec("check message count of only system messages (without daymarkers)")
+        dc_array = ffi.gc(
+            lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_INFO_ONLY, 0),
+            lib.dc_array_unref
+        )
+        assert len(list(iter_array(dc_array, lambda x: x))) == 2
+
 
 def test_basic_imap_api(acfactory, tmpdir):
     ac1, ac2 = acfactory.get_two_online_accounts()
@@ -570,7 +653,7 @@ class TestOnlineAccount:
             config={"key_gen_type": str(const.DC_KEY_GEN_ED25519)}
         )
         # rsa key gen can be slow especially on CI, adjust timeout
-        ac1._evtracker.set_timeout(120)
+        ac1._evtracker.set_timeout(240)
         acfactory.wait_configure_and_start_io()
         chat = acfactory.get_accepted_chat(ac1, ac2)
 
@@ -607,7 +690,7 @@ class TestOnlineAccount:
         except Exception:
             pass
 
-    def test_export_import_self_keys(self, acfactory, tmpdir):
+    def test_export_import_self_keys(self, acfactory, tmpdir, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         dir = tmpdir.mkdir("exportdir")
@@ -615,8 +698,17 @@ class TestOnlineAccount:
         assert len(export_files) == 2
         for x in export_files:
             assert x.startswith(dir.strpath)
+        key_id, = ac1._evtracker.get_info_regex_groups(r".*xporting.*KeyId\((.*)\).*")
         ac1._evtracker.consume_events()
+
+        lp.sec("exported keys (private and public)")
+        for name in os.listdir(dir.strpath):
+            lp.indent(dir.strpath + os.sep + name)
+        lp.sec("importing into existing account")
         ac2.import_self_keys(dir.strpath)
+        key_id2, = ac2._evtracker.get_info_regex_groups(
+            r".*stored.*KeyId\((.*)\).*", check_error=False)
+        assert key_id2 == key_id
 
     def test_one_account_send_bcc_setting(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
@@ -872,7 +964,13 @@ class TestOnlineAccount:
         lp.sec("mark messages as seen on ac2, wait for changes on ac1")
         ac2.direct_imap.idle_start()
         ac1.direct_imap.idle_start()
+
         ac2.mark_seen_messages([msg2, msg4])
+        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+        assert msg2.chat.id == msg4.chat.id
+        assert ev.data1 == msg2.chat.id
+        assert ev.data2 == 0
+
         ac2.direct_imap.idle_check(terminate=True)
         lp.step("1")
         for i in range(2):
@@ -893,6 +991,54 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
+    @pytest.mark.parametrize("mvbox_move", [True, False])
+    def test_markseen_message_and_mdn(self, acfactory, mvbox_move):
+        # Please only change this test if you are very sure that it will still catch the issues it catches now.
+        # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
+        ac1 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        acfactory.wait_configure_and_start_io()
+
+        acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
+        msg = ac2._evtracker.wait_next_incoming_message()
+
+        folder = "mvbox" if mvbox_move else "inbox"
+        ac1.direct_imap.select_config_folder(folder)
+        ac2.direct_imap.select_config_folder(folder)
+        ac1.direct_imap.idle_start()
+        ac2.direct_imap.idle_start()
+
+        ac2.mark_seen_messages([msg])
+
+        ac1.direct_imap.idle_wait_for_seen()  # Check that the mdn is marked as seen
+        ac2.direct_imap.idle_wait_for_seen()  # Check that the original message is marked as seen
+        ac1.direct_imap.idle_done()
+        ac2.direct_imap.idle_done()
+
+    def test_reply_privately(self, acfactory):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        group1 = ac1.create_group_chat("group")
+        group1.add_contact(ac2)
+        group1.send_text("hello")
+
+        msg2 = ac2._evtracker.wait_next_messages_changed()
+        group2 = msg2.create_chat()
+        assert group2.get_name() == group1.get_name()
+
+        msg_reply = Message.new_empty(ac2, "text")
+        msg_reply.set_text("message reply")
+        msg_reply.quote = msg2
+
+        private_chat1 = ac1.create_chat(ac2)
+        private_chat2 = ac2.create_chat(ac1)
+        private_chat2.send_msg(msg_reply)
+
+        msg_reply1 = ac1._evtracker.wait_next_incoming_message()
+        assert msg_reply1.quoted_text == "hello"
+        assert not msg_reply1.chat.is_group()
+        assert msg_reply1.chat.id == private_chat1.id
+
     def test_mdn_asymetric(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts(move=True)
 
@@ -972,6 +1118,64 @@ class TestOnlineAccount:
         assert not msg.is_encrypted()
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    def test_gossip_encryption_preference(self, acfactory, lp):
+        """Test that encryption preference of group members is gossiped to new members.
+        This is a Delta Chat extension to Autocrypt 1.1.0, which Autocrypt-Gossip headers
+        SHOULD NOT contain encryption preference.
+        """
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+
+        lp.sec("ac1 learns that ac2 prefers encryption")
+        ac1.create_chat(ac2)
+        msg = ac2.create_chat(ac1).send_text("first message")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "first message"
+        assert not msg.is_encrypted()
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config('addr'))
+        assert msg.chat.get_encryption_info() == res
+        lp.sec("ac2 learns that ac3 prefers encryption")
+        ac2.create_chat(ac3)
+        msg = ac3.create_chat(ac2).send_text("I prefer encryption")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "I prefer encryption"
+        assert not msg.is_encrypted()
+
+        lp.sec("ac3 does not know that ac1 prefers encryption")
+        ac1.create_chat(ac3)
+        chat = ac3.create_chat(ac1)
+        res = "{} No encryption.".format(ac1.get_config('addr'))
+        assert chat.get_encryption_info() == res
+        msg = chat.send_text("not encrypted")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "not encrypted"
+        assert not msg.is_encrypted()
+
+        lp.sec("ac1 creates a group chat with ac2")
+        group_chat = ac1.create_group_chat("hello")
+        group_chat.add_contact(ac2)
+        encryption_info = group_chat.get_encryption_info()
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config("addr"))
+        assert encryption_info == res
+        msg = group_chat.send_text("hi")
+
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+        assert msg.text == "hi"
+
+        lp.sec("ac2 adds ac3 to the group")
+        msg.chat.add_contact(ac3)
+        assert msg.is_encrypted()
+
+        lp.sec("ac3 learns that ac1 prefers encryption")
+        msg = ac3._evtracker.wait_next_incoming_message()
+        encryption_info = msg.chat.get_encryption_info().splitlines()
+        res = "{} End-to-end encryption preferred.".format(ac1.get_config("addr"))
+        assert res in encryption_info
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config("addr"))
+        assert res in encryption_info
+        msg = chat.send_text("encrypted")
+        assert msg.is_encrypted()
+
     def test_send_first_message_as_long_unicode_with_cr(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac2.set_config("save_mime_headers", "1")
@@ -1001,7 +1205,114 @@ class TestOnlineAccount:
         assert msg_in.text == text2
         assert ac1.get_config("addr") in [x.addr for x in msg_in.chat.get_contacts()]
 
-    def test_reply_encrypted(self, acfactory, lp):
+    def test_no_draft_if_cant_send(self, acfactory):
+        """Tests that no quote can be set if the user can't send to this chat"""
+        ac1 = acfactory.get_one_online_account()
+        device_chat = ac1.get_device_chat()
+        msg = Message.new_empty(ac1, "text")
+        device_chat.set_draft(msg)
+
+        assert not device_chat.can_send()
+        assert device_chat.get_draft() is None
+
+    def test_dont_show_emails_in_draft_folder(self, acfactory):
+        """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them.
+        So: If it's outgoing AND there is no Received header AND it's not in the sentbox, then ignore the email."""
+        ac1 = acfactory.get_online_configuring_account()
+        ac1.set_config("show_emails", "2")
+        ac1.create_contact("alice@example.com").create_chat()
+
+        acfactory.wait_configure(ac1)
+        ac1.direct_imap.create_folder("Drafts")
+        ac1.direct_imap.create_folder("Sent")
+
+        acfactory.wait_configure_and_start_io()
+        # Wait until each folder was selected once and we are IDLEing again:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.stop_io()
+
+        ac1.direct_imap.append("Drafts", """
+            From: ac1 <{}>
+            Subject: subj
+            To: alice@example.com
+            Message-ID: <aepiors@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            message in Drafts
+        """.format(ac1.get_config("configured_addr")))
+        ac1.direct_imap.append("Sent", """
+            From: ac1 <{}>
+            Subject: subj
+            To: alice@example.com
+            Message-ID: <hsabaeni@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            message in Sent
+        """.format(ac1.get_config("configured_addr")))
+
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+        ac1.start_io()
+
+        msg = ac1._evtracker.wait_next_messages_changed()
+
+        # Wait until each folder was scanned, this is necessary for this test to test what it should test:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+
+        assert msg.text == "subj – message in Sent"
+        assert len(msg.chat.get_messages()) == 1
+
+    def test_prefer_encrypt(self, acfactory, lp):
+        """Test quorum rule for encryption preference in 1:1 and group chat."""
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+        ac1.set_config("e2ee_enabled", "0")
+        ac2.set_config("e2ee_enabled", "1")
+        ac3.set_config("e2ee_enabled", "0")
+
+        # Make sure we do not send a copy to ourselves. This is to
+        # test that we count own preference even when we are not in
+        # the recipient list.
+        ac1.set_config("bcc_self", "0")
+        ac2.set_config("bcc_self", "0")
+        ac3.set_config("bcc_self", "0")
+
+        acfactory.introduce_each_other([ac1, ac2, ac3])
+
+        lp.sec("ac1: sending message to ac2")
+        chat1 = ac1.create_chat(ac2)
+        msg1 = chat1.send_text("message1")
+        assert not msg1.is_encrypted()
+        ac2._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac2: sending message to ac1")
+        chat2 = ac2.create_chat(ac1)
+        msg2 = chat2.send_text("message2")
+        assert not msg2.is_encrypted()
+        ac1._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac1: sending message to group chat with ac2 and ac3")
+        group = ac1.create_group_chat("hello")
+        group.add_contact(ac2)
+        group.add_contact(ac3)
+        msg3 = group.send_text("message3")
+        assert not msg3.is_encrypted()
+        ac2._evtracker.wait_next_incoming_message()
+        ac3._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac3: start preferring encryption and inform ac1")
+        ac3.set_config("e2ee_enabled", "1")
+        chat3 = ac3.create_chat(ac1)
+        msg4 = chat3.send_text("message4")
+        # ac1 still does not prefer encryption
+        assert not msg4.is_encrypted()
+        ac1._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac1: sending another message to group chat with ac2 and ac3")
+        msg5 = group.send_text("message5")
+        # Majority prefers encryption now
+        assert msg5.is_encrypted()
+
+    def test_quote_encrypted(self, acfactory, lp):
+        """Test that replies to encrypted messages with quotes are encrypted."""
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         lp.sec("ac1: create chat with ac2")
@@ -1029,26 +1340,59 @@ class TestOnlineAccount:
         print("ac2: e2ee_enabled={}".format(ac2.get_config("e2ee_enabled")))
         ac1.set_config("e2ee_enabled", "0")
 
-        # Set unprepared and unencrypted draft to test that it is not
-        # taken into account when determining whether last message is
-        # encrypted.
-        msg_draft = Message.new_empty(ac1, "text")
-        msg_draft.set_text("message2 -- should be encrypted")
-        chat.set_draft(msg_draft)
+        for quoted_msg in msg1, msg3:
+            # Save the draft with a quote.
+            # It should be encrypted if quoted message is encrypted.
+            msg_draft = Message.new_empty(ac1, "text")
+            msg_draft.set_text("message reply")
+            msg_draft.quote = quoted_msg
+            chat.set_draft(msg_draft)
 
-        # Get the draft, prepare and send it.
-        msg_draft = chat.get_draft()
-        msg_out = chat.prepare_message(msg_draft)
-        chat.send_prepared(msg_out)
+            # Get the draft, prepare and send it.
+            msg_draft = chat.get_draft()
+            msg_out = chat.prepare_message(msg_draft)
+            chat.send_prepared(msg_out)
 
-        chat.set_draft(None)
-        assert chat.get_draft() is None
+            chat.set_draft(None)
+            assert chat.get_draft() is None
 
-        lp.sec("wait for ac2 to receive message")
-        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
-        msg_in = ac2.get_message_by_id(ev.data2)
-        assert msg_in.text == "message2 -- should be encrypted"
-        assert msg_in.is_encrypted()
+            msg_in = ac2._evtracker.wait_next_incoming_message()
+            assert msg_in.text == "message reply"
+            assert msg_in.quoted_text == quoted_msg.text
+            assert msg_in.is_encrypted() == quoted_msg.is_encrypted()
+
+    def test_quote_attachment(self, tmpdir, acfactory, lp):
+        """Test that replies with an attachment and a quote are received correctly."""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1 creates chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+
+        lp.sec("ac1 sends text message to ac2")
+        chat1.send_text("hi")
+
+        lp.sec("ac2 receives contact request from ac1")
+        received_message = ac2._evtracker.wait_next_messages_changed()
+        assert received_message.text == "hi"
+
+        basename = "attachment.txt"
+        p = os.path.join(tmpdir.strpath, basename)
+        with open(p, "w") as f:
+            f.write("data to send")
+
+        lp.sec("ac2 sends a reply to ac1")
+        chat2 = received_message.create_chat()
+        reply = Message.new_empty(ac2, "file")
+        reply.set_text("message reply")
+        reply.set_file(p)
+        reply.quote = received_message
+        chat2.send_msg(reply)
+
+        lp.sec("ac1 receives a reply from ac2")
+        received_reply = ac1._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "message reply"
+        assert received_reply.quoted_text == received_message.text
+        assert open(received_reply.filename).read() == "data to send"
 
     def test_saved_mime_on_received_message(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1143,19 +1487,52 @@ class TestOnlineAccount:
         m = message_queue.get()
         assert m == msg_in
 
-    def test_import_export_online_all(self, acfactory, tmpdir, lp):
+    def test_import_export_online_all(self, acfactory, tmpdir, data, lp):
         ac1 = acfactory.get_one_online_account()
 
         lp.sec("create some chat content")
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact1.create_chat().send_text("msg1")
+        chat1 = ac1.create_contact("some1@example.org", name="some1").create_chat()
+        chat1.send_text("msg1")
         assert len(ac1.get_contacts(query="some1")) == 1
+
+        original_image_path = data.get_path("d.png")
+        chat1.send_image(original_image_path)
+
+        def assert_account_is_proper(ac):
+            contacts = ac.get_contacts(query="some1")
+            assert len(contacts) == 1
+            contact2 = contacts[0]
+            assert contact2.addr == "some1@example.org"
+            chat2 = contact2.create_chat()
+            messages = chat2.get_messages()
+            assert len(messages) == 2
+            assert messages[0].text == "msg1"
+            assert messages[1].filemime == "image/png"
+            assert os.stat(messages[1].filename).st_size == os.stat(original_image_path).st_size
+            ac.set_config("displayname", "new displayname")
+            assert ac.get_config("displayname") == "new displayname"
+
+        assert_account_is_proper(ac1)
+
         backupdir = tmpdir.mkdir("backup")
 
         lp.sec("export all to {}".format(backupdir))
-        path = ac1.export_all(backupdir.strpath)
-        assert os.path.exists(path)
-        t = time.time()
+        with ac1.temp_plugin(ImexTracker()) as imex_tracker:
+
+            ac1.stop_io()
+            ac1.imex(backupdir.strpath, const.DC_IMEX_EXPORT_BACKUP)
+
+            # check progress events for export
+            assert imex_tracker.wait_progress(1, progress_upper_limit=249)
+            assert imex_tracker.wait_progress(250, progress_upper_limit=499)
+            assert imex_tracker.wait_progress(500, progress_upper_limit=749)
+            assert imex_tracker.wait_progress(750, progress_upper_limit=999)
+
+            paths = imex_tracker.wait_finish()
+            assert len(paths) == 1
+            path = paths[0]
+            assert os.path.exists(path)
+            ac1.start_io()
 
         lp.sec("get fresh empty account")
         ac2 = acfactory.get_unconfigured_account()
@@ -1165,22 +1542,20 @@ class TestOnlineAccount:
         assert path2 == path
 
         lp.sec("import backup and check it's proper")
-        ac2.import_all(path)
-        contacts = ac2.get_contacts(query="some1")
-        assert len(contacts) == 1
-        contact2 = contacts[0]
-        assert contact2.addr == "some1@example.org"
-        chat2 = contact2.create_chat()
-        messages = chat2.get_messages()
-        assert len(messages) == 1
-        assert messages[0].text == "msg1"
+        with ac2.temp_plugin(ImexTracker()) as imex_tracker:
+            ac2.import_all(path)
+
+            # check progress events for import
+            assert imex_tracker.wait_progress(1, progress_upper_limit=249)
+            assert imex_tracker.wait_progress(500, progress_upper_limit=749)
+            assert imex_tracker.wait_progress(750, progress_upper_limit=999)
+            assert imex_tracker.wait_progress(1000)
+
+        assert_account_is_proper(ac1)
+        assert_account_is_proper(ac2)
 
-        # wait until a second passed since last backup
-        # because get_latest_backupfile() shall return the latest backup
-        # from a UI it's unlikely anyone manages to export two
-        # backups in one second.
-        time.sleep(max(0, 1 - (time.time() - t)))
         lp.sec("Second-time export all to {}".format(backupdir))
+        ac1.stop_io()
         path2 = ac1.export_all(backupdir.strpath)
         assert os.path.exists(path2)
         assert path2 != path
@@ -1261,7 +1636,7 @@ class TestOnlineAccount:
         ac1, ac2 = acfactory.get_two_online_accounts()
         lp.sec("ac1: create verified-group QR, ac2 scans and joins")
         chat1 = ac1.create_group_chat("hello", verified=True)
-        assert chat1.is_verified()
+        assert chat1.is_protected()
         qr = chat1.get_join_qr()
         lp.sec("ac2: start QR-code based join-group protocol")
         chat2 = ac2.qr_join_chat(qr)
@@ -1280,7 +1655,7 @@ class TestOnlineAccount:
         lp.sec("ac2: read message and check it's verified chat")
         msg = ac2._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
-        assert msg.chat.is_verified()
+        assert msg.chat.is_protected()
         assert msg.is_encrypted()
 
         lp.sec("ac2: send message and let ac1 read it")
@@ -1415,8 +1790,10 @@ class TestOnlineAccount:
 
         lp.sec("ac1 blocks ac2")
         contact = ac1.create_contact(ac2)
-        contact.set_blocked()
+        contact.block()
         assert contact.is_blocked()
+        ev = ac1._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        assert ev.data1 == contact.id
 
         lp.sec("ac2 sends a message to ac1 that does not arrive because it is blocked")
         ac2.create_chat(ac1).send_text("This will not arrive!")
@@ -1512,7 +1889,7 @@ class TestOnlineAccount:
 
         ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5)
         ac1._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED")
-        chat1.send_text("hello")
+        chat1.send_text("🍞")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
         lp.sec("ac2: wait for incoming location message")
@@ -1526,6 +1903,7 @@ class TestOnlineAccount:
         assert locations[0].longitude == 3.0
         assert locations[0].accuracy == 0.5
         assert locations[0].timestamp > now
+        assert locations[0].marker == "🍞"
 
         contact = ac2.create_contact(ac1)
         locations2 = chat2.get_locations(contact=contact)
@@ -1560,6 +1938,7 @@ class TestOnlineAccount:
         chat41 = ac4.create_chat(ac1)
         chat42 = ac4.create_chat(ac2)
         ac4.start_io()
+        ac4._evtracker.wait_all_initial_fetches()
 
         lp.sec("ac1: creating group chat with 2 other members")
         chat = ac1.create_group_chat("title", contacts=[ac2, ac3])
@@ -1576,6 +1955,7 @@ class TestOnlineAccount:
         # Error message should be assigned to the chat with ac1.
         lp.sec("ac4: checking that message is assigned to the sender chat")
         error_msg = ac4._evtracker.wait_next_incoming_message()
+        assert error_msg.error  # There is an error decrypting the message
         assert error_msg.chat == chat41
 
         lp.sec("ac2: sending a reply to the chat")
@@ -1586,6 +1966,7 @@ class TestOnlineAccount:
 
         lp.sec("ac4: checking that reply is assigned to ac2 chat")
         error_reply = ac4._evtracker.wait_next_incoming_message()
+        assert error_reply.error  # There is an error decrypting the message
         assert error_reply.chat == chat42
 
         # Test that ac4 replies to error messages don't appear in the
@@ -1597,11 +1978,13 @@ class TestOnlineAccount:
         chat42.send_text("I can't decrypt your message, ac2!")
 
         msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.error is None
         assert msg.text == "I can't decrypt your message, ac1!"
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac1.create_chat(ac3)
 
         msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.error is None
         assert msg.text == "I can't decrypt your message, ac2!"
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac2.create_chat(ac4)
@@ -1727,11 +2110,44 @@ class TestOnlineAccount:
 
         assert len(imap2.get_all_messages()) == 1
 
+    def test_configure_error_msgs(self, acfactory):
+        ac1, configdict = acfactory.get_online_config()
+        ac1.update_config(configdict)
+        ac1.set_config("mail_pw", "abc")  # Wrong mail pw
+        ac1.configure()
+        while True:
+            ev = ac1._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+            if ev.data1 == 0:
+                break
+        # Password is wrong so it definitely has to say something about "password"
+        assert "password" in ev.data2
+
+        ac2, configdict = acfactory.get_online_config()
+        ac2.update_config(configdict)
+        ac2.set_config("addr", "abc@def.invalid")  # mail server can't be reached
+        ac2.configure()
+        while True:
+            ev = ac2._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+            if ev.data1 == 0:
+                break
+        # Can't connect so it probably should say something about "internet"
+        # again, should not repeat itself
+        # If this fails then probably `e.msg.to_lowercase().contains("could not resolve")`
+        # in configure/mod.rs returned false because the error message was changed
+        # (i.e. did not contain "could not resolve" anymore)
+        assert (ev.data2.count("internet") + ev.data2.count("network")) == 1
+        # Should mention that it can't connect:
+        assert ev.data2.count("connect") == 1
+        # The users do not know what "configuration" is
+        assert "configuration" not in ev.data2.lower()
+
     def test_name_changes(self, acfactory):
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac1.set_config("displayname", "Account 1")
 
-        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        # Similar to acfactory.get_accepted_chat, but without setting the contact name.
+        ac2.create_contact(ac1.get_config("addr")).create_chat()
+        chat12 = ac1.create_contact(ac2.get_config("addr")).create_chat()
         contact = None
 
         def update_name():
@@ -1750,6 +2166,8 @@ class TestOnlineAccount:
         # Explicitly rename contact on ac2 to "Renamed"
         ac2.create_contact(contact, name="Renamed")
         assert contact.name == "Renamed"
+        ev = ac2._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        assert ev.data1 == contact.id
 
         # ac1 also renames itself into "Renamed"
         assert update_name() == "Renamed"
@@ -1760,12 +2178,220 @@ class TestOnlineAccount:
         # so it should not be changed.
         ac1.set_config("displayname", "Renamed again")
         updated_name = update_name()
-        if updated_name == "Renamed again":
-            # Known bug, mark as XFAIL
-            pytest.xfail("Contact was renamed after explicit rename")
+        assert updated_name == "Renamed"
+
+    def test_status(self, acfactory):
+        """Test that status is transferred over the network."""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        ac1.set_config("selfstatus", "New status")
+        chat12.send_text("hi")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hi"
+        assert msg.get_sender_contact().status == "New status"
+
+        ac1.set_config("selfstatus", "")
+        chat12.send_text("hello")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+        assert msg.get_sender_contact().status == ""
+
+    def test_group_quote(self, acfactory, lp):
+        """Test quoting in a group with a new member who have not seen the quoted message."""
+        ac1, ac2, ac3 = accounts = acfactory.get_many_online_accounts(3)
+        acfactory.introduce_each_other(accounts)
+        chat = ac1.create_group_chat(name="quote group")
+        chat.add_contact(ac2)
+
+        lp.sec("ac1: sending message")
+        out_msg = chat.send_text("hello")
+
+        lp.sec("ac2: receiving message")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        chat.add_contact(ac3)
+        ac2._evtracker.wait_next_incoming_message()
+        ac3._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac2: sending reply with a quote")
+        reply_msg = Message.new_empty(msg.chat.account, "text")
+        reply_msg.set_text("reply")
+        reply_msg.quote = msg
+        reply_msg = msg.chat.prepare_message(reply_msg)
+        assert reply_msg.quoted_text == "hello"
+        msg.chat.send_prepared(reply_msg)
+
+        lp.sec("ac3: receiving reply")
+        received_reply = ac3._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "reply"
+        assert received_reply.quoted_text == "hello"
+        # ac3 was not in the group and has not received quoted message
+        assert received_reply.quote is None
+
+        lp.sec("ac1: receiving reply")
+        received_reply = ac1._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "reply"
+        assert received_reply.quoted_text == "hello"
+        assert received_reply.quote.id == out_msg.id
+
+    @pytest.mark.parametrize("folder,move,expected_destination,", [
+        ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
+        ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
+        ("Spam", False, "INBOX")  # ...emails are moved from the spam folder to the Inbox
+    ])
+    # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
+    # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
+    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination):
+        """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
+        variant = folder + "-" + str(move) + "-" + expected_destination
+        lp.sec("Testing variant " + variant)
+        ac1 = acfactory.get_online_configuring_account(move=move)
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure(ac1)
+        ac1.direct_imap.create_folder(folder)
+
+        acfactory.wait_configure_and_start_io()
+        # Wait until each folder was selected once and we are IDLEing:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.stop_io()
+
+        # Send a message to ac1 and move it to the mvbox:
+        ac1.direct_imap.select_config_folder("inbox")
+        ac1.direct_imap.idle_start()
+        acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
+
+        lp.sec("Everything prepared, now see if DeltaChat finds the message (" + variant + ")")
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+        ac1.start_io()
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        # Wait until the message was moved (if at all) and we are IDLEing again:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.direct_imap.select_folder(expected_destination)
+        assert len(ac1.direct_imap.get_all_messages()) == 1
+        if folder != expected_destination:
+            ac1.direct_imap.select_folder(folder)
+            assert len(ac1.direct_imap.get_all_messages()) == 0
+
+    @pytest.mark.parametrize("mvbox_move", [False, True])
+    def test_fetch_existing(self, acfactory, lp, mvbox_move):
+        """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
+        This way, we can already offer them some email addresses they can write to.
+
+        Also, the newest existing emails from each folder are fetched during onboarding.
+
+        Additionally tests that bcc_self messages moved to the mvbox/sentbox are marked as read."""
+
+        def assert_folders_configured(ac):
+            """There was a bug that scan_folders() set the configured folders to None under some circumstances.
+            So, check that they are still configured:"""
+            assert ac.get_config("configured_sentbox_folder") == "Sent"
+            if mvbox_move:
+                assert ac.get_config("configured_mvbox_folder")
+
+        ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
+        ac1.set_config("sentbox_move", "1")
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure(ac1)
+
+        ac1.direct_imap.create_folder("Sent")
+        ac1.set_config("sentbox_watch", "1")
+
+        # We need to reconfigure to find the new "Sent" folder.
+        # `scan_folders()`, which runs automatically shortly after `start_io()` is invoked,
+        # would also find the "Sent" folder, but it would be too late:
+        # The sentbox thread, started by `start_io()`, would have seen that there is no
+        # ConfiguredSentboxFolder and do nothing.
+        ac1._configtracker = ac1.configure(reconfigure=True)
+        acfactory.wait_configure_and_start_io()
+        assert_folders_configured(ac1)
+
+        if mvbox_move:
+            ac1.direct_imap.select_config_folder("mvbox")
         else:
-            # No renames should happen after explicit rename
-            assert updated_name == "Renamed"
+            ac1.direct_imap.select_config_folder("sentbox")
+        ac1.direct_imap.idle_start()
+
+        lp.sec("send out message with bcc to ourselves")
+        ac1.set_config("bcc_self", "1")
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+        chat.send_text("message text")
+        assert_folders_configured(ac1)
+
+        # now wait until the bcc_self message arrives
+        # Also test that bcc_self messages moved to the mvbox are marked as read.
+        assert ac1.direct_imap.idle_wait_for_seen()
+        assert_folders_configured(ac1)
+
+        ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone.set_config("fetch_existing_msgs", "1")
+        ac1_clone._configtracker.wait_finish()
+        ac1_clone.start_io()
+        assert_folders_configured(ac1_clone)
+
+        ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        ac2_addr = ac2.get_config("addr")
+        assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
+        assert_folders_configured(ac1_clone)
+
+        msg = ac1_clone._evtracker.wait_next_messages_changed()
+        assert msg.text == "message text"
+        assert_folders_configured(ac1)
+        assert_folders_configured(ac1_clone)
+
+    def test_fetch_existing_msgs_group_and_single(self, acfactory, lp):
+        """There was a bug concerning fetch-existing-msgs:
+
+        A sent a message to you, adding you to a group. This created a contact request.
+        You wrote a message to A, creating a chat.
+        ...but the group stayed blocked.
+        So, after fetch-existing-msgs you have one contact request and one chat with the same person.
+
+        See https://github.com/deltachat/deltachat-core-rust/issues/2097"""
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure_and_start_io()
+
+        lp.sec("receive a message")
+        ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
+        ac1._evtracker.wait_next_messages_changed()
+
+        lp.sec("send out message with bcc to ourselves")
+        ac1.direct_imap.idle_start()
+        ac1.set_config("bcc_self", "1")
+        ac1.create_chat(ac2).send_text("outgoing, encrypted direct message, creating a chat")
+
+        # now wait until the bcc_self message arrives
+        assert ac1.direct_imap.idle_wait_for_seen()
+
+        lp.sec("Clone online account and let it fetch the existing messages")
+        ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone.set_config("fetch_existing_msgs", "1")
+        ac1_clone._configtracker.wait_finish()
+
+        ac1_clone.start_io()
+        ac1_clone._evtracker.wait_all_initial_fetches()
+        chats = ac1_clone.get_chats()
+        assert len(chats) == 4  # two newly created chats + self-chat + device-chat
+        group_chat = [c for c in chats if c.get_name() == "group name"][0]
+        assert group_chat.is_group()
+        private_chat = [c for c in chats if c.get_name() == "ac2"][0]
+        assert not private_chat.is_group()
+
+        group_messages = group_chat.get_messages()
+        assert len(group_messages) == 1
+        assert group_messages[0].text == "incoming, unencrypted group message"
+        private_messages = private_chat.get_messages()
+        # We can't decrypt the message in this chat, so the chat is empty:
+        assert len(private_messages) == 0
 
 
 class TestGroupStressTests:

python/tox.ini

@@ -47,7 +47,9 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx
+    # With Python 3.7 and Sphinx 3.5.0, it throws an exception.
+    # Pin the version to the working one.
+    sphinx==3.4.3
     breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html

rust-toolchain

@@ -1 +1 @@
-1.45.0
+1.50.0

set_core_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys
@@ -63,13 +63,14 @@ def main():
     ffi_toml = read_toml_version("deltachat-ffi/Cargo.toml")
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
-    for line in open("CHANGELOG.md"):
-        ## 1.25.0
-        if line.startswith("## "):
-            if line[2:].strip().startswith(newversion):
-                break
-    else:
-        raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
+    if "alpha" not in newversion:
+        for line in open("CHANGELOG.md"):
+            ## 1.25.0
+            if line.startswith("## "):
+                if line[2:].strip().startswith(newversion):
+                    break
+        else:
+            raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
 
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)

spec.md

@@ -32,7 +32,7 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
+by the [Protected Headers](https://tools.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages

src/accounts.rs

@@ -1,18 +1,15 @@
 use std::collections::BTreeMap;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::task::{Context as TaskContext, Poll};
 
 use async_std::fs;
 use async_std::path::PathBuf;
+use async_std::prelude::*;
 use async_std::sync::{Arc, RwLock};
 use uuid::Uuid;
 
-use anyhow::{ensure, Context as _};
+use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
-use crate::error::Result;
 use crate::events::Event;
 
 /// Account manager, that can handle multiple accounts in a single place.
@@ -190,7 +187,7 @@ impl Accounts {
         let id = self.add_account().await?;
         let ctx = self.get_account(id).await.expect("just added");
 
-        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, Some(file)).await {
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, &file).await {
             Ok(_) => Ok(id),
             Err(err) => {
                 // remove temp account
@@ -225,61 +222,42 @@ impl Accounts {
 
     /// Unified event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
-        let emitters = self
+        let emitters: Vec<_> = self
             .accounts
             .read()
             .await
             .iter()
-            .map(|(id, a)| EmitterWrapper {
-                id: *id,
-                emitter: a.get_event_emitter(),
-                done: AtomicBool::new(false),
-            })
+            .map(|(_id, a)| a.get_event_emitter())
             .collect();
 
-        EventEmitter(emitters)
+        EventEmitter(futures::stream::select_all(emitters))
     }
 }
 
+#[derive(Debug)]
+pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
+
 impl EventEmitter {
-    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
-    pub fn recv_sync(&self) -> Option<Event> {
+    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
+    pub fn recv_sync(&mut self) -> Option<Event> {
         async_std::task::block_on(self.recv())
     }
 
-    /// Async recv of an event. Return `None` if the `Sender` has been droped.
-    pub async fn recv(&self) -> Option<Event> {
-        futures::future::poll_fn(|cx| Pin::new(self).recv_poll(cx)).await
-    }
-
-    fn recv_poll(self: Pin<&Self>, _cx: &mut TaskContext<'_>) -> Poll<Option<Event>> {
-        for e in &*self.0 {
-            if e.done.load(Ordering::Acquire) {
-                // skip emitters that are already done
-                continue;
-            }
-
-            match e.emitter.try_recv() {
-                Ok(event) => return Poll::Ready(Some(event)),
-                Err(async_std::sync::TryRecvError::Disconnected) => {
-                    e.done.store(true, Ordering::Release);
-                }
-                Err(async_std::sync::TryRecvError::Empty) => {}
-            }
-        }
-
-        Poll::Pending
+    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
+    pub async fn recv(&mut self) -> Option<Event> {
+        self.0.next().await
     }
 }
 
-#[derive(Debug)]
-pub struct EventEmitter(Vec<EmitterWrapper>);
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
 
-#[derive(Debug)]
-struct EmitterWrapper {
-    id: u32,
-    emitter: crate::events::EventEmitter,
-    done: AtomicBool,
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
+    }
 }
 
 pub const CONFIG_NAME: &str = "accounts.toml";
@@ -368,7 +346,6 @@ impl Config {
 
             inner.accounts.push(AccountConfig {
                 id,
-                name: String::new(),
                 dir: target_dir.into(),
                 uuid,
             });
@@ -435,8 +412,6 @@ impl Config {
 pub struct AccountConfig {
     /// Unique id.
     pub id: u32,
-    /// Display name
-    pub name: String,
     /// Root directory for all data for this account.
     pub dir: std::path::PathBuf,
     pub uuid: Uuid,

src/aheader.rs

@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
-use crate::contact::*;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};

src/blob.rs

@@ -7,14 +7,17 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::Error;
 use image::GenericImageView;
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::{
+    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
+    WORSE_IMAGE_SIZE,
+};
 use crate::context::Context;
-use crate::error::Error;
 use crate::events::EventType;
 use crate::message;
 
@@ -55,7 +58,7 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
-        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext).await?;
+        let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
             .map_err(|err| BlobError::WriteFailure {
@@ -63,6 +66,12 @@ impl<'a> BlobObject<'a> {
                 blobname: name.clone(),
                 cause: err.into(),
             })?;
+
+        // workaround a bug in async-std
+        // (the executor does not handle blocking operation in Drop correctly,
+        // see https://github.com/async-rs/async-std/issues/900 )
+        let _ = file.flush().await;
+
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
@@ -151,6 +160,10 @@ impl<'a> BlobObject<'a> {
                 cause: err,
             });
         }
+
+        // workaround, see create() for details
+        let _ = dst_file.flush().await;
+
         let blob = BlobObject {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
@@ -367,27 +380,18 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
         let blob_abs = self.to_abs_path();
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
-        })?;
 
-        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
-            return Ok(());
-        }
+        let img_wh =
+            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+                .unwrap_or_default()
+            {
+                MediaQuality::Balanced => BALANCED_AVATAR_SIZE,
+                MediaQuality::Worse => WORSE_AVATAR_SIZE,
+            };
 
-        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
-        Ok(())
+        self.recode_to_size(context, blob_abs, img_wh).await
     }
 
     pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
@@ -398,39 +402,54 @@ impl<'a> BlobObject<'a> {
             return Ok(());
         }
 
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+        let img_wh =
+            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+                .unwrap_or_default()
+            {
+                MediaQuality::Balanced => BALANCED_IMAGE_SIZE,
+                MediaQuality::Worse => WORSE_IMAGE_SIZE,
+            };
+
+        self.recode_to_size(context, blob_abs, img_wh).await
+    }
+
+    async fn recode_to_size(
+        &self,
+        context: &Context,
+        blob_abs: PathBuf,
+        img_wh: u32,
+    ) -> Result<(), BlobError> {
+        let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
+        let orientation = self.get_exif_orientation(context);
 
-        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
-            .unwrap_or_default()
-            == MediaQuality::Balanced
-        {
-            BALANCED_IMAGE_SIZE
-        } else {
-            WORSE_IMAGE_SIZE
-        };
+        let do_scale = img.width() > img_wh || img.height() > img_wh;
+        let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
 
-        if img.width() <= img_wh && img.height() <= img_wh {
-            return Ok(());
+        if do_scale || do_rotate {
+            if do_scale {
+                img = img.thumbnail(img_wh, img_wh);
+            }
+
+            if do_rotate {
+                img = match orientation {
+                    Ok(90) => img.rotate90(),
+                    Ok(180) => img.rotate180(),
+                    Ok(270) => img.rotate270(),
+                    _ => img,
+                }
+            }
+
+            img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+                blobdir: context.get_blobdir().to_path_buf(),
+                blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+                cause: err.into(),
+            })?;
         }
 
-        let mut img = img.thumbnail(img_wh, img_wh);
-        match self.get_exif_orientation(context) {
-            Ok(90) => img = img.rotate90(),
-            Ok(180) => img = img.rotate180(),
-            Ok(270) => img = img.rotate270(),
-            _ => {}
-        }
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
         Ok(())
     }
 
@@ -501,69 +520,57 @@ pub enum BlobError {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_create() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
-        let fname = t.ctx.get_blobdir().join("foo");
+        let blob = BlobObject::create(&t, "foo", b"hello").await.unwrap();
+        let fname = t.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
         assert_eq!(data, b"hello");
         assert_eq!(blob.as_name(), "$BLOBDIR/foo");
-        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
+        assert_eq!(blob.to_abs_path(), t.get_blobdir().join("foo"));
     }
 
     #[async_std::test]
     async fn test_lowercase_ext() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.TXT", b"hello").await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_file_name() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_rel_path() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
     #[async_std::test]
     async fn test_suffix() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.suffix(), Some("txt"));
-        let blob = BlobObject::create(&t.ctx, "bar", b"world").await.unwrap();
+        let blob = BlobObject::create(&t, "bar", b"world").await.unwrap();
         assert_eq!(blob.suffix(), None);
     }
 
     #[async_std::test]
     async fn test_create_dup() {
         let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
-        let foo_path = t.ctx.get_blobdir().join("foo.txt");
+        BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let foo_path = t.get_blobdir().join("foo.txt");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.txt", b"world")
-            .await
-            .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
+        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -579,15 +586,15 @@ mod tests {
     #[async_std::test]
     async fn test_double_ext_preserved() {
         let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
+        BlobObject::create(&t, "foo.tar.gz", b"hello")
             .await
             .unwrap();
-        let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
+        let foo_path = t.get_blobdir().join("foo.tar.gz");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"world")
+        BlobObject::create(&t, "foo.tar.gz", b"world")
             .await
             .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -605,7 +612,7 @@ mod tests {
     async fn test_create_long_names() {
         let t = TestContext::new().await;
         let s = "1".repeat(150);
-        let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
+        let blob = BlobObject::create(&t, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
         assert!(blobname.len() < 128);
     }
@@ -615,14 +622,14 @@ mod tests {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
+        let blob = BlobObject::create_and_copy(&t, &src).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).await.is_err());
-        let whoops = t.ctx.get_blobdir().join("whoops");
+        assert!(BlobObject::create_and_copy(&t, &whoops).await.is_err());
+        let whoops = t.get_blobdir().join("whoops");
         assert!(!whoops.exists().await);
     }
 
@@ -632,14 +639,14 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
-        let src_int = t.ctx.get_blobdir().join("internal");
+        let src_int = t.get_blobdir().join("internal");
         fs::write(&src_int, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_int).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_int).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
@@ -649,7 +656,7 @@ mod tests {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"

src/chatlist.rs

@@ -1,15 +1,20 @@
 //! # Chat list module
 
+use anyhow::{bail, ensure, Result};
+
 use crate::chat;
-use crate::chat::*;
-use crate::constants::*;
-use crate::contact::*;
-use crate::context::*;
+use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
+use crate::constants::{
+    Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_DEADDROP,
+    DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT,
+    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+};
+use crate::contact::Contact;
+use crate::context::Context;
 use crate::ephemeral::delete_expired_messages;
-use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 /// An object representing a single chatlist in memory.
 ///
@@ -58,9 +63,8 @@ impl Chatlist {
     ///   messages from addresses that have no relationship to the configured account.
     ///   The last of these messages is represented by DC_CHAT_ID_DEADDROP and you can retrieve details
     ///   about it with chatlist.get_msg_id(). Typically, the UI asks the user "Do you want to chat with NAME?"
-    ///   and offers the options "Yes" (call dc_create_chat_by_msg_id()), "Never" (call dc_block_contact())
-    ///   or "Not now".
-    ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
+    ///   and offers the options "Start chat", "Block" and "Not now";
+    ///   The decision should be passed to dc_decide_on_contact_request().
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
     ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
@@ -359,30 +363,29 @@ impl Chatlist {
             return ret;
         };
 
-        let mut lastcontact = None;
-
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
-            if lastmsg.from_id != DC_CONTACT_ID_SELF
-                && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
-            {
-                lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
-            }
-
-            Some(lastmsg)
-        } else {
-            None
-        };
+        let (lastmsg, lastcontact) =
+            if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
+                if lastmsg.from_id == DC_CONTACT_ID_SELF {
+                    (Some(lastmsg), None)
+                } else {
+                    match chat.typ {
+                        Chattype::Group | Chattype::Mailinglist => {
+                            let lastcontact =
+                                Contact::load_from_db(context, lastmsg.from_id).await.ok();
+                            (Some(lastmsg), lastcontact)
+                        }
+                        Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
+                    }
+                }
+            } else {
+                (None, None)
+            };
 
         if chat.id.is_archived_link() {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.text2 = Some(
-                context
-                    .stock_str(StockMessage::NoMessages)
-                    .await
-                    .to_string(),
-            );
+            ret.text2 = Some(stock_str::no_messages(context).await);
         } else {
             ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
                 .await;
@@ -435,23 +438,26 @@ async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::chat::{create_group_chat, ProtectionStatus};
+    use crate::constants::Viewtype;
+    use crate::stock_str::StockMessage;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_try_load() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat")
+        let chat_id2 = create_group_chat(&t, ProtectionStatus::Unprotected, "b chat")
             .await
             .unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat")
+        let chat_id3 = create_group_chat(&t, ProtectionStatus::Unprotected, "c chat")
             .await
             .unwrap();
 
         // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
         assert_eq!(chats.get_chat_id(0), chat_id3);
         assert_eq!(chats.get_chat_id(1), chat_id2);
@@ -460,26 +466,24 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t.ctx, Some(&mut msg)).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        chat_id2.set_draft(&t, Some(&mut msg)).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None)
-            .await
-            .unwrap();
+        let chats = Chatlist::try_load(&t, 0, Some("b"), None).await.unwrap();
         assert_eq!(chats.len(), 1);
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
         chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .set_visibility(&t, ChatVisibility::Archived)
             .await
             .ok();
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
@@ -488,23 +492,23 @@ mod tests {
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
         let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
-        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        t.update_device_chats().await.unwrap();
+        create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_FOR_FORWARDING, None, None)
             .await
             .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+        assert!(Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -513,31 +517,29 @@ mod tests {
     #[async_std::test]
     async fn test_search_special_chat_names() {
         let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
+        t.update_device_chats().await.unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
-        t.ctx
-            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+        t.set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
 
-        t.ctx
-            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+        t.set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
@@ -546,16 +548,16 @@ mod tests {
     #[async_std::test]
     async fn test_get_summary_unwrap() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t.ctx, Some(&mut msg)).await;
+        chat_id1.set_draft(&t, Some(&mut msg)).await;
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        let summary = chats.get_summary(&t.ctx, 0, None).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let summary = chats.get_summary(&t, 0, None).await;
         assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/color.rs

@@ -0,0 +1,62 @@
+//! Implementation of Consistent Color Generation
+//!
+//! Consistent Color Generation is defined in XEP-0392.
+//!
+//! Color Vision Deficiency correction is not implemented as Delta Chat does not offer
+//! corresponding settings.
+use hsluv::hsluv_to_rgb;
+use sha1::{Digest, Sha1};
+
+/// Converts an identifier to Hue angle.
+fn str_to_angle(s: impl AsRef<str>) -> f64 {
+    let bytes = s.as_ref().as_bytes();
+    let result = Sha1::digest(bytes);
+    let checksum: u16 = result.get(0).map_or(0, |&x| u16::from(x))
+        + 256 * result.get(1).map_or(0, |&x| u16::from(x));
+    f64::from(checksum) / 65536.0 * 360.0
+}
+
+/// Converts RGB tuple to a 24-bit number.
+///
+/// Returns a 24-bit number with 8 least significant bits corresponding to the blue color and 8
+/// most significant bits corresponding to the red color.
+fn rgb_to_u32((r, g, b): (f64, f64, f64)) -> u32 {
+    let r = ((r * 256.0) as u32).min(255);
+    let g = ((g * 256.0) as u32).min(255);
+    let b = ((b * 256.0) as u32).min(255);
+    65536 * r + 256 * g + b
+}
+
+/// Converts an identifier to RGB color.
+///
+/// Saturation is set to maximum (100.0) to make colors distinguishable, and lightness is set to
+/// half (50.0) to make colors suitable both for light and dark theme.
+pub(crate) fn str_to_color(s: impl AsRef<str>) -> u32 {
+    rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_str_to_angle() {
+        // Test against test vectors from
+        // https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd
+        assert!((str_to_angle("Romeo") - 327.255249).abs() < 1e-6);
+        assert!((str_to_angle("juliet@capulet.lit") - 209.410400).abs() < 1e-6);
+        assert!((str_to_angle("😺") - 331.199341).abs() < 1e-6);
+        assert!((str_to_angle("council") - 359.994507).abs() < 1e-6);
+        assert!((str_to_angle("Board") - 171.430664).abs() < 1e-6);
+    }
+
+    #[test]
+    fn test_rgb_to_u32() {
+        assert_eq!(rgb_to_u32((0.0, 0.0, 0.0)), 0);
+        assert_eq!(rgb_to_u32((1.0, 1.0, 1.0)), 0xffffff);
+        assert_eq!(rgb_to_u32((0.0, 0.0, 1.0)), 0x0000ff);
+        assert_eq!(rgb_to_u32((0.0, 1.0, 0.0)), 0x00ff00);
+        assert_eq!(rgb_to_u32((1.0, 0.0, 0.0)), 0xff0000);
+        assert_eq!(rgb_to_u32((1.0, 0.5, 0.0)), 0xff8000);
+    }
+}

src/config.rs

@@ -7,12 +7,13 @@ use crate::blob::BlobObject;
 use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
 use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
-use crate::stock::StockMessage;
+use crate::provider::{get_provider_by_id, Provider};
+use crate::stock_str;
 
 /// The available configuration keys.
 #[derive(
@@ -35,9 +36,6 @@ pub enum Config {
     SmtpCertificateChecks,
     ServerFlags,
 
-    #[strum(props(default = "INBOX"))]
-    ImapFolder,
-
     Displayname,
     Selfstatus,
     Selfavatar,
@@ -63,12 +61,21 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MvboxMove,
 
+    #[strum(props(default = "0"))]
+    SentboxMove, // If `MvboxMove` is true, this config is ignored. Currently only used in tests.
+
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
     #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
     MediaQuality,
 
+    /// If set to "1", on the first time `start_io()` is called after configuring,
+    /// the newest existing messages are fetched.
+    /// Existing recipients are added to the contact database regardless of this setting.
+    #[strum(props(default = "1"))]
+    FetchExistingMsgs,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -110,6 +117,9 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
+    ConfiguredSpamFolder,
+    ConfiguredTimestamp,
+    ConfiguredProvider,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -121,13 +131,22 @@ pub enum Config {
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
 
-    #[strum(props(default = "0"))]
+    Bot,
+
     /// Whether we send a warning if the password is wrong (set to false when we send a warning
     /// because we do not want to send a second warning)
+    #[strum(props(default = "0"))]
     NotifyAboutWrongPw,
 
     /// address to webrtc instance to use for videochats
     WebrtcInstance,
+
+    /// Timestamp of the last time housekeeping was run
+    LastHousekeeping,
+
+    /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
+    #[strum(props(default = "60"))]
+    ScanAllFoldersDebounceSecs,
 }
 
 impl Context {
@@ -154,7 +173,7 @@ impl Context {
 
         // Default values
         match key {
-            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::Selfstatus => Some(stock_str::status_line(self).await),
             Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
@@ -167,6 +186,20 @@ impl Context {
             .unwrap_or_default()
     }
 
+    pub async fn get_config_i64(&self, key: Config) -> i64 {
+        self.get_config(key)
+            .await
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
+    }
+
+    pub async fn get_config_u64(&self, key: Config) -> u64 {
+        self.get_config(key)
+            .await
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
+    }
+
     pub async fn get_config_bool(&self, key: Config) -> bool {
         self.get_config_int(key).await != 0
     }
@@ -183,6 +216,14 @@ impl Context {
         }
     }
 
+    /// Gets the configured provider, as saved in the `configured_provider` value.
+    ///
+    /// The provider is determined by `get_provider_info()` during configuration and then saved
+    /// to the db in `param.save_to_database()`, together with all the other `configured_*` values.
+    pub async fn get_configured_provider(&self) -> Option<&'static Provider> {
+        get_provider_by_id(&self.get_config(Config::ConfiguredProvider).await?)
+    }
+
     /// Gets configured "delete_device_after" value.
     ///
     /// `None` means never delete the message, `Some(x)` means delete
@@ -207,8 +248,8 @@ impl Context {
                     .await?;
                 match value {
                     Some(value) => {
-                        let blob = BlobObject::new_from_path(&self, value).await?;
-                        blob.recode_to_avatar_size(self)?;
+                        let blob = BlobObject::new_from_path(self, value).await?;
+                        blob.recode_to_avatar_size(self).await?;
                         self.sql
                             .set_raw_config(self, key, Some(blob.as_name()))
                             .await
@@ -217,7 +258,7 @@ impl Context {
                 }
             }
             Config::Selfstatus => {
-                let def = self.stock_str(StockMessage::StatusLine).await;
+                let def = stock_str::status_line(self).await;
                 let val = if value.is_none() || value.unwrap() == def {
                     None
                 } else {
@@ -247,6 +288,11 @@ impl Context {
             _ => self.sql.set_raw_config(self, key, value).await,
         }
     }
+
+    pub async fn set_config_bool(&self, key: Config, value: bool) -> crate::sql::Result<()> {
+        self.set_config(key, if value { Some("1") } else { None })
+            .await
+    }
 }
 
 /// Returns all available configuration keys concated together.
@@ -268,8 +314,8 @@ mod tests {
     use std::string::ToString;
 
     use crate::constants;
-    use crate::constants::AVATAR_SIZE;
-    use crate::test_utils::*;
+    use crate::constants::BALANCED_AVATAR_SIZE;
+    use crate::test_utils::TestContext;
     use image::GenericImageView;
     use num_traits::FromPrimitive;
     use std::fs::File;
@@ -287,11 +333,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_default_prop() {
-        assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
-    }
-
     #[async_std::test]
     async fn test_selfavatar_outside_blobdir() {
         let t = TestContext::new().await;
@@ -301,15 +342,14 @@ mod tests {
             .unwrap()
             .write_all(avatar_bytes)
             .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
+        let avatar_blob = t.get_blobdir().join("avatar.jpg");
         assert!(!avatar_blob.exists().await);
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
         assert!(avatar_blob.exists().await);
         assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
@@ -317,14 +357,14 @@ mod tests {
         assert_eq!(img.height(), 1000);
 
         let img = image::open(avatar_blob).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
     }
 
     #[async_std::test]
     async fn test_selfavatar_in_blobdir() {
         let t = TestContext::new().await;
-        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_src = t.get_blobdir().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
             .unwrap()
@@ -335,16 +375,15 @@ mod tests {
         assert_eq!(img.width(), 900);
         assert_eq!(img.height(), 900);
 
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
 
         let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
     }
 
     #[async_std::test]
@@ -356,10 +395,9 @@ mod tests {
             .unwrap()
             .write_all(avatar_bytes)
             .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_blob = t.get_blobdir().join("avatar.png");
         assert!(!avatar_blob.exists().await);
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
             .await
             .unwrap();
         assert!(avatar_blob.exists().await);
@@ -367,24 +405,21 @@ mod tests {
             std::fs::metadata(&avatar_blob).unwrap().len(),
             avatar_bytes.len() as u64
         );
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.get_config(Config::Selfavatar).await;
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
     }
 
     #[async_std::test]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        let media_quality = t.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
         assert_eq!(media_quality, constants::MediaQuality::Balanced);
 
-        t.ctx
-            .set_config(Config::MediaQuality, Some("1"))
-            .await
-            .unwrap();
+        t.set_config(Config::MediaQuality, Some("1")).await.unwrap();
 
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        let media_quality = t.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 1);
         assert_eq!(constants::MediaQuality::Worse as i32, 1);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure/auto_mozilla.rs

@@ -251,10 +251,10 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
 
 pub(crate) async fn moz_autoconfigure(
     context: &Context,
-    url: &str,
+    url: impl AsRef<str>,
     param_in: &LoginParam,
 ) -> Result<Vec<ServerParams>, Error> {
-    let xml_raw = read_url(context, url).await?;
+    let xml_raw = read_url(context, url.as_ref()).await?;
 
     let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {

src/configure/auto_outlook.rs

@@ -73,7 +73,7 @@ fn parse_protocol<B: BufRead>(
                 }
             }
             Event::Text(ref e) => {
-                let val = e.unescape_and_decode(&reader).unwrap_or_default();
+                let val = e.unescape_and_decode(reader).unwrap_or_default();
 
                 if let Some(ref tag) = current_tag {
                     match tag.as_str() {
@@ -117,7 +117,7 @@ fn parse_redirecturl<B: BufRead>(
     let mut buf = Vec::new();
     match reader.read_event(&mut buf)? {
         Event::Text(ref e) => {
-            let val = e.unescape_and_decode(&reader).unwrap_or_default();
+            let val = e.unescape_and_decode(reader).unwrap_or_default();
             Ok(val.trim().to_string())
         }
         _ => Ok("".to_string()),
@@ -154,7 +154,7 @@ fn parse_xml_reader<B: BufRead>(
 }
 
 fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
-    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
     parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
@@ -187,9 +187,8 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
 
 pub(crate) async fn outlk_autodiscover(
     context: &Context,
-    url: &str,
+    mut url: String,
 ) -> Result<Vec<ServerParams>, Error> {
-    let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
         let xml_raw = read_url(context, &url).await?;

src/configure/mod.rs

@@ -8,20 +8,25 @@ mod server_params;
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
 use async_std::task;
+use itertools::Itertools;
+use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
-use crate::config::Config;
-use crate::constants::*;
-use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
 use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
-use crate::oauth2::*;
+use crate::oauth2::dc_get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
-use crate::stock::StockMessage;
+use crate::stock_str;
 use crate::{chat, e2ee, provider};
+use crate::{config::Config, dc_tools::time};
+use crate::{
+    constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2},
+    job,
+};
+use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
@@ -83,7 +88,7 @@ impl Context {
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
-        if let Some(provider) = provider::get_provider_info(&param.addr) {
+        if let Some(provider) = param.provider {
             if let Some(config_defaults) = &provider.config_defaults {
                 for def in config_defaults.iter() {
                     if !self.config_exists(def.key).await {
@@ -122,9 +127,11 @@ impl Context {
                     self,
                     0,
                     Some(
-                        self.stock_string_repl_str(
-                            StockMessage::ConfigurationFailed,
-                            err.to_string(),
+                        stock_str::configuration_failed(
+                            self,
+                            // We are using Anyhow's .context() and to show the
+                            // inner error, too, we need the {:#}:
+                            format!("{:#}", err),
                         )
                         .await
                     )
@@ -160,6 +167,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         DC_LP_AUTH_NORMAL as i32
     };
 
+    let ctx2 = ctx.clone();
+    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
+
     // Step 1: Load the parameters and check email-address and password
 
     if oauth2 {
@@ -199,9 +209,51 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     {
         // no advanced parameters entered by the user: query provider-database or do Autoconfig
 
-        if let Some(servers) = get_offline_autoconfig(ctx, &param.addr) {
-            param_autoconfig = Some(servers);
+        info!(
+            ctx,
+            "checking internal provider-info for offline autoconfig"
+        );
+
+        if let Some(provider) = provider::get_provider_info(&param_domain).await {
+            param.provider = Some(provider);
+            match provider.status {
+                provider::Status::OK | provider::Status::PREPARATION => {
+                    if provider.server.is_empty() {
+                        info!(ctx, "offline autoconfig found, but no servers defined");
+                        param_autoconfig = None;
+                    } else {
+                        info!(ctx, "offline autoconfig found");
+                        let servers = provider
+                            .server
+                            .iter()
+                            .map(|s| ServerParams {
+                                protocol: s.protocol,
+                                socket: s.socket,
+                                hostname: s.hostname.to_string(),
+                                port: s.port,
+                                username: match s.username_pattern {
+                                    UsernamePattern::EMAIL => param.addr.to_string(),
+                                    UsernamePattern::EMAILLOCALPART => {
+                                        if let Some(at) = param.addr.find('@') {
+                                            param.addr.split_at(at).0.to_string()
+                                        } else {
+                                            param.addr.to_string()
+                                        }
+                                    }
+                                },
+                            })
+                            .collect();
+
+                        param_autoconfig = Some(servers)
+                    }
+                }
+                provider::Status::BROKEN => {
+                    info!(ctx, "offline autoconfig found, provider is broken");
+                    param_autoconfig = None;
+                }
+            }
         } else {
+            info!(ctx, "no offline autoconfig found");
             param_autoconfig =
                 get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
         }
@@ -211,28 +263,32 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 500);
 
-    let servers = expand_param_vector(
-        param_autoconfig.unwrap_or_else(|| {
-            vec![
-                ServerParams {
-                    protocol: Protocol::IMAP,
-                    hostname: param.imap.server.clone(),
-                    port: param.imap.port,
-                    socket: param.imap.security,
-                    username: param.imap.user.clone(),
-                },
-                ServerParams {
-                    protocol: Protocol::SMTP,
-                    hostname: param.smtp.server.clone(),
-                    port: param.smtp.port,
-                    socket: param.smtp.security,
-                    username: param.smtp.user.clone(),
-                },
-            ]
-        }),
-        &param.addr,
-        &param_domain,
-    );
+    let mut servers = param_autoconfig.unwrap_or_default();
+    if !servers
+        .iter()
+        .any(|server| server.protocol == Protocol::IMAP)
+    {
+        servers.push(ServerParams {
+            protocol: Protocol::IMAP,
+            hostname: param.imap.server.clone(),
+            port: param.imap.port,
+            socket: param.imap.security,
+            username: param.imap.user.clone(),
+        })
+    }
+    if !servers
+        .iter()
+        .any(|server| server.protocol == Protocol::SMTP)
+    {
+        servers.push(ServerParams {
+            protocol: Protocol::SMTP,
+            hostname: param.smtp.server.clone(),
+            port: param.smtp.port,
+            socket: param.smtp.security,
+            username: param.smtp.user.clone(),
+        })
+    }
+    let servers = expand_param_vector(servers, &param.addr, &param_domain);
 
     progress!(ctx, 550);
 
@@ -247,32 +303,46 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         .filter(|params| params.protocol == Protocol::SMTP)
         .cloned()
         .collect();
+    let provider_strict_tls = param.provider.map_or(false, |provider| provider.strict_tls);
 
     let smtp_config_task = task::spawn(async move {
         let mut smtp_configured = false;
+        let mut errors = Vec::new();
         for smtp_server in smtp_servers {
             smtp_param.user = smtp_server.username.clone();
             smtp_param.server = smtp_server.hostname.clone();
             smtp_param.port = smtp_server.port;
             smtp_param.security = smtp_server.socket;
 
-            if try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp).await {
-                smtp_configured = true;
-                break;
+            match try_smtp_one_param(
+                &context_smtp,
+                &smtp_param,
+                &smtp_addr,
+                oauth2,
+                provider_strict_tls,
+                &mut smtp,
+            )
+            .await
+            {
+                Ok(_) => {
+                    smtp_configured = true;
+                    break;
+                }
+                Err(e) => errors.push(e),
             }
         }
 
         if smtp_configured {
-            Some(smtp_param)
+            Ok(smtp_param)
         } else {
-            None
+            Err(errors)
         }
     });
 
     progress!(ctx, 600);
 
     // Configure IMAP
-    let (_s, r) = async_std::sync::channel(1);
+    let (_s, r) = async_std::channel::bounded(1);
     let mut imap = Imap::new(r);
 
     let mut imap_configured = false;
@@ -281,15 +351,28 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         .filter(|params| params.protocol == Protocol::IMAP)
         .collect();
     let imap_servers_count = imap_servers.len();
+    let mut errors = Vec::new();
     for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
         param.imap.user = imap_server.username.clone();
         param.imap.server = imap_server.hostname.clone();
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
 
-        if try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
-            imap_configured = true;
-            break;
+        match try_imap_one_param(
+            ctx,
+            &param.imap,
+            &param.addr,
+            oauth2,
+            provider_strict_tls,
+            &mut imap,
+        )
+        .await
+        {
+            Ok(_) => {
+                imap_configured = true;
+                break;
+            }
+            Err(e) => errors.push(e),
         }
         progress!(
             ctx,
@@ -297,16 +380,19 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         );
     }
     if !imap_configured {
-        bail!("IMAP autoconfig did not succeed");
+        bail!(nicer_configuration_error(ctx, errors).await);
     }
 
     progress!(ctx, 850);
 
     // Wait for SMTP configuration
-    if let Some(smtp_param) = smtp_config_task.await {
-        param.smtp = smtp_param;
-    } else {
-        bail!("SMTP autoconfig did not succeed");
+    match smtp_config_task.await {
+        Ok(smtp_param) => {
+            param.smtp = smtp_param;
+        }
+        Err(errors) => {
+            bail!(nicer_configuration_error(ctx, errors).await);
+        }
     }
 
     progress!(ctx, 900);
@@ -328,77 +414,26 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     // the trailing underscore is correct
     param.save_to_database(ctx, "configured_").await?;
     ctx.sql.set_raw_config_bool(ctx, "configured", true).await?;
+    ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
+        .await?;
 
     progress!(ctx, 920);
 
     e2ee::ensure_secret_key_exists(ctx).await?;
     info!(ctx, "key generation completed");
 
+    job::add(
+        ctx,
+        job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
+    )
+    .await;
+
     progress!(ctx, 940);
+    update_device_chats_handle.await?;
 
     Ok(())
 }
 
-#[derive(Debug, PartialEq, Eq)]
-enum AutoconfigProvider {
-    Mozilla,
-    Outlook,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-struct AutoconfigSource {
-    provider: AutoconfigProvider,
-    url: String,
-}
-
-impl AutoconfigSource {
-    fn all(domain: &str, addr: &str) -> [Self; 5] {
-        [
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!(
-                    "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-                    domain, addr,
-                ),
-            },
-            // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!(
-                    "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
-                    domain, addr
-                ),
-            },
-            AutoconfigSource {
-                provider: AutoconfigProvider::Outlook,
-                url: format!("https://{}/autodiscover/autodiscover.xml", domain),
-            },
-            // Outlook uses always SSL but different domains (this comment describes the next two steps)
-            AutoconfigSource {
-                provider: AutoconfigProvider::Outlook,
-                url: format!(
-                    "https://autodiscover.{}/autodiscover/autodiscover.xml",
-                    domain
-                ),
-            },
-            // always SSL for Thunderbird's database
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!("https://autoconfig.thunderbird.net/v1.1/{}", domain),
-            },
-        ]
-    }
-
-    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<Vec<ServerParams>> {
-        let params = match self.provider {
-            AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
-            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url).await?,
-        };
-
-        Ok(params)
-    }
-}
-
 /// Retrieve available autoconfigurations.
 ///
 /// A Search configurations from the domain used in the email-address, prefer encrypted
@@ -409,88 +444,99 @@ async fn get_autoconfig(
     param_domain: &str,
     param_addr_urlencoded: &str,
 ) -> Option<Vec<ServerParams>> {
-    let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        format!(
+            "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
+            param_domain, param_addr_urlencoded
+        ),
+        param,
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 300);
 
-    let mut progress = 300;
-    for source in &sources {
-        let res = source.fetch(ctx, param).await;
-        progress!(ctx, progress);
-        progress += 10;
-        if let Ok(res) = res {
-            return Some(res);
-        }
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
+        format!(
+            "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
+            &param_domain, &param_addr_urlencoded
+        ),
+        param,
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 310);
+
+    // Outlook uses always SSL but different domains (this comment describes the next two steps)
+    if let Ok(res) = outlk_autodiscover(
+        ctx,
+        format!("https://{}/autodiscover/autodiscover.xml", &param_domain),
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 320);
+
+    if let Ok(res) = outlk_autodiscover(
+        ctx,
+        format!(
+            "https://autodiscover.{}/autodiscover/autodiscover.xml",
+            &param_domain
+        ),
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 330);
+
+    // always SSL for Thunderbird's database
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
+        param,
+    )
+    .await
+    {
+        return Some(res);
     }
 
     None
 }
 
-fn get_offline_autoconfig(context: &Context, addr: &str) -> Option<Vec<ServerParams>> {
-    info!(
-        context,
-        "checking internal provider-info for offline autoconfig"
-    );
-
-    if let Some(provider) = provider::get_provider_info(&addr) {
-        match provider.status {
-            provider::Status::OK | provider::Status::PREPARATION => {
-                if provider.server.is_empty() {
-                    info!(context, "offline autoconfig found, but no servers defined");
-                    None
-                } else {
-                    info!(context, "offline autoconfig found");
-                    let servers = provider
-                        .server
-                        .iter()
-                        .map(|s| ServerParams {
-                            protocol: s.protocol,
-                            socket: s.socket,
-                            hostname: s.hostname.to_string(),
-                            port: s.port,
-                            username: match s.username_pattern {
-                                UsernamePattern::EMAIL => addr.to_string(),
-                                UsernamePattern::EMAILLOCALPART => {
-                                    if let Some(at) = addr.find('@') {
-                                        addr.split_at(at).0.to_string()
-                                    } else {
-                                        addr.to_string()
-                                    }
-                                }
-                            },
-                        })
-                        .collect();
-                    Some(servers)
-                }
-            }
-            provider::Status::BROKEN => {
-                info!(context, "offline autoconfig found, provider is broken");
-                None
-            }
-        }
-    } else {
-        info!(context, "no offline autoconfig found");
-        None
-    }
-}
-
 async fn try_imap_one_param(
     context: &Context,
     param: &ServerLoginParam,
     addr: &str,
     oauth2: bool,
+    provider_strict_tls: bool,
     imap: &mut Imap,
-) -> bool {
+) -> Result<(), ConfigurationError> {
     let inf = format!(
         "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
         param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = imap.connect(context, param, addr, oauth2).await {
+    if let Err(err) = imap
+        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .await
+    {
         info!(context, "failure: {}", err);
-        false
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
     } else {
         info!(context, "success: {}", inf);
-        true
+        Ok(())
     }
 }
 
@@ -499,37 +545,74 @@ async fn try_smtp_one_param(
     param: &ServerLoginParam,
     addr: &str,
     oauth2: bool,
+    provider_strict_tls: bool,
     smtp: &mut Smtp,
-) -> bool {
+) -> Result<(), ConfigurationError> {
     let inf = format!(
         "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
         param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
+    if let Err(err) = smtp
+        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .await
+    {
         info!(context, "failure: {}", err);
-        false
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
     } else {
         info!(context, "success: {}", inf);
         smtp.disconnect().await;
-        true
+        Ok(())
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+#[error("Trying {config}…\nError: {msg}")]
+pub struct ConfigurationError {
+    config: String,
+    msg: String,
+}
+
+async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationError>) -> String {
+    let first_err = if let Some(f) = errors.first() {
+        f
+    } else {
+        // This means configuration failed but no errors have been captured. This should never
+        // happen, but if it does, the user will see classic "Error: no error".
+        return "no error".to_string();
+    };
+
+    if errors
+        .iter()
+        .all(|e| e.msg.to_lowercase().contains("could not resolve"))
+    {
+        return stock_str::error_no_network(context).await;
+    }
+
+    if errors.iter().all(|e| e.msg == first_err.msg) {
+        return first_err.msg.to_string();
+    }
+
+    errors.iter().map(|e| e.to_string()).join("\n\n")
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("Invalid email address: {0:?}")]
     InvalidEmailAddress(String),
 
-    #[error("XML error at position {position}")]
+    #[error("XML error at position {position}: {error}")]
     InvalidXml {
         position: usize,
         #[source]
         error: quick_xml::Error,
     },
 
-    #[error("Failed to get URL")]
+    #[error("Failed to get URL: {0}")]
     ReadUrlError(#[from] self::read_url::Error),
 
     #[error("Number of redirection is exceeded")]
@@ -540,37 +623,16 @@ pub enum Error {
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
-    use super::*;
-    use crate::config::*;
-    use crate::test_utils::*;
+    use crate::config::Config;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_no_panic_on_bad_credentials() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::Addr, Some("probably@unexistant.addr"))
+        t.set_config(Config::Addr, Some("probably@unexistant.addr"))
             .await
             .unwrap();
-        t.ctx
-            .set_config(Config::MailPw, Some("123456"))
-            .await
-            .unwrap();
-        assert!(t.ctx.configure().await.is_err());
-    }
-
-    #[async_std::test]
-    async fn test_get_offline_autoconfig() {
-        let context = TestContext::new().await.ctx;
-
-        let addr = "someone123@example.org";
-        assert!(get_offline_autoconfig(&context, addr).is_none());
-
-        let addr = "someone123@nauta.cu";
-        let found_params = get_offline_autoconfig(&context, addr).unwrap();
-        assert_eq!(found_params.len(), 2);
-        assert_eq!(found_params[0].protocol, Protocol::IMAP);
-        assert_eq!(found_params[0].hostname, "imap.nauta.cu".to_string());
-        assert_eq!(found_params[1].protocol, Protocol::SMTP);
-        assert_eq!(found_params[1].hostname, "smtp.nauta.cu".to_string());
+        t.set_config(Config::MailPw, Some("123456")).await.unwrap();
+        assert!(t.configure().await.is_err());
     }
 }

src/configure/read_url.rs

@@ -12,7 +12,7 @@ pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
     match surf::get(url).recv_string().await {
         Ok(res) => Ok(res),
         Err(err) => {
-            info!(context, "Can\'t read URL {}", url);
+            info!(context, "Can\'t read URL {}: {}", url, err);
 
             Err(Error::GetError(err))
         }

src/constants.rs

@@ -1,11 +1,9 @@
 //! # Constants
-use deltachat_derive::*;
-use lazy_static::lazy_static;
+use deltachat_derive::{FromSql, ToSql};
+use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
-lazy_static! {
-    pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
-}
+pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION").to_string());
 
 #[derive(
     Debug,
@@ -34,7 +32,9 @@ impl Default for Blocked {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum ShowEmails {
     Off = 0,
@@ -48,7 +48,9 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum MediaQuality {
     Balanced = 0,
@@ -61,7 +63,9 @@ impl Default for MediaQuality {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum KeyGenType {
     Default = 0,
@@ -75,7 +79,9 @@ impl Default for KeyGenType {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(i8)]
 pub enum VideochatType {
     Unknown = 0,
@@ -101,21 +107,24 @@ pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
 pub const DC_GCL_FOR_FORWARDING: usize = 0x08;
 
 pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
+pub const DC_GCM_INFO_ONLY: u32 = 0x02;
 
-pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
-pub const DC_GCL_ADD_SELF: usize = 0x02;
+pub const DC_GCL_VERIFIED_ONLY: u32 = 0x01;
+pub const DC_GCL_ADD_SELF: u32 = 0x02;
 
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 
+// warn about an outdated app after a given number of days.
+// as we use the "provider-db generation date" as reference (that might not be updated very often)
+// and as not all system get speedy updates,
+// do not use too small value that will annoy users checking for nonexistant updates.
+pub const DC_OUTDATED_WARNING_DAYS: i64 = 365;
+
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 pub const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
 pub const DC_CHAT_ID_TRASH: u32 = 3;
-/// a message is just in creation but not yet assigned to a chat (eg. we may need the message ID to set up blobs; this avoids unready message to be sent and shown)
-pub const DC_CHAT_ID_MSGS_IN_CREATION: u32 = 4;
-/// virtual chat showing all messages flagged with msgs.starred=2
-pub const DC_CHAT_ID_STARRED: u32 = 5;
 /// only an indicator in a chatlist
 pub const DC_CHAT_ID_ARCHIVED_LINK: u32 = 6;
 /// only an indicator in a chatlist
@@ -143,7 +152,7 @@ pub enum Chattype {
     Undefined = 0,
     Single = 100,
     Group = 120,
-    VerifiedGroup = 130,
+    Mailinglist = 140,
 }
 
 impl Default for Chattype {
@@ -156,8 +165,34 @@ pub const DC_MSG_ID_MARKER1: u32 = 1;
 pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
-/// approx. max. length returned by dc_msg_get_text()
-pub const DC_MAX_GET_TEXT_LEN: usize = 30000;
+/// string that indicates sth. is left out or truncated
+pub const DC_ELLIPSE: &str = "[...]";
+
+/// to keep bubbles and chat flow usable,
+/// and to avoid problems with controls using very long texts,
+/// we limit the text length to DC_DESIRED_TEXT_LEN.
+/// if the text is longer, the full text can be retrieved usind has_html()/get_html().
+///
+/// we are using a bit less than DC_MAX_GET_TEXT_LEN to avoid cutting twice
+/// (a bit less as truncation may not be exact and ellipses may be added).
+///
+/// note, that DC_DESIRED_TEXT_LEN and DC_MAX_GET_TEXT_LEN
+/// define max. number of bytes, _not_ unicode graphemes.
+/// in general, that seems to be okay for such an upper limit,
+/// esp. as calculating the number of graphemes is not simple
+/// (one graphemes may be a sequence of code points which is a sequence of bytes).
+/// also even if we have the exact number of graphemes,
+/// that would not always help on getting an idea about the screen space used
+/// (to keep bubbles and chat flow usable).
+///
+/// therefore, the number of bytes is only a very rough estimation,
+/// however, the ~30K seems to work okayish for a while,
+/// if it turns out, it is too few for some alphabet, we can still increase.
+pub const DC_DESIRED_TEXT_LEN: usize = 29_000;
+
+/// approx. max. length (number of bytes) returned by dc_msg_get_text()
+pub const DC_MAX_GET_TEXT_LEN: usize = 30_000;
+
 /// approx. max. length returned by dc_get_msg_info()
 pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
@@ -193,8 +228,12 @@ pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
 
+/// How many existing messages shall be fetched after configuration.
+pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
+
 // max. width/height of an avatar
-pub const AVATAR_SIZE: u32 = 192;
+pub const BALANCED_AVATAR_SIZE: u32 = 256;
+pub const WORSE_AVATAR_SIZE: u32 = 128;
 
 // max. width/height of images
 pub const BALANCED_IMAGE_SIZE: u32 = 1280;
@@ -203,6 +242,11 @@ pub const WORSE_IMAGE_SIZE: u32 = 640;
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
+// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is splitted to chunks.
+// this does not affect MIME'e `To:` header.
+// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
+pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
+
 #[derive(
     Debug,
     Display,

src/context.rs

@@ -3,25 +3,28 @@
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
+use std::time::{Instant, SystemTime};
 
-use async_std::path::{Path, PathBuf};
-use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
-use async_std::task;
+use anyhow::{bail, ensure, Result};
+use async_std::{
+    channel::{self, Receiver, Sender},
+    path::{Path, PathBuf},
+    sync::{Arc, Mutex, RwLock},
+    task,
+};
 
-use crate::chat::*;
+use crate::chat::{get_chat_cnt, ChatId};
 use crate::config::Config;
-use crate::constants::*;
-use crate::contact::*;
-use crate::dc_tools::duration_to_str;
-use crate::error::*;
+use crate::constants::DC_VERSION_STR;
+use crate::contact::Contact;
+use crate::dc_tools::{duration_to_str, time};
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::message::{self, MsgId};
+use crate::message::{self, MessageState, MsgId};
 use crate::scheduler::Scheduler;
 use crate::securejoin::Bob;
 use crate::sql::Sql;
-use std::time::SystemTime;
 
 #[derive(Clone, Debug)]
 pub struct Context {
@@ -44,7 +47,7 @@ pub struct InnerContext {
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
     pub(crate) os_name: Option<String>,
-    pub(crate) bob: RwLock<Bob>,
+    pub(crate) bob: Bob,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -59,7 +62,12 @@ pub struct InnerContext {
     pub(crate) scheduler: RwLock<Scheduler>,
     pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
-    /// Id for this context on the current device.
+    pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
+
+    /// ID for this `Context` in the current process.
+    ///
+    /// This allows for multiple `Context`s open in a single process where each context can
+    /// be identified by this ID.
     pub(crate) id: u32,
 
     creation_time: SystemTime,
@@ -121,7 +129,7 @@ impl Context {
             os_name: Some(os_name),
             running_state: RwLock::new(Default::default()),
             sql: Sql::new(),
-            bob: RwLock::new(Default::default()),
+            bob: Default::default(),
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
@@ -131,15 +139,13 @@ impl Context {
             scheduler: RwLock::new(Scheduler::Stopped),
             ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
+            last_full_folder_scan: Mutex::new(None),
         };
 
         let ctx = Context {
             inner: Arc::new(inner),
         };
-        ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, false).await,
-            "Failed opening sqlite database"
-        );
+        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }
@@ -147,7 +153,7 @@ impl Context {
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
         info!(self, "starting IO");
-        if self.is_io_running().await {
+        if self.inner.is_io_running().await {
             info!(self, "IO is already running");
             return;
         }
@@ -158,18 +164,9 @@ impl Context {
         }
     }
 
-    /// Returns if the IO scheduler is running.
-    pub async fn is_io_running(&self) -> bool {
-        self.inner.is_io_running().await
-    }
-
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
-        if !self.is_io_running().await {
-            info!(self, "IO is not running");
-            return;
-        }
 
         self.inner.stop_io().await;
     }
@@ -200,7 +197,10 @@ impl Context {
         });
     }
 
-    /// Get the next queued event.
+    /// Returns a receiver for emitted events.
+    ///
+    /// Multiple emitters can be created, but note that in this case each emitted event will
+    /// only be received by one of the emitters, not by all of them.
     pub fn get_event_emitter(&self) -> EventEmitter {
         self.events.get_emitter()
     }
@@ -222,7 +222,7 @@ impl Context {
 
         s.ongoing_running = true;
         s.shall_stop_ongoing = false;
-        let (sender, receiver) = channel(1);
+        let (sender, receiver) = channel::bounded(1);
         s.cancel_sender = Some(sender);
 
         Ok(receiver)
@@ -249,7 +249,9 @@ impl Context {
         let s_a = &self.running_state;
         let mut s = s_a.write().await;
         if let Some(cancel) = s.cancel_sender.take() {
-            cancel.send(()).await;
+            if let Err(err) = cancel.send(()).await {
+                warn!(self, "could not cancel ongoing: {:?}", err);
+            }
         }
 
         if s.ongoing_running && !s.shall_stop_ongoing {
@@ -310,6 +312,7 @@ impl Context {
         let sentbox_watch = self.get_config_int(Config::SentboxWatch).await;
         let mvbox_watch = self.get_config_int(Config::MvboxWatch).await;
         let mvbox_move = self.get_config_int(Config::MvboxMove).await;
+        let sentbox_move = self.get_config_int(Config::SentboxMove).await;
         let folders_configured = self
             .sql
             .get_raw_config_int(self, "folders_configured")
@@ -326,6 +329,9 @@ impl Context {
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
+
+        // insert values
+        res.insert("bot", self.get_config_int(Config::Bot).await.to_string());
         res.insert("number_of_chats", chats.to_string());
         res.insert("number_of_chat_messages", real_msgs.to_string());
         res.insert("messages_in_contact_requests", deaddrop_msgs.to_string());
@@ -344,15 +350,30 @@ impl Context {
         res.insert("is_configured", is_configured.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
+        res.insert(
+            "fetch_existing_msgs",
+            self.get_config_int(Config::FetchExistingMsgs)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "show_emails",
+            self.get_config_int(Config::ShowEmails).await.to_string(),
+        );
         res.insert("inbox_watch", inbox_watch.to_string());
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_watch", mvbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
+        res.insert("sentbox_move", sentbox_move.to_string());
         res.insert("folders_configured", folders_configured.to_string());
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
         res.insert("mdns_enabled", mdns_enabled.to_string());
         res.insert("e2ee_enabled", e2ee_enabled.to_string());
+        res.insert(
+            "key_gen_type",
+            self.get_config_int(Config::KeyGenType).await.to_string(),
+        );
         res.insert("bcc_self", bcc_self.to_string());
         res.insert(
             "private_key_count",
@@ -363,6 +384,40 @@ impl Context {
             pub_key_cnt.unwrap_or_default().to_string(),
         );
         res.insert("fingerprint", fingerprint_str);
+        res.insert(
+            "webrtc_instance",
+            self.get_config(Config::WebrtcInstance)
+                .await
+                .unwrap_or_else(|| "<unset>".to_string()),
+        );
+        res.insert(
+            "media_quality",
+            self.get_config_int(Config::MediaQuality).await.to_string(),
+        );
+        res.insert(
+            "delete_device_after",
+            self.get_config_int(Config::DeleteDeviceAfter)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "delete_server_after",
+            self.get_config_int(Config::DeleteServerAfter)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "last_housekeeping",
+            self.get_config_int(Config::LastHousekeeping)
+                .await
+                .to_string(),
+        );
+        res.insert(
+            "scan_all_folders_debounce_secs",
+            self.get_config_int(Config::ScanAllFoldersDebounceSecs)
+                .await
+                .to_string(),
+        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
@@ -370,9 +425,15 @@ impl Context {
         res
     }
 
-    pub async fn get_fresh_msgs(&self) -> Vec<MsgId> {
-        let show_deaddrop: i32 = 0;
-        self.sql
+    /// Get a list of fresh, unmuted messages in any chat but deaddrop.
+    ///
+    /// The list starts with the most recent message
+    /// and is typically used to show notifications.
+    /// Moreover, the number of returned messages
+    /// can be used for a badge counter on the app icon.
+    pub async fn get_fresh_msgs(&self) -> Result<Vec<MsgId>> {
+        let ret = self
+            .sql
             .query_map(
                 concat!(
                     "SELECT m.id",
@@ -383,12 +444,13 @@ impl Context {
                     "        ON m.chat_id=c.id",
                     " WHERE m.state=?",
                     "   AND m.hidden=0",
-                    "   AND m.chat_id>?",
+                    "   AND m.chat_id>9",
                     "   AND ct.blocked=0",
-                    "   AND (c.blocked=0 OR c.blocked=?)",
+                    "   AND c.blocked=0",
+                    "   AND NOT(c.muted_until=-1 OR c.muted_until>?)",
                     " ORDER BY m.timestamp DESC,m.id DESC;"
                 ),
-                paramsv![10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
+                paramsv![MessageState::InFresh, time()],
                 |row| row.get::<_, MsgId>(0),
                 |rows| {
                     let mut ret = Vec::new();
@@ -398,52 +460,26 @@ impl Context {
                     Ok(ret)
                 },
             )
-            .await
-            .unwrap_or_default()
+            .await?;
+        Ok(ret)
     }
 
-    #[allow(non_snake_case)]
-    pub async fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
+    /// Searches for messages containing the query string.
+    ///
+    /// If `chat_id` is provided this searches only for messages in this chat, if `chat_id`
+    /// is `None` this searches messages from all chats.
+    pub async fn search_msgs(&self, chat_id: Option<ChatId>, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
         }
-        let strLikeInText = format!("%{}%", real_query);
-        let strLikeBeg = format!("{}%", real_query);
+        let str_like_in_text = format!("%{}%", real_query);
+        let str_like_beg = format!("{}%", real_query);
 
-        let query = if !chat_id.is_unset() {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " WHERE m.chat_id=?",
-                "   AND m.hidden=0",
-                "   AND ct.blocked=0",
-                "   AND (txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp,m.id;"
-            )
-        } else {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " LEFT JOIN chats c",
-                "        ON m.chat_id=c.id",
-                " WHERE m.chat_id>9",
-                "   AND m.hidden=0",
-                "   AND (c.blocked=0 OR c.blocked=?)",
-                "   AND ct.blocked=0",
-                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp DESC,m.id DESC;"
-            )
-        };
-
-        self.sql
-            .query_map(
+        let do_query = |query, params| {
+            self.sql.query_map(
                 query,
-                paramsv![chat_id, strLikeInText, strLikeBeg],
+                params,
                 |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
@@ -453,8 +489,42 @@ impl Context {
                     Ok(ret)
                 },
             )
+        };
+
+        if let Some(chat_id) = chat_id {
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
+                 FROM msgs m
+                 LEFT JOIN contacts ct
+                        ON m.from_id=ct.id
+                 WHERE m.chat_id=?
+                   AND m.hidden=0
+                   AND ct.blocked=0
+                   AND (txt LIKE ? OR ct.name LIKE ?)
+                 ORDER BY m.timestamp,m.id;",
+                paramsv![chat_id, str_like_in_text, str_like_beg],
+            )
             .await
             .unwrap_or_default()
+        } else {
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
+                 FROM msgs m
+                 LEFT JOIN contacts ct
+                        ON m.from_id=ct.id
+                 LEFT JOIN chats c
+                        ON m.chat_id=c.id
+                 WHERE m.chat_id>9
+                   AND m.hidden=0
+                   AND c.blocked=0
+                   AND ct.blocked=0
+                   AND (m.txt LIKE ? OR ct.name LIKE ?)
+                 ORDER BY m.timestamp DESC,m.id DESC;",
+                paramsv![str_like_in_text, str_like_beg],
+            )
+            .await
+            .unwrap_or_default()
+        }
     }
 
     pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
@@ -472,6 +542,11 @@ impl Context {
             == Some(folder_name.as_ref().to_string())
     }
 
+    pub async fn is_spam_folder(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredSpamFolder).await
+            == Some(folder_name.as_ref().to_string())
+    }
+
     pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -486,14 +561,19 @@ impl InnerContext {
     }
 
     async fn stop_io(&self) {
-        assert!(self.is_io_running().await, "context is already stopped");
-        let token = {
-            let lock = &*self.scheduler.read().await;
-            lock.pre_stop().await
-        };
-        {
-            let lock = &mut *self.scheduler.write().await;
-            lock.stop(token).await;
+        if self.is_io_running().await {
+            let token = {
+                let lock = &*self.scheduler.read().await;
+                lock.pre_stop().await
+            };
+            {
+                let lock = &mut *self.scheduler.write().await;
+                lock.stop(token).await;
+            }
+        }
+
+        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
+            ephemeral_task.cancel().await;
         }
     }
 }
@@ -516,7 +596,12 @@ pub fn get_version_str() -> &'static str {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::chat::{get_chat_contacts, get_chat_msgs, set_muted, Chat, MuteDuration};
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::dc_tools::dc_create_outgoing_rfc724_mid;
+    use crate::test_utils::TestContext;
+    use std::time::Duration;
+    use strum::IntoEnumIterator;
 
     #[async_std::test]
     async fn test_wrong_db() {
@@ -530,10 +615,140 @@ mod tests {
     #[async_std::test]
     async fn test_get_fresh_msgs() {
         let t = TestContext::new().await;
-        let fresh = t.ctx.get_fresh_msgs().await;
+        let fresh = t.get_fresh_msgs().await.unwrap();
         assert!(fresh.is_empty())
     }
 
+    async fn receive_msg(t: &TestContext, chat: &Chat) {
+        let members = get_chat_contacts(t, chat.id).await;
+        let contact = Contact::load_from_db(t, *members.first().unwrap())
+            .await
+            .unwrap();
+        let msg = format!(
+            "From: {}\n\
+             To: alice@example.com\n\
+             Message-ID: <{}>\n\
+             Chat-Version: 1.0\n\
+             Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+             \n\
+             hello\n",
+            contact.get_addr(),
+            dc_create_outgoing_rfc724_mid(None, contact.get_addr())
+        );
+        println!("{}", msg);
+        dc_receive_imf(t, msg.as_bytes(), "INBOX", 1, false)
+            .await
+            .unwrap();
+    }
+
+    #[async_std::test]
+    async fn test_get_fresh_msgs_and_muted_chats() {
+        // receive various mails in 3 chats
+        let t = TestContext::new_alice().await;
+        let bob = t.create_chat_with_contact("", "bob@g.it").await;
+        let claire = t.create_chat_with_contact("", "claire@g.it").await;
+        let dave = t.create_chat_with_contact("", "dave@g.it").await;
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+        receive_msg(&t, &bob).await;
+        assert_eq!(get_chat_msgs(&t, bob.id, 0, None).await.len(), 1);
+        assert_eq!(bob.id.get_fresh_msg_cnt(&t).await, 1);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        receive_msg(&t, &claire).await;
+        receive_msg(&t, &claire).await;
+        assert_eq!(get_chat_msgs(&t, claire.id, 0, None).await.len(), 2);
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await, 2);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 3);
+
+        receive_msg(&t, &dave).await;
+        receive_msg(&t, &dave).await;
+        receive_msg(&t, &dave).await;
+        assert_eq!(get_chat_msgs(&t, dave.id, 0, None).await.len(), 3);
+        assert_eq!(dave.id.get_fresh_msg_cnt(&t).await, 3);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6);
+
+        // mute one of the chats
+        set_muted(&t, claire.id, MuteDuration::Forever)
+            .await
+            .unwrap();
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await, 2);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 4); // muted claires messages are no longer counted
+
+        // receive more messages
+        receive_msg(&t, &bob).await;
+        receive_msg(&t, &claire).await;
+        receive_msg(&t, &dave).await;
+        assert_eq!(get_chat_msgs(&t, claire.id, 0, None).await.len(), 3);
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await, 3);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6); // muted claire is not counted
+
+        // unmute claire again
+        set_muted(&t, claire.id, MuteDuration::NotMuted)
+            .await
+            .unwrap();
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await, 3);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 9); // claire is counted again
+    }
+
+    #[async_std::test]
+    async fn test_get_fresh_msgs_and_muted_until() {
+        let t = TestContext::new_alice().await;
+        let bob = t.create_chat_with_contact("", "bob@g.it").await;
+        receive_msg(&t, &bob).await;
+        assert_eq!(get_chat_msgs(&t, bob.id, 0, None).await.len(), 1);
+
+        // chat is unmuted by default, here and in the following assert(),
+        // we check mainly that the SQL-statements in is_muted() and get_fresh_msgs()
+        // have the same view to the database.
+        assert!(!bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        // test get_fresh_msgs() with mute_until in the future
+        set_muted(
+            &t,
+            bob.id,
+            MuteDuration::Until(SystemTime::now() + Duration::from_secs(3600)),
+        )
+        .await
+        .unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+        // to test get_fresh_msgs() with mute_until in the past,
+        // we need to modify the database directly
+        t.sql
+            .execute(
+                "UPDATE chats SET muted_until=? WHERE id=?;",
+                paramsv![time() - 3600, bob.id],
+            )
+            .await
+            .unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(!bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        // test get_fresh_msgs() with "forever" mute_until
+        set_muted(&t, bob.id, MuteDuration::Forever).await.unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+        // to test get_fresh_msgs() with invalid mute_until (everything < -1),
+        // that results in "muted forever" by definition.
+        t.sql
+            .execute(
+                "UPDATE chats SET muted_until=-2 WHERE id=?;",
+                paramsv![bob.id],
+            )
+            .await
+            .unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(!bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+    }
+
     #[async_std::test]
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
@@ -589,14 +804,14 @@ mod tests {
     #[async_std::test]
     async fn no_crashes_on_context_deref() {
         let t = TestContext::new().await;
-        std::mem::drop(t.ctx);
+        std::mem::drop(t);
     }
 
     #[async_std::test]
     async fn test_get_info() {
         let t = TestContext::new().await;
 
-        let info = t.ctx.get_info().await;
+        let info = t.get_info().await;
         assert!(info.get("database_dir").is_some());
     }
 
@@ -607,4 +822,47 @@ mod tests {
         assert!(info.get("database_dir").is_none());
         assert_eq!(info.get("level").unwrap(), "awesome");
     }
+
+    #[async_std::test]
+    async fn test_get_info_completeness() {
+        // For easier debugging,
+        // get_info() shall return all important information configurable by the Config-values.
+        //
+        // There are exceptions for Config-values considered to be unimportant,
+        // too sensitive or summarized in another item.
+        let skip_from_get_info = vec![
+            "addr",
+            "displayname",
+            "imap_certificate_checks",
+            "mail_server",
+            "mail_user",
+            "mail_pw",
+            "mail_port",
+            "mail_security",
+            "notify_about_wrong_pw",
+            "save_mime_headers",
+            "selfstatus",
+            "send_server",
+            "send_user",
+            "send_pw",
+            "send_port",
+            "send_security",
+            "server_flags",
+            "smtp_certificate_checks",
+        ];
+        let t = TestContext::new().await;
+        let info = t.get_info().await;
+        for key in Config::iter() {
+            let key: String = key.to_string();
+            if !skip_from_get_info.contains(&&*key)
+                && !key.starts_with("configured")
+                && !key.starts_with("sys.")
+            {
+                assert!(
+                    info.contains_key(&*key),
+                    format!("'{}' missing in get_info() output", key)
+                );
+            }
+        }
+    }
 }

src/dc_tools.rs

@@ -12,21 +12,24 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::{bail, Error};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
+use crate::chat::{add_device_msg, add_device_msg_with_importance};
+use crate::constants::{Viewtype, DC_ELLIPSE, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
-use crate::error::{bail, Error};
 use crate::events::EventType;
+use crate::message::Message;
+use crate::provider::get_provider_update_timestamp;
+use crate::stock_str;
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
 #[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
-    let ellipse = "[...]";
-
     let count = buf.chars().count();
-    if approx_chars > 0 && count > approx_chars + ellipse.len() {
+    if approx_chars > 0 && count > approx_chars + DC_ELLIPSE.len() {
         let end_pos = buf
             .char_indices()
             .nth(approx_chars)
@@ -34,40 +37,15 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
             .unwrap_or_default();
 
         if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
-            Cow::Owned(format!("{}{}", &buf[..=index], ellipse))
+            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSE))
         } else {
-            Cow::Owned(format!("{}{}", &buf[..end_pos], ellipse))
+            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSE))
         }
     } else {
         Cow::Borrowed(buf)
     }
 }
 
-/// the colors must fulfill some criterions as:
-/// - contrast to black and to white
-/// - work as a text-color
-/// - being noticeable on a typical map
-/// - harmonize together while being different enough
-/// (therefore, we cannot just use random rgb colors :)
-const COLORS: [u32; 16] = [
-    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
-    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
-];
-
-#[allow(clippy::indexing_slicing)]
-pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
-    let str_lower = s.as_ref().to_lowercase();
-    let mut checksum = 0;
-    let bytes = str_lower.as_bytes();
-    for (i, byte) in bytes.iter().enumerate() {
-        checksum += (i + 1) * *byte as usize;
-        checksum %= 0x00ff_ffff;
-    }
-    let color_index = checksum % COLORS.len();
-
-    COLORS[color_index]
-}
-
 /* ******************************************************************************
  * date/time tools
  ******************************************************************************/
@@ -151,6 +129,67 @@ pub(crate) async fn dc_create_smeared_timestamps(context: &Context, count: usize
     start
 }
 
+// if the system time is not plausible, once a day, add a device message.
+// for testing we're using time() as that is also used for message timestamps.
+// moreover, add a warning if the app is outdated.
+pub(crate) async fn maybe_add_time_based_warnings(context: &Context) {
+    if !maybe_warn_on_bad_time(context, time(), get_provider_update_timestamp()).await {
+        maybe_warn_on_outdated(context, time(), get_provider_update_timestamp()).await;
+    }
+}
+
+async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestamp: i64) -> bool {
+    if now < known_past_timestamp {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(
+            stock_str::bad_time_msg_body(
+                context,
+                Local
+                    .timestamp(now, 0)
+                    .format("%Y-%m-%d %H:%M:%S")
+                    .to_string(),
+            )
+            .await,
+        );
+        add_device_msg_with_importance(
+            context,
+            Some(
+                format!(
+                    "bad-time-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m-%d") // repeat every day
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+            true,
+        )
+        .await
+        .ok();
+        return true;
+    }
+    false
+}
+
+async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time: i64) {
+    if now > approx_compile_time + DC_OUTDATED_WARNING_DAYS * 24 * 60 * 60 {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(stock_str::update_reminder_msg_body(context).await);
+        add_device_msg(
+            context,
+            Some(
+                format!(
+                    "outdated-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m") // repeat every month
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+        )
+        .await
+        .ok();
+    }
+}
+
 /* Message-ID tools */
 pub(crate) fn dc_create_id() -> String {
     /* generate an id. the generated ID should be as short and as unique as possible:
@@ -465,34 +504,15 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn get_next_backup_path_old(
-    folder: impl AsRef<Path>,
-    backup_time: i64,
-) -> Result<PathBuf, Error> {
-    let folder = PathBuf::from(folder.as_ref());
-    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
-        .format("delta-chat-%Y-%m-%d")
-        .to_string();
-
-    // 64 backup files per day should be enough for everyone
-    for i in 0..64 {
-        let mut path = folder.clone();
-        path.push(format!("{}-{}.bak", stem, i));
-        if !path.exists().await {
-            return Ok(path);
-        }
-    }
-    bail!("could not create backup file, disk full?");
-}
-
 /// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
 /// it can be renamed to dest_path. This guarantees that the backup is complete.
-pub(crate) async fn get_next_backup_path_new(
+pub(crate) async fn get_next_backup_path(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<(PathBuf, PathBuf), Error> {
     let folder = PathBuf::from(folder.as_ref());
     let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
         .format("delta-chat-backup-%Y-%m-%d")
         .to_string();
 
@@ -620,13 +640,6 @@ impl rusqlite::types::ToSql for EmailAddress {
     }
 }
 
-/// Utility to check if a in the binary represantion of listflags
-/// the bit at position bitindex is 1.
-pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
-    let listflags = listflags as usize;
-    (listflags & bitindex) == bitindex
-}
-
 /// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
 pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
     input
@@ -637,15 +650,45 @@ pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
         .to_string()
 }
 
+pub(crate) trait IsNoneOrEmpty<T> {
+    fn is_none_or_empty(&self) -> bool;
+}
+impl<T> IsNoneOrEmpty<T> for Option<T>
+where
+    T: AsRef<str>,
+{
+    fn is_none_or_empty(&self) -> bool {
+        !matches!(self, Some(s) if !s.as_ref().is_empty())
+    }
+}
+
+pub fn remove_subject_prefix(last_subject: &str) -> String {
+    let subject_start = if last_subject.starts_with("Chat:") {
+        0
+    } else {
+        // "Antw:" is the longest abbreviation in
+        // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+        // so look at the first _5_ characters:
+        match last_subject.chars().take(5).position(|c| c == ':') {
+            Some(prefix_end) => prefix_end + 1,
+            None => 0,
+        }
+    };
+    last_subject
+        .chars()
+        .skip(subject_start)
+        .collect::<String>()
+        .trim()
+        .to_string()
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
-    use std::convert::TryInto;
 
-    use crate::constants::*;
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_rust_ftoa() {
@@ -800,6 +843,9 @@ mod tests {
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
+    use crate::chat;
+    use crate::chatlist::Chatlist;
+    use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
     use proptest::prelude::*;
 
     proptest! {
@@ -831,7 +877,7 @@ mod tests {
     #[async_std::test]
     async fn test_file_handling() {
         let t = TestContext::new().await;
-        let context = &t.ctx;
+        let context = &t;
         macro_rules! dc_file_exist {
             ($ctx:expr, $fname:expr) => {
                 $ctx.get_blobdir()
@@ -893,29 +939,15 @@ mod tests {
         assert!(!dc_file_exist!(context, &fn0).await);
     }
 
-    #[test]
-    fn test_listflags_has() {
-        let listflags: u32 = 0x1101;
-        assert!(listflags_has(listflags, 0x1));
-        assert!(!listflags_has(listflags, 0x10));
-        assert!(listflags_has(listflags, 0x100));
-        assert!(listflags_has(listflags, 0x1000));
-        let listflags: u32 = (DC_GCL_ADD_SELF | DC_GCL_VERIFIED_ONLY).try_into().unwrap();
-        assert!(listflags_has(listflags, DC_GCL_VERIFIED_ONLY));
-        assert!(listflags_has(listflags, DC_GCL_ADD_SELF));
-        let listflags: u32 = DC_GCL_VERIFIED_ONLY.try_into().unwrap();
-        assert!(!listflags_has(listflags, DC_GCL_ADD_SELF));
-    }
-
     #[async_std::test]
     async fn test_create_smeared_timestamp() {
         let t = TestContext::new().await;
         assert_ne!(
-            dc_create_smeared_timestamp(&t.ctx).await,
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t).await,
+            dc_create_smeared_timestamp(&t).await
         );
         assert!(
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t).await
                 >= SystemTime::now()
                     .duration_since(SystemTime::UNIX_EPOCH)
                     .unwrap()
@@ -927,13 +959,13 @@ mod tests {
     async fn test_create_smeared_timestamps() {
         let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t, count as usize).await;
+        let next = dc_smeared_time(&t).await;
         assert!((start + count - 1) < next);
 
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t, count as usize).await;
+        let next = dc_smeared_time(&t).await;
         assert!((start + count - 1) < next);
     }
 
@@ -993,4 +1025,132 @@ mod tests {
         assert_eq!(improve_single_line_input("Hi\naiae "), "Hi aiae");
         assert_eq!(improve_single_line_input("\r\nahte\n\r"), "ahte");
     }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_bad_time() {
+        let t = TestContext::new().await;
+        let timestamp_now = time();
+        let timestamp_future = timestamp_now + 60 * 60 * 24 * 7;
+        let timestamp_past = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 9, 1),
+            NaiveTime::from_hms(0, 0, 0),
+        )
+        .timestamp_millis()
+            / 1_000;
+
+        // a correct time must not add a device message
+        maybe_warn_on_bad_time(&t, timestamp_now, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // we cannot find out if a date in the future is wrong - a device message is not added
+        maybe_warn_on_bad_time(&t, timestamp_future, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // a date in the past must add a device message
+        maybe_warn_on_bad_time(&t, timestamp_past, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // the message should be added only once a day - test that an hour later and nearly a day later
+        maybe_warn_on_bad_time(
+            &t,
+            timestamp_past + 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        maybe_warn_on_bad_time(
+            &t,
+            timestamp_past + 60 * 60 * 24 - 1,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // next day, there should be another device message
+        maybe_warn_on_bad_time(
+            &t,
+            timestamp_past + 60 * 60 * 24,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        assert_eq!(device_chat_id, chats.get_chat_id(0));
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 2);
+    }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_outdated() {
+        let t = TestContext::new().await;
+        let timestamp_now: i64 = time();
+
+        // in about 6 months, the app should not be outdated
+        // (if this fails, provider-db is not updated since 6 months)
+        maybe_warn_on_outdated(
+            &t,
+            timestamp_now + 180 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // in 1 year, the app should be considered as outdated
+        maybe_warn_on_outdated(
+            &t,
+            timestamp_now + 365 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // do not repeat the warning every day ...
+        // (we test that for the 2 subsequent days, this may be the next month, so the result should be 1 or 2 device message)
+        maybe_warn_on_outdated(
+            &t,
+            timestamp_now + (365 + 1) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        maybe_warn_on_outdated(
+            &t,
+            timestamp_now + (365 + 2) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
+        let test_len = msgs.len();
+        assert!(test_len == 1 || test_len == 2);
+
+        // ... but every month
+        // (forward generous 33 days to avoid being in the same month as in the previous check)
+        maybe_warn_on_outdated(
+            &t,
+            timestamp_now + (365 + 33) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), test_len + 1);
+    }
 }

src/dehtml.rs

@@ -2,20 +2,51 @@
 //!
 //! A module to remove HTML tags from the email text
 
-use lazy_static::lazy_static;
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+use std::io::BufRead;
 
-lazy_static! {
-    static ref LINE_RE: regex::Regex = regex::Regex::new(r"(\r?\n)+").unwrap();
-}
+use once_cell::sync::Lazy;
+use quick_xml::{
+    events::{BytesEnd, BytesStart, BytesText},
+    Reader,
+};
+
+static LINE_RE: Lazy<regex::Regex> = Lazy::new(|| regex::Regex::new(r"(\r?\n)+").unwrap());
 
 struct Dehtml {
     strbuilder: String,
     add_text: AddText,
     last_href: Option<String>,
+    /// GMX wraps a quote in `<div name="quote">`. After a `<div name="quote">`, this count is
+    /// increased at each `<div>` and decreased at each `</div>`. This way we know when the quote ends.
+    /// If this is > `0`, then we are inside a `<div name="quote">`
+    divs_since_quote_div: u32,
+    /// Everything between <div name="quote"> and <div name="quoted-content"> is usually metadata
+    /// If this is > `0`, then we are inside a `<div name="quoted-content">`.
+    divs_since_quoted_content_div: u32,
 }
 
-#[derive(Debug, PartialEq)]
+impl Dehtml {
+    fn line_prefix(&self) -> &str {
+        if self.divs_since_quoted_content_div > 0 {
+            "> "
+        } else {
+            ""
+        }
+    }
+    fn append_prefix(&self, line_end: impl AsRef<str>) -> String {
+        // line_end is e.g. "\n\n". We add "> " if necessary.
+        line_end.as_ref().to_owned() + self.line_prefix()
+    }
+    fn get_add_text(&self) -> AddText {
+        if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {
+            AddText::No // Everything between <div name="quoted"> and <div name="quoted_content"> is metadata which we don't want
+        } else {
+            self.add_text
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
 enum AddText {
     No,
     YesRemoveLineEnds,
@@ -24,16 +55,16 @@ enum AddText {
 
 // dehtml() returns way too many newlines; however, an optimisation on this issue is not needed as
 // the newlines are typically removed in further processing by the caller
-pub fn dehtml(buf: &str) -> String {
+pub fn dehtml(buf: &str) -> Option<String> {
     let s = dehtml_quick_xml(buf);
     if !s.trim().is_empty() {
-        return s;
+        return Some(s);
     }
     let s = dehtml_manually(buf);
     if !s.trim().is_empty() {
-        return s;
+        return Some(s);
     }
-    buf.to_string()
+    None
 }
 
 pub fn dehtml_quick_xml(buf: &str) -> String {
@@ -43,6 +74,8 @@ pub fn dehtml_quick_xml(buf: &str) -> String {
         strbuilder: String::with_capacity(buf.len()),
         add_text: AddText::YesRemoveLineEnds,
         last_href: None,
+        divs_since_quote_div: 0,
+        divs_since_quoted_content_div: 0,
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
@@ -81,13 +114,16 @@ pub fn dehtml_quick_xml(buf: &str) -> String {
 }
 
 fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.add_text == AddText::YesPreserveLineEnds
-        || dehtml.add_text == AddText::YesRemoveLineEnds
+    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
+        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.add_text == AddText::YesRemoveLineEnds {
+        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
+        } else if !dehtml.line_prefix().is_empty() {
+            let l = dehtml.append_prefix("\n");
+            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -95,13 +131,16 @@ fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
 }
 
 fn dehtml_cdata_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.add_text == AddText::YesPreserveLineEnds
-        || dehtml.add_text == AddText::YesRemoveLineEnds
+    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
+        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.add_text == AddText::YesRemoveLineEnds {
+        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
+        } else if !dehtml.line_prefix().is_empty() {
+            let l = dehtml.append_prefix("\n");
+            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -112,8 +151,15 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "div" | "table" | "td" | "style" | "script" | "title" | "pre" => {
-            dehtml.strbuilder += "\n\n";
+        "p" | "table" | "td" | "style" | "script" | "title" | "pre" => {
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+            dehtml.add_text = AddText::YesRemoveLineEnds;
+        }
+        "div" => {
+            pop_tag(&mut dehtml.divs_since_quote_div);
+            pop_tag(&mut dehtml.divs_since_quoted_content_div);
+
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "a" => {
@@ -124,10 +170,14 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
             }
         }
         "b" | "strong" => {
-            dehtml.strbuilder += "*";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "*";
+            }
         }
         "i" | "em" => {
-            dehtml.strbuilder += "_";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "_";
+            }
         }
         _ => {}
     }
@@ -141,19 +191,27 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "div" | "table" | "td" => {
-            dehtml.strbuilder += "\n\n";
+        "p" | "table" | "td" => {
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+            dehtml.add_text = AddText::YesRemoveLineEnds;
+        }
+        #[rustfmt::skip]
+        "div" => {
+            maybe_push_tag(event, reader, "quote", &mut dehtml.divs_since_quote_div);
+            maybe_push_tag(event, reader, "quoted-content", &mut dehtml.divs_since_quoted_content_div);
+
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "br" => {
-            dehtml.strbuilder += "\n";
+            dehtml.strbuilder += &dehtml.append_prefix("\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "style" | "script" | "title" => {
             dehtml.add_text = AddText::No;
         }
         "pre" => {
-            dehtml.strbuilder += "\n\n";
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
@@ -174,15 +232,51 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             }
         }
         "b" | "strong" => {
-            dehtml.strbuilder += "*";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "*";
+            }
         }
         "i" | "em" => {
-            dehtml.strbuilder += "_";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "_";
+            }
         }
         _ => {}
     }
 }
 
+/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
+/// The `counts`s are stored in the `Dehtml` struct.
+fn pop_tag(count: &mut u32) {
+    if *count > 0 {
+        *count -= 1;
+    }
+}
+
+/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
+/// The `counts`s are stored in the `Dehtml` struct.
+fn maybe_push_tag(
+    event: &BytesStart,
+    reader: &Reader<impl BufRead>,
+    tag_name: &str,
+    count: &mut u32,
+) {
+    if *count > 0 || tag_contains_attr(event, reader, tag_name) {
+        *count += 1;
+    }
+}
+
+fn tag_contains_attr(event: &BytesStart, reader: &Reader<impl BufRead>, name: &str) -> bool {
+    event.attributes().any(|r| {
+        r.map(|a| {
+            a.unescape_and_decode_value(reader)
+                .map(|v| v == name)
+                .unwrap_or(false)
+        })
+        .unwrap_or(false)
+    })
+}
+
 pub fn dehtml_manually(buf: &str) -> String {
     // Just strip out everything between "<" and ">"
     let mut strbuilder = String::new();
@@ -222,21 +316,23 @@ mod tests {
                 "<a href='https://get.delta.chat/'/>",
                 "[](https://get.delta.chat/)",
             ),
-            ("", ""),
             ("<!doctype html>\n<b>fat text</b>", "*fat text*"),
             // Invalid html (at least DC should show the text if the html is invalid):
             ("<!some invalid html code>\n<b>some text</b>", "some text"),
-            ("<This text is in brackets>", "<This text is in brackets>"),
         ];
         for (input, output) in cases {
-            assert_eq!(simplify(dehtml(input), true).0, output);
+            assert_eq!(simplify(dehtml(input).unwrap(), true).0, output);
+        }
+        let none_cases = vec!["<html> </html>", ""];
+        for input in none_cases {
+            assert_eq!(dehtml(input), None);
         }
     }
 
     #[test]
     fn test_dehtml_parse_br() {
         let html = "\r\r\nline1<br>\r\n\r\n\r\rline2<br/>line3\n\r";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "line1\n\r\r\rline2\nline3");
     }
@@ -244,7 +340,7 @@ mod tests {
     #[test]
     fn test_dehtml_parse_href() {
         let html = "<a href=url>text</a";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "[text](url)");
     }
@@ -252,7 +348,7 @@ mod tests {
     #[test]
     fn test_dehtml_bold_text() {
         let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "text *bold*<>");
     }
@@ -262,7 +358,7 @@ mod tests {
         let html =
                 "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
 
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(
             plain,
@@ -285,7 +381,20 @@ mod tests {
         </body>
         </html>
         "##;
-        let txt = dehtml(input);
+        let txt = dehtml(input).unwrap();
         assert_eq!(txt.trim(), "lots of text");
     }
+
+    #[async_std::test]
+    async fn test_quote_div() {
+        let input = include_str!("../test-data/message/gmx-quote-body.eml");
+        let dehtml = dehtml(input).unwrap();
+        println!("{}", dehtml);
+        let (msg, forwarded, cut, top_quote, footer) = simplify(dehtml, false);
+        assert_eq!(msg, "Test");
+        assert_eq!(forwarded, false);
+        assert_eq!(cut, false);
+        assert_eq!(top_quote.as_deref(), Some("test"));
+        assert_eq!(footer, None);
+    }
 }

src/e2ee.rs

@@ -2,18 +2,18 @@
 
 use std::collections::HashSet;
 
+use anyhow::{bail, ensure, format_err, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
-use crate::aheader::*;
+use crate::aheader::{Aheader, EncryptPreference};
 use crate::config::Config;
 use crate::context::Context;
-use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
-use crate::keyring::*;
-use crate::peerstate::*;
+use crate::keyring::Keyring;
+use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::pgp;
 
 #[derive(Debug)]
@@ -51,23 +51,42 @@ impl EncryptHelper {
     }
 
     /// Determines if we can and should encrypt.
+    ///
+    /// For encryption to be enabled, `e2ee_guaranteed` should be true, or strictly more than a half
+    /// of peerstates should prefer encryption. Own preference is counted equally to peer
+    /// preferences, even if message copy is not sent to self.
+    ///
+    /// `e2ee_guaranteed` should be set to true for replies to encrypted messages (as required by
+    /// Autocrypt Level 1, version 1.1) and for messages sent in protected groups.
+    ///
+    /// Returns an error if `e2ee_guaranteed` is true, but one or more keys are missing.
     pub fn should_encrypt(
         &self,
         context: &Context,
         e2ee_guaranteed: bool,
         peerstates: &[(Option<Peerstate>, &str)],
     ) -> Result<bool> {
-        if !(self.prefer_encrypt == EncryptPreference::Mutual || e2ee_guaranteed) {
-            return Ok(false);
-        }
-
+        let mut prefer_encrypt_count = if self.prefer_encrypt == EncryptPreference::Mutual {
+            1
+        } else {
+            0
+        };
         for (peerstate, addr) in peerstates {
             match peerstate {
                 Some(peerstate) => {
-                    if peerstate.prefer_encrypt != EncryptPreference::Mutual && !e2ee_guaranteed {
-                        info!(context, "peerstate for {:?} is no-encrypt", addr);
-                        return Ok(false);
-                    }
+                    info!(
+                        context,
+                        "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
+                    );
+                    match peerstate.prefer_encrypt {
+                        EncryptPreference::NoPreference => {}
+                        EncryptPreference::Mutual => prefer_encrypt_count += 1,
+                        EncryptPreference::Reset => {
+                            if !e2ee_guaranteed {
+                                return Ok(false);
+                            }
+                        }
+                    };
                 }
                 None => {
                     let msg = format!("peerstate for {:?} missing, cannot encrypt", addr);
@@ -81,7 +100,11 @@ impl EncryptHelper {
             }
         }
 
-        Ok(true)
+        // Count number of recipients, including self.
+        // This does not depend on whether we send a copy to self or not.
+        let recipients_count = peerstates.len() + 1;
+
+        Ok(e2ee_guaranteed || 2 * prefer_encrypt_count > recipients_count)
     }
 
     /// Tries to encrypt the passed in `mail`.
@@ -90,7 +113,7 @@ impl EncryptHelper {
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: Vec<(Option<Peerstate<'_>>, &str)>,
+        peerstates: Vec<(Option<Peerstate>, &str)>,
     ) -> Result<String> {
         let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
 
@@ -130,7 +153,7 @@ pub async fn try_decrypt(
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
+        .and_then(|from_addr| mailparse::addrparse_header(from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -140,10 +163,10 @@ pub async fn try_decrypt(
     // Apply Autocrypt header
     if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
         if let Some(ref mut peerstate) = peerstate {
-            peerstate.apply_header(&header, message_time);
+            peerstate.apply_header(header, message_time);
             peerstate.save_to_db(&context.sql, false).await?;
         } else {
-            let p = Peerstate::from_header(context, header, message_time);
+            let p = Peerstate::from_header(header, message_time);
             p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
@@ -212,9 +235,9 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
     }
 }
 
-async fn decrypt_if_autocrypt_message<'a>(
+async fn decrypt_if_autocrypt_message(
     context: &Context,
-    mail: &ParsedMail<'a>,
+    mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
     ret_valid_signatures: &mut HashSet<Fingerprint>,
@@ -323,10 +346,10 @@ mod tests {
 
     use crate::chat;
     use crate::constants::Viewtype;
-    use crate::contact::{Contact, Origin};
     use crate::message::Message;
     use crate::param::Param;
-    use crate::test_utils::*;
+    use crate::peerstate::ToSave;
+    use crate::test_utils::{bob_keypair, TestContext};
 
     mod ensure_secret_key_exists {
         use super::*;
@@ -335,13 +358,13 @@ mod tests {
         async fn test_prexisting() {
             let t = TestContext::new().await;
             let test_addr = t.configure_alice().await;
-            assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
+            assert_eq!(ensure_secret_key_exists(&t).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
         async fn test_not_configured() {
             let t = TestContext::new().await;
-            assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
+            assert!(ensure_secret_key_exists(&t).await.is_err());
         }
     }
 
@@ -388,27 +411,12 @@ Sent with my Delta Chat Messenger: https://delta.chat";
     }
 
     #[async_std::test]
-    async fn test_encrypted_no_autocrypt() -> crate::error::Result<()> {
+    async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
-        let (contact_alice_id, _modified) = Contact::add_or_lookup(
-            &bob.ctx,
-            "Alice",
-            "alice@example.com",
-            Origin::ManuallyCreated,
-        )
-        .await?;
-        let (contact_bob_id, _modified) = Contact::add_or_lookup(
-            &alice.ctx,
-            "Bob",
-            "bob@example.net",
-            Origin::ManuallyCreated,
-        )
-        .await?;
-
-        let chat_alice = chat::create_by_contact_id(&alice.ctx, contact_bob_id).await?;
-        let chat_bob = chat::create_by_contact_id(&bob.ctx, contact_alice_id).await?;
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
 
         // Alice sends unencrypted message to Bob
         let mut msg = Message::new(Viewtype::Text);
@@ -488,4 +496,55 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
         Ok(())
     }
+
+    fn new_peerstates(prefer_encrypt: EncryptPreference) -> Vec<(Option<Peerstate>, &'static str)> {
+        let addr = "bob@foo.bar";
+        let pub_key = bob_keypair().public;
+        let peerstate = Peerstate {
+            addr: addr.into(),
+            last_seen: 13,
+            last_seen_autocrypt: 14,
+            prefer_encrypt,
+            public_key: Some(pub_key.clone()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
+            gossip_key: Some(pub_key.clone()),
+            gossip_timestamp: 15,
+            gossip_key_fingerprint: Some(pub_key.fingerprint()),
+            verified_key: Some(pub_key.clone()),
+            verified_key_fingerprint: Some(pub_key.fingerprint()),
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        };
+        let mut peerstates = Vec::new();
+        peerstates.push((Some(peerstate), addr));
+        peerstates
+    }
+
+    #[async_std::test]
+    async fn test_should_encrypt() {
+        let t = TestContext::new_alice().await;
+        let encrypt_helper = EncryptHelper::new(&t).await.unwrap();
+
+        // test with EncryptPreference::NoPreference:
+        // if e2ee_eguaranteed is unset, there is no encryption as not more than half of peers want encryption
+        let ps = new_peerstates(EncryptPreference::NoPreference);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
+
+        // test with EncryptPreference::Reset
+        let ps = new_peerstates(EncryptPreference::Reset);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
+
+        // test with EncryptPreference::Mutual (self is also Mutual)
+        let ps = new_peerstates(EncryptPreference::Mutual);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
+
+        // test with missing peerstate
+        let mut ps = Vec::new();
+        ps.push((None, "bob@foo.bar"));
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).is_err());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
+    }
 }

src/ephemeral.rs

@@ -56,24 +56,26 @@
 //! the database entries which are expired either according to their
 //! ephemeral message timers or global `delete_server_after` setting.
 
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use anyhow::{ensure, Error};
+use async_std::task;
+use serde::{Deserialize, Serialize};
+
 use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
 use crate::constants::{
     Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
 };
 use crate::context::Context;
 use crate::dc_tools::time;
-use crate::error::{ensure, Error};
 use crate::events::EventType;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::sql;
-use crate::stock::StockMessage;
-use async_std::task;
-use serde::{Deserialize, Serialize};
-use std::convert::{TryFrom, TryInto};
-use std::num::ParseIntError;
-use std::str::FromStr;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use crate::stock_str;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -211,21 +213,50 @@ pub(crate) async fn stock_ephemeral_timer_changed(
     timer: Timer,
     from_id: u32,
 ) -> String {
-    let stock_message = match timer {
-        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+    match timer {
+        Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
         Timer::Enabled { duration } => match duration {
-            60 => StockMessage::MsgEphemeralTimerMinute,
-            3600 => StockMessage::MsgEphemeralTimerHour,
-            86400 => StockMessage::MsgEphemeralTimerDay,
-            604_800 => StockMessage::MsgEphemeralTimerWeek,
-            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
-            _ => StockMessage::MsgEphemeralTimerEnabled,
+            0..=59 => {
+                stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id).await
+            }
+            60 => stock_str::msg_ephemeral_timer_minute(context, from_id).await,
+            61..=3599 => {
+                stock_str::msg_ephemeral_timer_minutes(
+                    context,
+                    format!("{}", (f64::from(duration) / 6.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            3600 => stock_str::msg_ephemeral_timer_hour(context, from_id).await,
+            3601..=86399 => {
+                stock_str::msg_ephemeral_timer_hours(
+                    context,
+                    format!("{}", (f64::from(duration) / 360.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            86400 => stock_str::msg_ephemeral_timer_day(context, from_id).await,
+            86401..=604_799 => {
+                stock_str::msg_ephemeral_timer_days(
+                    context,
+                    format!("{}", (f64::from(duration) / 8640.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            604_800 => stock_str::msg_ephemeral_timer_week(context, from_id).await,
+            _ => {
+                stock_str::msg_ephemeral_timer_weeks(
+                    context,
+                    format!("{}", (f64::from(duration) / 60480.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
         },
-    };
-
-    context
-        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
-        .await
+    }
 }
 
 impl MsgId {
@@ -277,11 +308,13 @@ pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, E
     let mut updated = context
         .sql
         .execute(
+            // If you change which information is removed here, also change MsgId::trash() and
+            // which information dc_receive_imf::add_parts() still adds to the db if the chat_id is TRASH
             "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
+             SET chat_id=?, txt='', subject='', txt_raw='', mime_headers='', from_id=0, to_id=0, param='' \
              WHERE \
              ephemeral_timestamp != 0 \
-             AND ephemeral_timestamp < ? \
+             AND ephemeral_timestamp <= ? \
              AND chat_id != ?",
             paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
         )
@@ -417,7 +450,7 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
             "SELECT id FROM msgs \
          WHERE ( \
          timestamp < ? \
-         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
          LIMIT 1",
@@ -459,12 +492,18 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()>
 
 #[cfg(test)]
 mod tests {
+    use async_std::task::sleep;
+
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
+    use crate::{
+        chat::{self, Chat, ChatItem},
+        dc_tools::IsNoneOrEmpty,
+    };
 
     #[async_std::test]
     async fn test_stock_ephemeral_messages() {
-        let context = TestContext::new().await.ctx;
+        let context = TestContext::new().await;
 
         assert_eq!(
             stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
@@ -472,24 +511,78 @@ mod tests {
         );
 
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
-            "Message deletion timer is disabled."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 1 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 s by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
-            "Message deletion timer is set to 1 s."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 30 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 30 s by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
-            "Message deletion timer is set to 30 s."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 minute by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
-            "Message deletion timer is set to 1 minute."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 90 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1.5 minutes by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
-            "Message deletion timer is set to 1 hour."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 30 * 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 30 minutes by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 60 * 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 hour by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 5400 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1.5 hours by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 2 * 60 * 60
+                },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 2 hours by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -497,10 +590,21 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 1 day."
+            "Message deletion timer is set to 1 day by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 2 * 24 * 60 * 60
+                },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 2 days by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -508,10 +612,10 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 1 week."
+            "Message deletion timer is set to 1 week by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -519,10 +623,119 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 4 weeks."
+            "Message deletion timer is set to 4 weeks by me."
         );
     }
+
+    #[async_std::test]
+    async fn test_ephemeral_timer() -> anyhow::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
+
+        // Alice sends message to Bob
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+
+        // Alice sends second message to Bob, with no timer
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Disabled
+        );
+
+        // Bob sets ephemeral timer and sends a message about timer change
+        chat_bob
+            .set_ephemeral_timer(&bob.ctx, Timer::Enabled { duration: 60 })
+            .await?;
+        let sent_timer_change = bob.pop_sent_msg().await;
+
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        // Bob receives message from Alice.
+        // Alice message has no timer. However, Bob should not disable timer,
+        // because Alice replies to old message.
+        bob.recv_msg(&sent).await;
+
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Disabled
+        );
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        // Alice receives message from Bob
+        alice.recv_msg(&sent_timer_change).await;
+
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_ephemeral_delete_msgs() {
+        let t = TestContext::new_alice().await;
+        let chat = t.get_self_chat().await;
+
+        t.send_text(chat.id, "Saved message, which we delete manually")
+            .await;
+        let msg = t.get_last_msg_in(chat.id).await;
+        msg.id.delete_from_db(&t).await.unwrap();
+        check_msg_was_deleted(&t, &chat, msg.id).await;
+
+        chat.id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
+            .await
+            .unwrap();
+        let msg = t
+            .send_text(chat.id, "Saved message, disappearing after 1s")
+            .await;
+
+        sleep(Duration::from_millis(1100)).await;
+
+        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
+    }
+
+    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+        let chat_items = chat::get_chat_msgs(t, chat.id, 0, None).await;
+        // Check that the chat is empty except for possibly info messages:
+        for item in &chat_items {
+            if let ChatItem::Message { msg_id } = item {
+                let msg = Message::load_from_db(t, *msg_id).await.unwrap();
+                assert!(msg.is_info())
+            }
+        }
+
+        // Check that if there is a message left, the text and metadata are gone
+        if let Ok(msg) = Message::load_from_db(t, msg_id).await {
+            assert_eq!(msg.from_id, 0);
+            assert_eq!(msg.to_id, 0);
+            assert!(msg.text.is_none_or_empty(), msg.text);
+            let rawtxt: Option<String> = t
+                .sql
+                .query_get_value(t, "SELECT txt_raw FROM msgs WHERE id=?;", paramsv![msg_id])
+                .await;
+            assert!(rawtxt.is_none_or_empty(), rawtxt);
+        }
+    }
 }

src/error.rs

@@ -1,7 +1,5 @@
 //! # Error handling
 
-pub use anyhow::{bail, ensure, format_err, Error, Result};
-
 #[macro_export]
 macro_rules! ensure_eq {
     ($left:expr, $right:expr) => ({

src/events.rs

@@ -2,8 +2,8 @@
 
 use std::ops::Deref;
 
+use async_std::channel::{self, Receiver, Sender, TrySendError};
 use async_std::path::PathBuf;
-use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
@@ -18,7 +18,7 @@ pub struct Events {
 
 impl Default for Events {
     fn default() -> Self {
-        let (sender, receiver) = channel(1_000);
+        let (sender, receiver) = channel::bounded(1_000);
 
         Self { receiver, sender }
     }
@@ -35,7 +35,7 @@ impl Events {
                 // try again
                 self.emit(event);
             }
-            Err(TrySendError::Disconnected(_)) => {
+            Err(TrySendError::Closed(_)) => {
                 unreachable!("unable to emit event, channel disconnected");
             }
         }
@@ -47,6 +47,17 @@ impl Events {
     }
 }
 
+/// A receiver of events from a [`Context`].
+///
+/// See [`Context::get_event_emitter`] to create an instance.  If multiple instances are
+/// created events emitted by the [`Context`] will only be delivered to one of the
+/// `EventEmitter`s.
+///
+/// The `EventEmitter` is also a [`Stream`], so a typical usage is in a `while let` loop.
+///
+/// [`Context`]: crate::context::Context
+/// [`Context::get_event_emitter`]: crate::context::Context::get_event_emitter
+/// [`Stream`]: async_std::stream::Stream
 #[derive(Debug, Clone)]
 pub struct EventEmitter(Receiver<Event>);
 
@@ -58,18 +69,42 @@ impl EventEmitter {
 
     /// Async recv of an event. Return `None` if the `Sender` has been droped.
     pub async fn recv(&self) -> Option<Event> {
-        // TODO: change once we can use async channels internally.
         self.0.recv().await.ok()
     }
-
-    pub fn try_recv(&self) -> Result<Event, async_std::sync::TryRecvError> {
-        self.0.try_recv()
-    }
 }
 
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
+    }
+}
+
+/// The event emitted by a [`Context`] from an [`EventEmitter`].
+///
+/// Events are documented on the C/FFI API in `deltachat.h` as `DC_EVENT_*` contants.  The
+/// context emits them in relation to various operations happening, a lot of these are again
+/// documented in `deltachat.h`.
+///
+/// This struct [`Deref`]s to the [`EventType`].
+///
+/// [`Context`]: crate::context::Context
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Event {
+    /// The ID of the [`Context`] which emitted this event.
+    ///
+    /// This allows using multiple [`Context`]s in a single process as they are identified
+    /// by this ID.
+    ///
+    /// [`Context`]: crate::context::Context
     pub id: u32,
+    /// The event payload.
+    ///
+    /// These are documented in `deltachat.h` as the `DC_EVENT_*` constants.
     pub typ: EventType,
 }
 
@@ -82,7 +117,9 @@ impl Deref for Event {
 }
 
 impl EventType {
-    /// Returns the corresponding Event id.
+    /// Returns the corresponding Event ID.
+    ///
+    /// These are the IDs used in the `DC_EVENT_*` constants in `deltachat.h`.
     pub fn as_id(&self) -> i32 {
         self.get_str("id")
             .expect("missing id")
@@ -94,8 +131,9 @@ impl EventType {
 #[derive(Debug, Clone, PartialEq, Eq, EnumProperty)]
 pub enum EventType {
     /// The library-user may write an informational string to the log.
-    /// Passed to the callback given to dc_context_new().
-    /// This event should not be reported to the end-user using a popup or something like that.
+    ///
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
     #[strum(props(id = "100"))]
     Info(String),
 
@@ -128,14 +166,13 @@ pub enum EventType {
     DeletedBlobFile(String),
 
     /// The library-user should write a warning string to the log.
-    /// Passed to the callback given to dc_context_new().
     ///
-    /// This event should not be reported to the end-user using a popup or something like that.
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
     #[strum(props(id = "300"))]
     Warning(String),
 
     /// The library-user should report an error to the end-user.
-    /// Passed to the callback given to dc_context_new().
     ///
     /// As most things are asynchronous, things may go wrong at any time and the user
     /// should not be disturbed by a dialog or so.  Instead, use a bubble or so.
@@ -186,6 +223,11 @@ pub enum EventType {
     #[strum(props(id = "2005"))]
     IncomingMsg { chat_id: ChatId, msg_id: MsgId },
 
+    /// Messages were seen or noticed.
+    /// chat id is always set.
+    #[strum(props(id = "2008"))]
+    MsgsNoticed(ChatId),
+
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
     #[strum(props(id = "2010"))]

src/format_flowed.rs

@@ -21,21 +21,28 @@
 /// length. However, this should be rare and should not result in
 /// immediate mail rejection: SMTP (RFC 2821) limit is 998 characters,
 /// and Spam Assassin limit is 78 characters.
-fn format_line_flowed(line: &str) -> String {
+fn format_line_flowed(line: &str, prefix: &str) -> String {
     let mut result = String::new();
-    let mut buffer = String::new();
+    let mut buffer = prefix.to_string();
     let mut after_space = false;
 
     for c in line.chars() {
         if c == ' ' {
             buffer.push(c);
             after_space = true;
+        } else if c == '>' {
+            if buffer.is_empty() {
+                // Space stuffing, see RFC 3676
+                buffer.push(' ');
+            }
+            buffer.push(c);
+            after_space = false;
         } else {
-            if after_space && buffer.len() >= 72 && !c.is_whitespace() && c != '>' {
+            if after_space && buffer.len() >= 72 && !c.is_whitespace() {
                 // Flush the buffer and insert soft break (SP CRLF).
                 result += &buffer;
                 result += "\r\n";
-                buffer = String::new();
+                buffer = prefix.to_string();
             }
             buffer.push(c);
             after_space = false;
@@ -44,6 +51,28 @@ fn format_line_flowed(line: &str) -> String {
     result + &buffer
 }
 
+fn format_flowed_prefix(text: &str, prefix: &str) -> String {
+    let mut result = String::new();
+
+    for line in text.split('\n') {
+        if !result.is_empty() {
+            result += "\r\n";
+        }
+        let line = line.trim_end();
+        if prefix.len() + line.len() > 78 {
+            result += &format_line_flowed(line, prefix);
+        } else {
+            result += prefix;
+            if prefix.is_empty() && line.starts_with('>') {
+                // Space stuffing, see RFC 3676
+                result.push(' ');
+            }
+            result += line;
+        }
+    }
+    result
+}
+
 /// Returns text formatted according to RFC 3767 (format=flowed).
 ///
 /// This function accepts text separated by LF, but returns text
@@ -52,20 +81,12 @@ fn format_line_flowed(line: &str) -> String {
 /// RFC 2646 technique is used to insert soft line breaks, so DelSp
 /// SHOULD be set to "no" when sending.
 pub fn format_flowed(text: &str) -> String {
-    let mut result = String::new();
+    format_flowed_prefix(text, "")
+}
 
-    for line in text.split('\n') {
-        if !result.is_empty() {
-            result += "\r\n";
-        }
-        let line = line.trim_end();
-        if line.len() > 78 {
-            result += &format_line_flowed(line);
-        } else {
-            result += line;
-        }
-    }
-    result
+/// Same as format_flowed(), but adds "> " prefix to each line.
+pub fn format_flowed_quote(text: &str) -> String {
+    format_flowed_prefix(text, "> ")
 }
 
 /// Joins lines in format=flowed text.
@@ -122,6 +143,9 @@ mod tests {
                         To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
                         client and enter the setup code presented on the generating device.";
         assert_eq!(format_flowed(text), expected);
+
+        let text = "> Not a quote";
+        assert_eq!(format_flowed(text), " > Not a quote");
     }
 
     #[test]
@@ -133,4 +157,21 @@ mod tests {
                         unwrapped on the receiver";
         assert_eq!(unformat_flowed(text, false), expected);
     }
+
+    #[test]
+    fn test_format_flowed_quote() {
+        let quote = "this is a quoted line";
+        let expected = "> this is a quoted line";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "> foo bar baz";
+        let expected = "> > foo bar baz";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "this is a very long quote that should be wrapped using format=flowed and unwrapped on the receiver";
+        let expected =
+            "> this is a very long quote that should be wrapped using format=flowed and \r\n\
+            > unwrapped on the receiver";
+        assert_eq!(format_flowed_quote(quote), expected);
+    }
 }

src/headerdef.rs

@@ -11,16 +11,26 @@ pub enum HeaderDef {
     To,
     Cc,
     Disposition,
+
+    /// Used in the "Body Part Header" of MDNs as of RFC 8098.
+    /// Indicates the Message-ID of the message for which the MDN is being issued.
     OriginalMessageId,
 
     /// Delta Chat extension for message IDs in combined MDNs
     AdditionalMessageIds,
 
+    /// Outlook-SMTP-server replace the `Message-ID:`-header
+    /// and write the original ID to `X-Microsoft-Original-Message-ID`.
+    /// To sort things correctly and to not show outgoing messages twice,
+    /// we need to check that header as well.
+    XMicrosoftOriginalMessageId,
+
     ListId,
     References,
     InReplyTo,
     Precedence,
     ContentType,
+    ContentId,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -42,7 +52,9 @@ pub enum HeaderDef {
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
     SecureJoinAuth,
+    Sender,
     EphemeralTimer,
+    Received,
     _TestHeader,
 }
 

src/html.rs

@@ -0,0 +1,565 @@
+///! # Get message as HTML.
+///!
+///! Use `Message.has_html()` to check if the UI shall render a
+///! corresponding button and `MsgId.get_html()` to get the full message.
+///!
+///! Even when the original mime-message is not HTML,
+///! `MsgId.get_html()` will return HTML -
+///! this allows nice quoting, handling linebreaks properly etc.
+use futures::future::FutureExt;
+use std::future::Future;
+use std::pin::Pin;
+
+use anyhow::Result;
+use lettre_email::mime::{self, Mime};
+
+use crate::context::Context;
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::message::{Message, MsgId};
+use crate::mimeparser::parse_message_id;
+use crate::param::Param::SendHtml;
+use crate::plaintext::PlainText;
+use lettre_email::PartBuilder;
+use mailparse::ParsedContentType;
+
+impl Message {
+    /// Check if the message can be retrieved as HTML.
+    /// Typically, this is the case, when the mime structure of a Message is modified,
+    /// meaning that some text is cut or the original message
+    /// is in HTML and `simplify()` may hide some maybe important information.
+    /// The corresponding ffi-function is `dc_msg_has_html()`.
+    /// To get the HTML-code of the message, use `MsgId.get_html()`.
+    pub fn has_html(&self) -> bool {
+        self.mime_modified
+    }
+
+    /// Set HTML-part part of a message that is about to be sent.
+    /// The HTML-part is written to the database before sending and
+    /// used as the `text/html` part in the MIME-structure.
+    ///
+    /// Received HTML parts are handled differently,
+    /// they are saved together with the whole MIME-structure
+    /// in `mime_headers` and the HTML-part is extracted using `MsgId::get_html()`.
+    /// (To underline this asynchronicity, we are using the wording "SendHtml")
+    pub fn set_html(&mut self, html: Option<String>) {
+        if let Some(html) = html {
+            self.param.set(SendHtml, html);
+            self.mime_modified = true;
+        } else {
+            self.param.remove(SendHtml);
+            self.mime_modified = false;
+        }
+    }
+}
+
+/// Type defining a rough mime-type.
+/// This is mainly useful on iterating
+/// to decide whether a mime-part has subtypes.
+enum MimeMultipartType {
+    Multiple,
+    Single,
+    Message,
+}
+
+/// Function takes a content type from a ParsedMail structure
+/// and checks and returns the rough mime-type.
+async fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
+    let mimetype = ctype.mimetype.to_lowercase();
+    if mimetype.starts_with("multipart") && ctype.params.get("boundary").is_some() {
+        MimeMultipartType::Multiple
+    } else if mimetype == "message/rfc822" {
+        MimeMultipartType::Message
+    } else {
+        MimeMultipartType::Single
+    }
+}
+
+/// HtmlMsgParser converts a mime-message to HTML.
+#[derive(Debug)]
+struct HtmlMsgParser {
+    pub html: String,
+    pub plain: Option<PlainText>,
+}
+
+impl HtmlMsgParser {
+    /// Function takes a raw mime-message string,
+    /// searches for the main-text part
+    /// and returns that as parser.html
+    pub async fn from_bytes(context: &Context, rawmime: &[u8]) -> Result<Self> {
+        let mut parser = HtmlMsgParser {
+            html: "".to_string(),
+            plain: None,
+        };
+
+        let parsedmail = mailparse::parse_mail(rawmime)?;
+
+        parser.collect_texts_recursive(context, &parsedmail).await?;
+
+        if parser.html.is_empty() {
+            if let Some(plain) = &parser.plain {
+                parser.html = plain.to_html().await;
+            }
+        } else {
+            parser.cid_to_data_recursive(context, &parsedmail).await?;
+        }
+
+        Ok(parser)
+    }
+
+    /// Function iterates over all mime-parts
+    /// and searches for text/plain and text/html parts and saves the
+    /// first one found.
+    /// in the corresponding structure fields.
+    ///
+    /// Usually, there is at most one plain-text and one HTML-text part,
+    /// multiple plain-text parts might be used for mailinglist-footers,
+    /// therefore we use the first one.
+    fn collect_texts_recursive<'a>(
+        &'a mut self,
+        context: &'a Context,
+        mail: &'a mailparse::ParsedMail<'a>,
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
+        // Boxed future to deal with recursion
+        async move {
+            match get_mime_multipart_type(&mail.ctype).await {
+                MimeMultipartType::Multiple => {
+                    for cur_data in mail.subparts.iter() {
+                        self.collect_texts_recursive(context, cur_data).await?
+                    }
+                    Ok(())
+                }
+                MimeMultipartType::Message => {
+                    let raw = mail.get_body_raw()?;
+                    if raw.is_empty() {
+                        return Ok(());
+                    }
+                    let mail = mailparse::parse_mail(&raw).unwrap();
+                    self.collect_texts_recursive(context, &mail).await
+                }
+                MimeMultipartType::Single => {
+                    let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+                    if mimetype == mime::TEXT_HTML {
+                        if self.html.is_empty() {
+                            if let Ok(decoded_data) = mail.get_body() {
+                                self.html = decoded_data;
+                            }
+                        }
+                    } else if mimetype == mime::TEXT_PLAIN && self.plain.is_none() {
+                        if let Ok(decoded_data) = mail.get_body() {
+                            self.plain = Some(PlainText {
+                                text: decoded_data,
+                                flowed: if let Some(format) = mail.ctype.params.get("format") {
+                                    format.as_str().to_ascii_lowercase() == "flowed"
+                                } else {
+                                    false
+                                },
+                                delsp: if let Some(delsp) = mail.ctype.params.get("delsp") {
+                                    delsp.as_str().to_ascii_lowercase() == "yes"
+                                } else {
+                                    false
+                                },
+                            });
+                        }
+                    }
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+
+    /// Replace cid:-protocol by the data:-protocol where appropriate.
+    /// This allows the final html-file to be self-contained.
+    fn cid_to_data_recursive<'a>(
+        &'a mut self,
+        context: &'a Context,
+        mail: &'a mailparse::ParsedMail<'a>,
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
+        // Boxed future to deal with recursion
+        async move {
+            match get_mime_multipart_type(&mail.ctype).await {
+                MimeMultipartType::Multiple => {
+                    for cur_data in mail.subparts.iter() {
+                        self.cid_to_data_recursive(context, cur_data).await?;
+                    }
+                    Ok(())
+                }
+                MimeMultipartType::Message => {
+                    let raw = mail.get_body_raw()?;
+                    if raw.is_empty() {
+                        return Ok(());
+                    }
+                    let mail = mailparse::parse_mail(&raw).unwrap();
+                    self.cid_to_data_recursive(context, &mail).await
+                }
+                MimeMultipartType::Single => {
+                    let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+                    if mimetype.type_() == mime::IMAGE {
+                        if let Some(cid) = mail.headers.get_header_value(HeaderDef::ContentId) {
+                            if let Ok(cid) = parse_message_id(&cid) {
+                                if let Ok(replacement) = mimepart_to_data_url(mail).await {
+                                    let re_string = format!(
+                                        "(<img[^>]*src[^>]*=[^>]*)(cid:{})([^>]*>)",
+                                        regex::escape(&cid)
+                                    );
+                                    match regex::Regex::new(&re_string) {
+                                        Ok(re) => {
+                                            self.html = re
+                                                .replace_all(
+                                                    &*self.html,
+                                                    format!("${{1}}{}${{3}}", replacement).as_str(),
+                                                )
+                                                .as_ref()
+                                                .to_string()
+                                        }
+                                        Err(e) => warn!(
+                                            context,
+                                            "Cannot create regex for cid: {} throws {}",
+                                            re_string,
+                                            e
+                                        ),
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+}
+
+/// Convert a mime part to a data: url as defined in [RFC 2397](https://tools.ietf.org/html/rfc2397).
+async fn mimepart_to_data_url(mail: &mailparse::ParsedMail<'_>) -> Result<String> {
+    let data = mail.get_body_raw()?;
+    let data = base64::encode(&data);
+    Ok(format!("data:{};base64,{}", mail.ctype.mimetype, data))
+}
+
+impl MsgId {
+    /// Get HTML from a message-id.
+    /// This requires `mime_headers` field to be set for the message;
+    /// this is the case at least when `Message.has_html()` returns true
+    /// (we do not save raw mime unconditionally in the database to save space).
+    /// The corresponding ffi-function is `dc_get_msg_html()`.
+    pub async fn get_html(self, context: &Context) -> Option<String> {
+        let rawmime: Option<String> = context
+            .sql
+            .query_get_value(
+                context,
+                "SELECT mime_headers FROM msgs WHERE id=?;",
+                paramsv![self],
+            )
+            .await;
+
+        if let Some(rawmime) = rawmime {
+            if !rawmime.is_empty() {
+                match HtmlMsgParser::from_bytes(context, rawmime.as_bytes()).await {
+                    Err(err) => {
+                        warn!(context, "get_html: parser error: {}", err);
+                        None
+                    }
+                    Ok(parser) => Some(parser.html),
+                }
+            } else {
+                warn!(context, "get_html: empty mime for {}", self);
+                None
+            }
+        } else {
+            warn!(context, "get_html: no mime for {}", self);
+            None
+        }
+    }
+}
+
+/// Wraps HTML text into a new text/html mimepart structure.
+///
+/// Used on forwarding messages to avoid leaking the original mime structure
+/// and also to avoid sending too much, maybe large data.
+pub async fn new_html_mimepart(html: String) -> PartBuilder {
+    PartBuilder::new()
+        .content_type(&"text/html; charset=utf-8".parse::<mime::Mime>().unwrap())
+        .body(html)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chat;
+    use crate::chat::forward_msgs;
+    use crate::config::Config;
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::message::MessengerMessage;
+    use crate::test_utils::TestContext;
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_unspecified() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+This message does not have Content-Type nor Subject.<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_iso88591() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_iso88591.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+message with a non-UTF-8 encoding: äöüßÄÖÜ<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_flowed() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_flowed.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(parser.plain.unwrap().flowed);
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+This line ends with a space and will be merged with the next one due to format=flowed.<br/>
+<br/>
+This line does not end with a space<br/>
+and will be wrapped as usual.<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_plain() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+mime-modified should not be set set as there is no html and no special stuff;<br/>
+although not being a delta-message.<br/>
+test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x27; :)<br/>
+<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+
+        // on windows, `\r\n` linends are returned from mimeparser,
+        // however, rust multiline-strings use just `\n`;
+        // therefore, we just remove `\r` before comparison.
+        assert_eq!(
+            parser.html.replace("\r", ""),
+            r##"
+<html>
+  <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
+</html>"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            r##"<html>
+  <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
+</html>
+
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_plain_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            r##"<html>
+  <p>
+    this is <b>html</b>
+  </p>
+</html>
+
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_apple_cid_jpg() {
+        // load raw mime html-data with related image-part (cid:)
+        // and make sure, Content-Id has angle-brackets that are removed correctly.
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/apple_cid_jpg.eml");
+        let test = String::from_utf8_lossy(raw);
+        assert!(test.contains("Content-Id: <8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box>"));
+        assert!(test.contains("cid:8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box"));
+        assert!(test.find("data:").is_none());
+
+        // parsing converts cid: to data:
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(parser.html.contains("<html>"));
+        assert!(!parser.html.contains("Content-Id:"));
+        assert!(parser.html.contains("data:image/jpeg;base64,/9j/4AAQ"));
+        assert!(!parser.html.contains("cid:"));
+    }
+
+    #[async_std::test]
+    async fn test_get_html_empty() {
+        let t = TestContext::new().await;
+        let msg_id = MsgId::new_unset();
+        assert!(msg_id.get_html(&t).await.is_none())
+    }
+
+    #[async_std::test]
+    async fn test_html_forwarding() {
+        // alice receives a non-delta html-message
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("2")).await.ok();
+        let chat = alice
+            .create_chat_with_contact("", "sender@testrun.org")
+            .await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::No);
+        assert!(!msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+
+        // alice: create chat with bob and forward received html-message there
+        let chat = alice.create_chat_with_contact("", "bob@example.net").await;
+        forward_msgs(&alice, &[msg.get_id()], chat.get_id())
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+
+        // bob: check that bob also got the html-part of the forwarded message
+        let bob = TestContext::new_bob().await;
+        let chat = bob.create_chat_with_contact("", "alice@example.com").await;
+        bob.recv_msg(&alice.pop_sent_msg().await).await;
+        let msg = bob.get_last_msg_in(chat.get_id()).await;
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&bob).await.unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+    }
+
+    #[async_std::test]
+    async fn test_html_forwarding_encrypted() {
+        // Alice receives a non-delta html-message
+        // (`ShowEmails=1` lets Alice actually receive non-delta messages for known contacts,
+        // the contact is marked as known by creating a chat using `chat_with_contact()`)
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("1")).await.ok();
+        let chat = alice
+            .create_chat_with_contact("", "sender@testrun.org")
+            .await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+
+        // forward the message to saved-messages,
+        // this will encrypt the message as new_alice() has set up keys
+        let chat = alice.get_self_chat().await;
+        forward_msgs(&alice, &[msg.get_id()], chat.get_id())
+            .await
+            .unwrap();
+        let msg = alice.pop_sent_msg().await;
+
+        // receive the message on another device
+        let alice = TestContext::new_alice().await;
+        assert_eq!(alice.get_config_int(Config::ShowEmails).await, 0); // set to "1" above, make sure it is another db
+        alice.recv_msg(&msg).await;
+        let chat = alice.get_self_chat().await;
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.get_showpadlock());
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+    }
+
+    #[async_std::test]
+    async fn test_set_html() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // alice sends a message with html-part to bob
+        let chat_id = alice.create_chat(&bob).await.id;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("plain text".to_string()));
+        msg.set_html(Some("<b>html</b> text".to_string()));
+        assert!(msg.mime_modified);
+        chat::send_msg(&alice, chat_id, &mut msg).await.unwrap();
+
+        // check the message is written correctly to alice's db
+        let msg = alice.get_last_msg_in(chat_id).await;
+        assert_eq!(msg.get_text(), Some("plain text".to_string()));
+        assert!(!msg.is_forwarded());
+        assert!(msg.mime_modified);
+        let html = msg.get_id().get_html(&alice).await.unwrap();
+        assert!(html.contains("<b>html</b> text"));
+
+        // let bob receive the message
+        let chat_id = bob.create_chat(&alice).await.id;
+        bob.recv_msg(&alice.pop_sent_msg().await).await;
+        let msg = bob.get_last_msg_in(chat_id).await;
+        assert_eq!(msg.get_text(), Some("plain text".to_string()));
+        assert!(!msg.is_forwarded());
+        assert!(msg.mime_modified);
+        let html = msg.get_id().get_html(&bob).await.unwrap();
+        assert!(html.contains("<b>html</b> text"));
+    }
+}

src/imap/idle.rs

@@ -1,11 +1,11 @@
 use super::Imap;
 
+use anyhow::{bail, format_err, Result};
 use async_imap::extensions::idle::IdleResponse;
 use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
-use crate::error::{bail, format_err, Result};
 use crate::{context::Context, scheduler::InterruptInfo};
 
 use super::session::Session;
@@ -50,6 +50,12 @@ impl Imap {
                 return Ok(info);
             }
 
+            if let Ok(info) = self.idle_interrupt.try_recv() {
+                info!(context, "skip idle, got interrupt {:?}", info);
+                self.session = Some(session);
+                return Ok(info);
+            }
+
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
                 bail!("IMAP IDLE protocol failed to init/complete: {}", err);
@@ -62,14 +68,18 @@ impl Imap {
                 Interrupt(InterruptInfo),
             }
 
-            info!(context, "Idle entering wait-on-remote state");
+            info!(
+                context,
+                "{}: Idle entering wait-on-remote state",
+                watch_folder.as_deref().unwrap_or("None")
+            );
             let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
-                let probe_network = self.idle_interrupt.recv().await;
+                let info = self.idle_interrupt.recv().await;
 
                 // cancel imap idle connection properly
                 drop(interrupt);
 
-                Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+                Ok(Event::Interrupt(info.unwrap_or_default()))
             });
 
             match fut.await {
@@ -113,12 +123,15 @@ impl Imap {
         // in this case, we're waiting for a configure job (and an interrupt).
 
         let fake_idle_start_time = SystemTime::now();
-        info!(context, "IMAP-fake-IDLEing...");
 
         // Do not poll, just wait for an interrupt when no folder is passed in.
-        if watch_folder.is_none() {
+        let watch_folder = if let Some(watch_folder) = watch_folder {
+            watch_folder
+        } else {
+            info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
             return self.idle_interrupt.recv().await.unwrap_or_default();
-        }
+        };
+        info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
         // check every minute if there are new messages
         // TODO: grow sleep durations / make them more flexible
@@ -159,19 +172,17 @@ impl Imap {
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
 
-                    if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_new_messages(context, watch_folder).await {
-                            Ok(res) => {
-                                info!(context, "fetch_new_messages returned {:?}", res);
-                                if res {
-                                    break InterruptInfo::new(false, None);
-                                }
-                            }
-                            Err(err) => {
-                                error!(context, "could not fetch from folder: {}", err);
-                                self.trigger_reconnect()
+                    match self.fetch_new_messages(context, &watch_folder, false).await {
+                        Ok(res) => {
+                            info!(context, "fetch_new_messages returned {:?}", res);
+                            if res {
+                                break InterruptInfo::new(false, None);
                             }
                         }
+                        Err(err) => {
+                            error!(context, "could not fetch from folder: {:#}", err);
+                            self.trigger_reconnect()
+                        }
                     }
                 }
                 Event::Interrupt(info) => {

src/imap/scan_folders.rs

@@ -0,0 +1,105 @@
+use std::time::Instant;
+
+use anyhow::{Context as _, Result};
+
+use crate::config::Config;
+use crate::context::Context;
+use crate::imap::Imap;
+use async_std::prelude::*;
+
+use super::{get_folder_meaning, get_folder_meaning_by_name, FolderMeaning};
+
+impl Imap {
+    pub async fn scan_folders(&mut self, context: &Context) -> Result<()> {
+        // First of all, debounce to once per minute:
+        let mut last_scan = context.last_full_folder_scan.lock().await;
+        if let Some(last_scan) = *last_scan {
+            let elapsed_secs = last_scan.elapsed().as_secs();
+            let debounce_secs = context
+                .get_config_u64(Config::ScanAllFoldersDebounceSecs)
+                .await;
+
+            if elapsed_secs < debounce_secs {
+                return Ok(());
+            }
+        }
+        info!(context, "Starting full folder scan");
+
+        self.setup_handle(context).await?;
+        let session = self.session.as_mut();
+        let session = session.context("scan_folders(): IMAP No Connection established")?;
+        let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
+        let watched_folders = get_watched_folders(context).await;
+
+        let mut sentbox_folder = None;
+        let mut spam_folder = None;
+
+        for folder in folders {
+            let folder = match folder {
+                Ok(f) => f,
+                Err(e) => {
+                    warn!(context, "Can't get folder: {}", e);
+                    continue;
+                }
+            };
+
+            let foldername = folder.name();
+            let folder_meaning = get_folder_meaning(&folder);
+            let folder_name_meaning = get_folder_meaning_by_name(foldername);
+
+            if folder_meaning == FolderMeaning::SentObjects {
+                // Always takes precedent
+                sentbox_folder = Some(folder.name().to_string());
+            } else if folder_meaning == FolderMeaning::Spam {
+                spam_folder = Some(folder.name().to_string());
+            } else if folder_name_meaning == FolderMeaning::SentObjects {
+                // only set iff none has been already set
+                if sentbox_folder.is_none() {
+                    sentbox_folder = Some(folder.name().to_string());
+                }
+            } else if folder_name_meaning == FolderMeaning::Spam && spam_folder.is_none() {
+                spam_folder = Some(folder.name().to_string());
+            }
+
+            if watched_folders.contains(&foldername.to_string()) {
+                info!(
+                    context,
+                    "Not scanning folder {} as it is watched anyway", foldername
+                );
+            } else {
+                info!(context, "Scanning folder: {}", foldername);
+
+                if let Err(e) = self.fetch_new_messages(context, foldername, false).await {
+                    warn!(context, "Can't fetch new msgs in scanned folder: {:#}", e);
+                }
+            }
+        }
+
+        context
+            .set_config(Config::ConfiguredSentboxFolder, sentbox_folder.as_deref())
+            .await?;
+        context
+            .set_config(Config::ConfiguredSpamFolder, spam_folder.as_deref())
+            .await?;
+
+        last_scan.replace(Instant::now());
+        Ok(())
+    }
+}
+
+async fn get_watched_folders(context: &Context) -> Vec<String> {
+    let mut res = Vec::new();
+    let folder_watched_configured = &[
+        (Config::SentboxWatch, Config::ConfiguredSentboxFolder),
+        (Config::MvboxWatch, Config::ConfiguredMvboxFolder),
+        (Config::InboxWatch, Config::ConfiguredInboxFolder),
+    ];
+    for (watched, configured) in folder_watched_configured {
+        if context.get_config_bool(*watched).await {
+            if let Some(folder) = context.get_config(*configured).await {
+                res.push(folder);
+            }
+        }
+    }
+    res
+}

src/imap/select_folder.rs

@@ -61,11 +61,12 @@ impl Imap {
 
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
+    /// Returns whether a new folder was selected.
     pub(super) async fn select_folder<S: AsRef<str>>(
         &mut self,
         context: &Context,
         folder: Option<S>,
-    ) -> Result<()> {
+    ) -> Result<NewlySelected> {
         if self.session.is_none() {
             self.config.selected_folder = None;
             self.config.selected_folder_needs_expunge = false;
@@ -78,7 +79,7 @@ impl Imap {
         if let Some(ref folder) = folder {
             if let Some(ref selected_folder) = self.config.selected_folder {
                 if folder.as_ref() == selected_folder {
-                    return Ok(());
+                    return Ok(NewlySelected::No);
                 }
             }
         }
@@ -99,7 +100,7 @@ impl Imap {
                     Ok(mailbox) => {
                         self.config.selected_folder = Some(folder.as_ref().to_string());
                         self.config.selected_mailbox = Some(mailbox);
-                        Ok(())
+                        Ok(NewlySelected::Yes)
                     }
                     Err(async_imap::error::Error::ConnectionLost) => {
                         self.trigger_reconnect();
@@ -119,7 +120,15 @@ impl Imap {
                 Err(Error::NoSession)
             }
         } else {
-            Ok(())
+            Ok(NewlySelected::No)
         }
     }
 }
+#[derive(PartialEq, Debug, Copy, Clone, Eq)]
+pub(super) enum NewlySelected {
+    /// The folder was newly selected during this call to select_folder().
+    Yes,
+    /// No SELECT command was run because the folder already was selected
+    /// and self.config.selected_mailbox was not updated (so, e.g. it may contain an outdated uid_next)
+    No,
+}

src/imex.rs

@@ -1,11 +1,9 @@
 //! # Import/export module
 
 use std::any::Any;
-use std::{
-    cmp::{max, min},
-    ffi::OsStr,
-};
+use std::ffi::OsStr;
 
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use async_std::path::{Path, PathBuf};
 use async_std::{
     fs::{self, File},
@@ -13,23 +11,26 @@ use async_std::{
 };
 use rand::{thread_rng, Rng};
 
-use crate::blob::BlobObject;
 use crate::chat;
 use crate::chat::delete_and_reset_all_device_msgs;
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{
+    dc_copy_file, dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
+    dc_open_file_std, dc_read_file, dc_write_file, get_next_backup_path, time, EmailAddress,
+};
 use crate::e2ee;
-use crate::error::*;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::param::*;
+use crate::param::Param;
 use crate::pgp;
 use crate::sql::{self, Sql};
-use crate::stock::StockMessage;
+use crate::stock_str;
+use crate::{blob::BlobObject, log::LogExt};
+use ::pgp::types::KeyTrait;
 use async_tar::Archive;
 
 // Name of the database file in the backup.
@@ -77,11 +78,7 @@ pub enum ImexMode {
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
-pub async fn imex(
-    context: &Context,
-    what: ImexMode,
-    param1: Option<impl AsRef<Path>>,
-) -> Result<()> {
+pub async fn imex(context: &Context, what: ImexMode, param1: impl AsRef<Path>) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
 
     let res = async {
@@ -94,7 +91,8 @@ pub async fn imex(
             }
             Err(err) => {
                 cleanup_aborted_imex(context, what).await;
-                error!(context, "{}", err);
+                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                error!(context, "{:#}", err);
                 context.emit_event(EventType::ImexProgress(0));
                 bail!("IMEX FAILED to complete: {}", err);
             }
@@ -118,7 +116,9 @@ async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
         dc_delete_files_in_dir(context, context.get_blobdir()).await;
     }
     if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
-        context.sql.open(context, context.get_dbfile(), false).await;
+        if let Err(e) = context.sql.open(context, context.get_dbfile(), false).await {
+            warn!(context, "Re-opening db after imex failed: {}", e);
+        }
     }
 }
 
@@ -158,6 +158,7 @@ pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Re
     let dir_name = dir_name.as_ref();
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
+    let mut newest_backup_name = "".to_string();
     let mut newest_backup_path: Option<PathBuf> = None;
     while let Some(dirent) = dir_iter.next().await {
         if let Ok(dirent) = dirent {
@@ -166,17 +167,35 @@ pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Re
             let name = name.to_string_lossy();
             if name.starts_with("delta-chat") && name.ends_with(".bak") {
                 let sql = Sql::new();
-                if sql.open(context, &path, true).await {
-                    let curr_backup_time = sql
-                        .get_raw_config_int(context, "backup_time")
-                        .await
-                        .unwrap_or_default();
-                    if curr_backup_time > newest_backup_time {
-                        newest_backup_path = Some(path);
-                        newest_backup_time = curr_backup_time;
+                match sql.open(context, &path, true).await {
+                    Ok(_) => {
+                        let curr_backup_time = sql
+                            .get_raw_config_int(context, "backup_time")
+                            .await
+                            .unwrap_or_default();
+                        if curr_backup_time > newest_backup_time {
+                            newest_backup_path = Some(path);
+                            newest_backup_time = curr_backup_time;
+                        }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close().await;
+                    }
+                    Err(e) => {
+                        warn!(
+                            context,
+                            "Found backup file {} which could not be opened: {}", name, e
+                        );
+                        // On some Android devices we can't open sql files that are not in our private directory
+                        // (see https://github.com/deltachat/deltachat-android/issues/1768). So, compare names
+                        // to still find the newest backup.
+                        let name: String = name.into();
+                        if newest_backup_time == 0
+                            && (newest_backup_name.is_empty() || name > newest_backup_name)
+                        {
+                            newest_backup_path = Some(path);
+                            newest_backup_name = name;
+                        }
                     }
-                    info!(context, "backup_time of {} is {}", name, curr_backup_time);
-                    sql.close().await;
                 }
             }
         }
@@ -257,7 +276,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
     };
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes()).await?;
+    let encr = pgp::symm_encrypt(passphrase, private_key_asc.as_bytes()).await?;
 
     let replacement = format!(
         concat!(
@@ -269,8 +288,8 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
-    let msg_subj = context.stock_str(StockMessage::AcSetupMsgSubject).await;
-    let msg_body = context.stock_str(StockMessage::AcSetupMsgBody).await;
+    let msg_subj = stock_str::ac_setup_msg_subject(context).await;
+    let msg_body = stock_str::ac_setup_msg_body(context).await;
     let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
     Ok(format!(
         concat!(
@@ -403,6 +422,8 @@ async fn set_self_key(
         },
     )
     .await?;
+
+    info!(context, "stored self key: {:?}", keypair.secret.key_id());
     Ok(())
 }
 
@@ -419,7 +440,7 @@ async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
 pub fn normalize_setup_code(s: &str) -> String {
     let mut out = String::new();
     for c in s.chars() {
-        if c >= '0' && c <= '9' {
+        if ('0'..='9').contains(&c) {
             out.push(c);
             if let 4 | 9 | 14 | 19 | 24 | 29 | 34 | 39 = out.len() {
                 out += "-"
@@ -429,19 +450,11 @@ pub fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-async fn imex_inner(
-    context: &Context,
-    what: ImexMode,
-    param: Option<impl AsRef<Path>>,
-) -> Result<()> {
-    ensure!(param.is_some(), "No Import/export dir/file given.");
-
-    info!(context, "Import/export process started.");
+async fn imex_inner(context: &Context, what: ImexMode, path: impl AsRef<Path>) -> Result<()> {
+    info!(context, "Import/export dir: {}", path.as_ref().display());
+    ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
 
-    ensure!(context.sql.is_open().await, "Database not opened.");
-
-    let path = param.ok_or_else(|| format_err!("Imex: Param was None"))?;
     if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
         if e2ee::ensure_secret_key_exists(context).await.is_err() {
@@ -455,9 +468,7 @@ async fn imex_inner(
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
 
-        // TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old().
-        // (now is 07/2020)
-        ImexMode::ExportBackup => export_backup_old(context, path).await,
+        ImexMode::ExportBackup => export_backup(context, path).await,
         // import_backup() will call import_backup_old() if this is an old backup.
         ImexMode::ImportBackup => import_backup(context, path).await,
     }
@@ -485,6 +496,10 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         !context.is_configured().await,
         "Cannot import backups to accounts in use."
     );
+    ensure!(
+        !context.scheduler.read().await.is_running(),
+        "cannot import backup, IO already running"
+    );
     context.sql.close().await;
     dc_delete_file(context, context.get_dbfile()).await;
     ensure!(
@@ -493,10 +508,20 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     );
 
     let backup_file = File::open(backup_to_import).await?;
+    let file_size = backup_file.metadata().await?.len();
     let archive = Archive::new(backup_file);
+
     let mut entries = archive.entries()?;
     while let Some(file) = entries.next().await {
         let f = &mut file?;
+
+        let current_pos = f.raw_file_position();
+        let progress = 1000 * current_pos / file_size;
+        if progress > 10 && progress < 1000 {
+            // We already emitted ImexProgress(10) above
+            context.emit_event(EventType::ImexProgress(progress as usize));
+        }
+
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
             // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
             f.unpack_in(context.get_blobdir()).await?;
@@ -505,7 +530,6 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
                 context.get_dbfile(),
             )
             .await?;
-            context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
         } else {
             // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
             f.unpack_in(context.get_blobdir()).await?;
@@ -520,15 +544,13 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         }
     }
 
-    ensure!(
-        context
-            .sql
-            .open(&context, &context.get_dbfile(), false)
-            .await,
-        "could not re-open db"
-    );
+    context
+        .sql
+        .open(context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
 
-    delete_and_reset_all_device_msgs(&context).await?;
+    delete_and_reset_all_device_msgs(context).await?;
 
     Ok(())
 }
@@ -545,6 +567,10 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
         !context.is_configured().await,
         "Cannot import backups to accounts in use."
     );
+    ensure!(
+        !context.scheduler.read().await.is_running(),
+        "cannot import backup, IO already running"
+    );
     context.sql.close().await;
     dc_delete_file(context, context.get_dbfile()).await;
     ensure!(
@@ -558,15 +584,13 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
     );
     /* error already logged */
     /* re-open copied database file */
-    ensure!(
-        context
-            .sql
-            .open(&context, &context.get_dbfile(), false)
-            .await,
-        "could not re-open db"
-    );
+    context
+        .sql
+        .open(context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
 
-    delete_and_reset_all_device_msgs(&context).await?;
+    delete_and_reset_all_device_msgs(context).await?;
 
     let total_files_cnt = context
         .sql
@@ -645,14 +669,14 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
 async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
-    let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+    let (temp_path, dest_path) = get_next_backup_path(dir, now).await?;
     let _d = DeleteOnDrop(temp_path.clone());
 
     context
         .sql
         .set_raw_config_int(context, "backup_time", now as i32)
         .await?;
-    sql::housekeeping(context).await;
+    sql::housekeeping(context).await.ok_or_log(context);
 
     context
         .sql
@@ -660,6 +684,11 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         .await
         .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
 
+    ensure!(
+        !context.scheduler.read().await.is_running(),
+        "cannot export backup, IO already running"
+    );
+
     // we close the database during the export
     context.sql.close().await;
 
@@ -675,7 +704,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // we re-open the database after export is finished
     context
         .sql
-        .open(&context, &context.get_dbfile(), false)
+        .open(context, &context.get_dbfile(), false)
         .await;
 
     match &res {
@@ -709,131 +738,37 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
         .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
         .await?;
 
-    context.emit_event(EventType::ImexProgress(500));
+    let read_dir: Vec<_> = fs::read_dir(context.get_blobdir()).await?.collect().await;
+    let count = read_dir.len();
+    let mut written_files = 0;
 
-    builder
-        .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
-        .await?;
+    for entry in read_dir.into_iter() {
+        let entry = entry?;
+        let name = entry.file_name();
+        if !entry.file_type().await?.is_file() {
+            warn!(
+                context,
+                "Export: Found dir entry {} that is not a file, ignoring",
+                name.to_string_lossy()
+            );
+            continue;
+        }
+        let mut file = File::open(entry.path()).await?;
+        let path_in_archive = PathBuf::from(BLOBS_BACKUP_NAME).join(name);
+        builder.append_file(path_in_archive, &mut file).await?;
+
+        written_files += 1;
+        let progress = 1000 * written_files / count;
+        if progress > 10 && progress < 1000 {
+            // We already emitted ImexProgress(10) above
+            emit_event!(context, EventType::ImexProgress(progress));
+        }
+    }
 
     builder.finish().await?;
     Ok(())
 }
 
-async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
-    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
-    // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
-    // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
-    let now = time();
-    let dest_path_filename = get_next_backup_path_old(dir, now).await?;
-    let dest_path_string = dest_path_filename.to_string_lossy().to_string();
-
-    sql::housekeeping(context).await;
-
-    context.sql.execute("VACUUM;", paramsv![]).await.ok();
-
-    // we close the database during the copy of the dbfile
-    context.sql.close().await;
-    info!(
-        context,
-        "Backup '{}' to '{}'.",
-        context.get_dbfile().display(),
-        dest_path_filename.display(),
-    );
-    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename).await;
-    context
-        .sql
-        .open(&context, &context.get_dbfile(), false)
-        .await;
-
-    if !copied {
-        bail!(
-            "could not copy file from '{}' to '{}'",
-            context.get_dbfile().display(),
-            dest_path_string
-        );
-    }
-    let dest_sql = Sql::new();
-    ensure!(
-        dest_sql.open(context, &dest_path_filename, false).await,
-        "could not open exported database {}",
-        dest_path_string
-    );
-    let res = match add_files_to_export(context, &dest_sql).await {
-        Err(err) => {
-            dc_delete_file(context, &dest_path_filename).await;
-            error!(context, "backup failed: {}", err);
-            Err(err)
-        }
-        Ok(()) => {
-            dest_sql
-                .set_raw_config_int(context, "backup_time", now as i32)
-                .await?;
-            context.emit_event(EventType::ImexFileWritten(dest_path_filename));
-            Ok(())
-        }
-    };
-    dest_sql.close().await;
-
-    Ok(res?)
-}
-
-async fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
-    // add all files as blobs to the database copy (this does not require
-    // the source to be locked, neigher the destination as it is used only here)
-    if !sql.table_exists("backup_blobs").await? {
-        sql.execute(
-            "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
-            paramsv![],
-        )
-        .await?;
-    }
-    // copy all files from BLOBDIR into backup-db
-    let mut total_files_cnt = 0;
-    let dir = context.get_blobdir();
-    let dir_handle = async_std::fs::read_dir(&dir).await?;
-    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count().await;
-
-    info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
-
-    sql.with_conn_async(|conn| async move {
-        // scan directory, pass 2: copy files
-        let mut dir_handle = async_std::fs::read_dir(&dir).await?;
-
-        let mut processed_files_cnt = 0;
-        while let Some(entry) = dir_handle.next().await {
-            let entry = entry?;
-            if context.shall_stop_ongoing().await {
-                return Ok(());
-            }
-            processed_files_cnt += 1;
-            let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
-            context.emit_event(EventType::ImexProgress(permille));
-
-            let name_f = entry.file_name();
-            let name = name_f.to_string_lossy();
-            if name.starts_with("delta-chat") && name.ends_with(".bak") {
-                continue;
-            }
-            info!(context, "EXPORT: copying filename={}", name);
-            let curr_path_filename = context.get_blobdir().join(entry.file_name());
-            if let Ok(buf) = dc_read_file(context, &curr_path_filename).await {
-                if buf.is_empty() {
-                    continue;
-                }
-                // bail out if we can't insert
-                let mut stmt = conn.prepare_cached(
-                    "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
-                )?;
-                stmt.execute(paramsv![name, buf])?;
-            }
-        }
-        Ok(())
-    })
-    .await?;
-
-    Ok(())
-}
-
 /*******************************************************************************
  * Classic key import
  ******************************************************************************/
@@ -869,6 +804,12 @@ async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
                 continue;
             }
         }
+        info!(
+            context,
+            "considering key file: {}",
+            path_plus_name.display()
+        );
+
         match dc_read_file(context, &path_plus_name).await {
             Ok(buf) => {
                 let armored = std::string::String::from_utf8_lossy(&buf);
@@ -958,7 +899,7 @@ where
         let any_key = key as &dyn Any;
         let kind = if any_key.downcast_ref::<SignedPublicKey>().is_some() {
             "public"
-        } else if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+        } else if any_key.downcast_ref::<SignedSecretKey>().is_some() {
             "private"
         } else {
             "unknown"
@@ -966,7 +907,12 @@ where
         let id = id.map_or("default".into(), |i| i.to_string());
         dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
     };
-    info!(context, "Exporting key {}", file_name.display());
+    info!(
+        context,
+        "Exporting key {:?} to {}",
+        key.key_id(),
+        file_name.display()
+    );
     dc_delete_file(context, &file_name).await;
 
     let content = key.to_asc(None).into_bytes();
@@ -982,8 +928,11 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+
     use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
-    use crate::test_utils::*;
+    use crate::stock_str::StockMessage;
+    use crate::test_utils::{alice_keypair, TestContext};
+
     use ::pgp::armor::BlockType;
 
     #[async_std::test]
@@ -991,7 +940,7 @@ mod tests {
         let t = TestContext::new().await;
 
         t.configure_alice().await;
-        let msg = render_setup_file(&t.ctx, "hello").await.unwrap();
+        let msg = render_setup_file(&t, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
         // In particular note the mixing of `\r\n` and `\n` depending
@@ -1008,12 +957,11 @@ mod tests {
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
+        t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
         t.configure_alice().await;
-        let msg = render_setup_file(&t.ctx, "pw").await.unwrap();
+        let msg = render_setup_file(&t, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
     }
@@ -1021,7 +969,7 @@ mod tests {
     #[async_std::test]
     async fn test_create_setup_code() {
         let t = TestContext::new().await;
-        let setupcode = create_setup_code(&t.ctx);
+        let setupcode = create_setup_code(&t);
         assert_eq!(setupcode.len(), 44);
         assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
         assert_eq!(setupcode.chars().nth(9).unwrap(), '-');
@@ -1034,7 +982,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_export_key_to_asc_file() {
+    async fn test_export_public_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
@@ -1048,6 +996,37 @@ mod tests {
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
+    #[async_std::test]
+    async fn test_export_private_key_to_asc_file() {
+        let context = TestContext::new().await;
+        let key = alice_keypair().secret;
+        let blobdir = "$BLOBDIR";
+        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
+            .await
+            .is_ok());
+        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        let filename = format!("{}/private-key-default.asc", blobdir);
+        let bytes = async_std::fs::read(&filename).await.unwrap();
+
+        assert_eq!(bytes, key.to_asc(None).into_bytes());
+    }
+
+    #[async_std::test]
+    async fn test_export_and_import_key() {
+        let context = TestContext::new().await;
+        context.configure_alice().await;
+        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir).await {
+            panic!("got error on export: {:?}", err);
+        }
+
+        let context2 = TestContext::new().await;
+        context2.configure_alice().await;
+        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir).await {
+            panic!("got error on import: {:?}", err);
+        }
+    }
+
     #[test]
     fn test_normalize_setup_code() {
         let norm = normalize_setup_code("123422343234423452346234723482349234");

src/job.rs

@@ -2,41 +2,43 @@
 //!
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
-
-use std::fmt;
 use std::future::Future;
+use std::{fmt, time::Duration};
 
+use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
+use async_smtp::smtp::response::{Category, Code, Detail};
+use async_std::task::sleep;
 use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
 
-use async_smtp::smtp::response::Category;
-use async_smtp::smtp::response::Code;
-use async_smtp::smtp::response::Detail;
-
-use crate::blob::BlobObject;
-use crate::chat::{self, ChatId};
-use crate::config::Config;
-use crate::contact::Contact;
-use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_delete_file, dc_read_file, time};
 use crate::ephemeral::load_imap_deletion_msgid;
-use crate::error::{bail, ensure, format_err, Error, Result};
 use crate::events::EventType;
-use crate::imap::*;
+use crate::imap::{Imap, ImapActionResult};
 use crate::location;
 use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
-use crate::param::*;
+use crate::param::{Param, Params};
 use crate::smtp::Smtp;
+use crate::{blob::BlobObject, contact::normalize_name, contact::Modifier, contact::Origin};
+use crate::{
+    chat::{self, Chat, ChatId, ChatItem},
+    constants::DC_CHAT_ID_DEADDROP,
+};
+use crate::{config::Config, constants::Blocked};
+use crate::{constants::Chattype, contact::Contact};
+use crate::{context::Context, log::LogExt};
 use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
 
 /// Thread IDs
-#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(i32)]
 pub(crate) enum Thread {
     Unknown = 0,
@@ -92,6 +94,7 @@ pub enum Action {
 
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
+    FetchExistingMsgs = 110,
     MarkseenMsgOnImap = 130,
 
     // Moving message is prioritized lower than deletion so we don't
@@ -124,6 +127,7 @@ impl From<Action> for Thread {
             Unknown => Thread::Unknown,
 
             Housekeeping => Thread::Imap,
+            FetchExistingMsgs => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
             ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
@@ -246,7 +250,7 @@ impl Job {
             info!(context, "smtp-sending out mime message:");
             println!("{}", String::from_utf8_lossy(&message));
         }
-        match smtp.send(context, recipients, message, job_id).await {
+        let status = match smtp.send(context, recipients, message, job_id).await {
             Err(crate::smtp::send::Error::SendError(err)) => {
                 // Remote error, retry later.
                 warn!(context, "SMTP failed to send: {}", err);
@@ -271,7 +275,7 @@ impl Job {
                                 // Other enhanced status codes, such as Postfix
                                 // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
                                 // are not ignored.
-                                response.message.get(0) == Some(&"5.5.0".to_string())
+                                response.first_word() == Some(&"5.5.0".to_string())
                             }
                             _ => false,
                         };
@@ -283,25 +287,30 @@ impl Job {
                             // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
                             // should definitely go here, because user has to open the link to
                             // resume message sending.
-                            let msg_id = MsgId::new(self.foreign_id);
-                            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
-                            match Message::load_from_db(context, msg_id).await {
-                                Ok(message) => {
-                                    chat::add_info_msg(context, message.chat_id, err.to_string())
-                                        .await
-                                }
-                                Err(e) => error!(
-                                    context,
-                                    "couldn't load chat_id to inform user about SMTP error: {}", e
-                                ),
-                            };
                             Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                         }
                     }
-                    async_smtp::smtp::error::Error::Transient(_) => {
+                    async_smtp::smtp::error::Error::Transient(ref response) => {
                         // We got a transient 4xx response from SMTP server.
                         // Give some time until the server-side error maybe goes away.
-                        Status::RetryLater
+
+                        if let Some(first_word) = response.first_word() {
+                            if first_word.ends_with(".1.1")
+                                || first_word.ends_with(".1.2")
+                                || first_word.ends_with(".1.3")
+                            {
+                                // Sometimes we receive transient errors that should be permanent.
+                                // Any extended smtp status codes like x.1.1, x.1.2 or x.1.3 that we
+                                // receive as a transient error are misconfigurations of the smtp server.
+                                // See https://tools.ietf.org/html/rfc3463#section-3.2
+                                info!(context, "Smtp-job #{} Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", self.job_id, first_word);
+                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } else {
+                                Status::RetryLater
+                            }
+                        } else {
+                            Status::RetryLater
+                        }
                     }
                     _ => {
                         if smtp.has_maybe_stale_connection().await {
@@ -334,7 +343,14 @@ impl Job {
                 job_try!(success_cb().await);
                 Status::Finished(Ok(()))
             }
+        };
+
+        if let Status::Finished(Err(err)) = &status {
+            // We couldn't send the message, so mark it as failed
+            let msg_id = MsgId::new(self.foreign_id);
+            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
         }
+        status
     }
 
     pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
@@ -475,7 +491,7 @@ impl Job {
         let msg = job_try!(Message::load_from_db(context, msg_id).await);
         let mimefactory =
             job_try!(MimeFactory::from_mdn(context, &msg, additional_rfc724_mids).await);
-        let rendered_msg = job_try!(mimefactory.render().await);
+        let rendered_msg = job_try!(mimefactory.render(context).await);
         let body = rendered_msg.message;
 
         let addr = contact.get_addr();
@@ -506,11 +522,27 @@ impl Job {
         }
 
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await;
+        let server_folder = &job_try!(msg
+            .server_folder
+            .context("Can't move message out of folder if we don't know the current folder"));
+
+        let move_res = msg.id.needs_move(context, server_folder).await;
+        let dest_folder = match move_res {
+            Err(e) => {
+                warn!(context, "could not load dest folder: {}", e);
+                return Status::RetryLater;
+            }
+            Ok(None) => {
+                warn!(
+                    context,
+                    "msg {} does not need to be moved from {}", msg.id, server_folder
+                );
+                return Status::Finished(Ok(()));
+            }
+            Ok(Some(config)) => context.get_config(config).await,
+        };
 
         if let Some(dest_folder) = dest_folder {
-            let server_folder = msg.server_folder.as_ref().unwrap();
-
             match imap
                 .mv(context, server_folder, msg.server_uid, &dest_folder)
                 .await
@@ -619,6 +651,76 @@ impl Job {
         }
     }
 
+    /// Read the recipients from old emails sent by the user and add them as contacts.
+    /// This way, we can already offer them some email addresses they can write to.
+    ///
+    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
+    /// and show them in the chat list.
+    async fn fetch_existing_msgs(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if context.get_config_bool(Config::Bot).await {
+            return Status::Finished(Ok(())); // Bots don't want those messages
+        }
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredSentboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredMvboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredInboxFolder).await;
+
+        if context.get_config_bool(Config::FetchExistingMsgs).await {
+            for config in &[
+                Config::ConfiguredMvboxFolder,
+                Config::ConfiguredInboxFolder,
+                Config::ConfiguredSentboxFolder,
+            ] {
+                if let Some(folder) = context.get_config(*config).await {
+                    if let Err(e) = imap.fetch_new_messages(context, folder, true).await {
+                        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                        warn!(context, "Could not fetch messages, retrying: {:#}", e);
+                        return Status::RetryLater;
+                    };
+                }
+            }
+        }
+
+        // Make sure that if there now is a chat with a contact (created by an outgoing
+        // message), then group contact requests from this contact should also be unblocked.
+        // See https://github.com/deltachat/deltachat-core-rust/issues/2097.
+        for item in chat::get_chat_msgs(context, ChatId::new(DC_CHAT_ID_DEADDROP), 0, None).await {
+            if let ChatItem::Message { msg_id } = item {
+                let msg = match Message::load_from_db(context, msg_id).await {
+                    Err(e) => {
+                        warn!(context, "can't get msg: {:#}", e);
+                        return Status::RetryLater;
+                    }
+                    Ok(m) => m,
+                };
+                let chat = match Chat::load_from_db(context, msg.chat_id).await {
+                    Err(e) => {
+                        warn!(context, "can't get chat: {:#}", e);
+                        return Status::RetryLater;
+                    }
+                    Ok(c) => c,
+                };
+                match chat.typ {
+                    Chattype::Group | Chattype::Mailinglist => {
+                        if let Ok((_1to1_chat, Blocked::Not)) =
+                            chat::lookup_by_contact_id(context, msg.from_id).await
+                        {
+                            chat.id.unblock(context).await;
+                        }
+                    }
+                    Chattype::Single | Chattype::Undefined => {}
+                }
+            }
+        }
+
+        info!(context, "Done fetching existing messages.");
+        Status::Finished(Ok(()))
+    }
+
     /// Synchronizes UIDs for sentbox, inbox and mvbox, in this order.
     ///
     /// If a copy of the message is present in multiple folders, mvbox
@@ -675,6 +777,7 @@ impl Job {
             // retry. If the message was moved, we will create another
             // job to mark the message as seen later. If it was
             // deleted, there is nothing to do.
+            info!(context, "Can't mark message as seen: No UID");
             ImapActionResult::Failed
         } else {
             imap.set_seen(context, folder, msg.server_uid).await
@@ -759,6 +862,51 @@ async fn set_delivered(context: &Context, msg_id: MsgId) {
     context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
 }
 
+async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
+    let mailbox = if let Some(m) = context.get_config(folder).await {
+        m
+    } else {
+        return;
+    };
+    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
+        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+        warn!(context, "Could not select {}: {:#}", mailbox, e);
+        return;
+    }
+    match imap.get_all_recipients(context).await {
+        Ok(contacts) => {
+            let mut any_modified = false;
+            for contact in contacts {
+                let display_name_normalized = contact
+                    .display_name
+                    .as_ref()
+                    .map(normalize_name)
+                    .unwrap_or_default();
+
+                match Contact::add_or_lookup(
+                    context,
+                    display_name_normalized,
+                    contact.addr,
+                    Origin::OutgoingTo,
+                )
+                .await
+                {
+                    Ok((_, modified)) => {
+                        if modified != Modifier::None {
+                            any_modified = true;
+                        }
+                    }
+                    Err(e) => warn!(context, "Could not add recipient: {}", e),
+                }
+            }
+            if any_modified {
+                context.emit_event(EventType::ContactsChanged(None));
+            }
+        }
+        Err(e) => warn!(context, "Could not add recipients: {}", e),
+    };
+}
+
 /// Constructs a job for sending a message.
 ///
 /// Returns `None` if no messages need to be sent out.
@@ -810,7 +958,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
         return Ok(None);
     }
 
-    let rendered_msg = match mimefactory.render().await {
+    let rendered_msg = match mimefactory.render(context).await {
         Ok(res) => Ok(res),
         Err(err) => {
             message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
@@ -871,6 +1019,9 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     param.set(Param::File, blob.as_name());
     param.set(Param::Recipients, &recipients);
 
+    msg.subject = rendered_msg.subject.clone();
+    msg.update_subject(context).await;
+
     let job = create(Action::SendMsgToSmtp, msg_id.to_u32() as i32, param, 0)?;
 
     Ok(Some(job))
@@ -1007,8 +1158,9 @@ async fn perform_job_action(
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
+        Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::Housekeeping => {
-            sql::housekeeping(context).await;
+            sql::housekeeping(context).await.ok_or_log(context);
             Status::Finished(Ok(()))
         }
     };
@@ -1072,6 +1224,7 @@ pub async fn add(context: &Context, job: Job) {
             | Action::DeleteMsgOnImap
             | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
+            | Action::FetchExistingMsgs
             | Action::MoveMsg => {
                 info!(context, "interrupt: imap");
                 context
@@ -1091,6 +1244,18 @@ pub async fn add(context: &Context, job: Job) {
     }
 }
 
+async fn load_housekeeping_job(context: &Context) -> Option<Job> {
+    let last_time = context.get_config_i64(Config::LastHousekeeping).await;
+
+    let next_time = last_time + (60 * 60 * 24);
+    if next_time <= time() {
+        kill_action(context, Action::Housekeeping).await;
+        Some(Job::new(Action::Housekeeping, 0, Params::new(), 0))
+    } else {
+        None
+    }
+}
+
 /// Load jobs from the database.
 ///
 /// Load jobs for this "[Thread]", i.e. either load SMTP jobs or load
@@ -1104,6 +1269,17 @@ pub(crate) async fn load_next(
 ) -> Option<Job> {
     info!(context, "loading job for {}-thread", thread);
 
+    while !context.sql.is_open().await {
+        // The db is closed, which means that this thread should not be running.
+        // Wait until the db is re-opened (if we returned None, this thread might do further damage)
+        warn!(
+            context,
+            "{}: load_next() was called but the db was not opened, THIS SHOULD NOT HAPPEN. Waiting...",
+            thread
+        );
+        sleep(Duration::from_millis(500)).await;
+    }
+
     let query;
     let params;
     let t = time();
@@ -1207,8 +1383,10 @@ LIMIT 1;
                 } else {
                     Some(job)
                 }
+            } else if let Some(job) = load_imap_deletion_job(context).await.unwrap_or_default() {
+                Some(job)
             } else {
-                load_imap_deletion_job(context).await.unwrap_or_default()
+                load_housekeeping_job(context).await
             }
         }
         Thread::Smtp => job,
@@ -1219,7 +1397,7 @@ LIMIT 1;
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     async fn insert_job(context: &Context, foreign_id: i64) {
         let now = time();
@@ -1248,18 +1426,19 @@ mod tests {
         // fails to load from the database instead of failing to load
         // all jobs.
         let t = TestContext::new().await;
-        insert_job(&t.ctx, -1).await; // This can not be loaded into Job struct.
+        insert_job(&t, -1).await; // This can not be loaded into Job struct.
         let jobs = load_next(
-            &t.ctx,
+            &t,
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
         .await;
-        assert!(jobs.is_none());
+        // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
+        assert!(jobs.unwrap().action == Action::Housekeeping);
 
-        insert_job(&t.ctx, 1).await;
+        insert_job(&t, 1).await;
         let jobs = load_next(
-            &t.ctx,
+            &t,
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
@@ -1271,10 +1450,10 @@ mod tests {
     async fn test_load_next_job_one() {
         let t = TestContext::new().await;
 
-        insert_job(&t.ctx, 1).await;
+        insert_job(&t, 1).await;
 
         let jobs = load_next(
-            &t.ctx,
+            &t,
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )

src/key.rs

@@ -12,7 +12,7 @@ use pgp::types::{KeyTrait, SecretKeyTrait};
 use thiserror::Error;
 
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::KeyGenType;
 use crate::context::Context;
 use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
 use crate::sql;
@@ -222,7 +222,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     let addr = context
         .get_config(Config::ConfiguredAddr)
         .await
-        .ok_or_else(|| Error::NoConfiguredAddr)?;
+        .ok_or(Error::NoConfiguredAddr)?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -407,7 +407,7 @@ impl std::str::FromStr for Fingerprint {
         let hex_repr: String = input
             .to_uppercase()
             .chars()
-            .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
+            .filter(|&c| ('0'..='9').contains(&c) || ('A'..='F').contains(&c))
             .collect();
         let v: Vec<u8> = hex::decode(hex_repr)?;
         let fp = Fingerprint::new(v)?;
@@ -426,16 +426,14 @@ pub enum FingerprintError {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::{alice_keypair, TestContext};
 
     use std::error::Error;
 
     use async_std::sync::Arc;
-    use lazy_static::lazy_static;
+    use once_cell::sync::Lazy;
 
-    lazy_static! {
-        static ref KEYPAIR: KeyPair = alice_keypair();
-    }
+    static KEYPAIR: Lazy<KeyPair> = Lazy::new(alice_keypair);
 
     #[test]
     fn test_from_armored_string() {
@@ -560,31 +558,29 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         let alice = alice_keypair();
         let t = TestContext::new().await;
         t.configure_alice().await;
-        let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
+        let pubkey = SignedPublicKey::load_self(&t).await.unwrap();
         assert_eq!(alice.public, pubkey);
-        let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
+        let seckey = SignedSecretKey::load_self(&t).await.unwrap();
         assert_eq!(alice.secret, seckey);
     }
 
     #[async_std::test]
     async fn test_load_self_generate_public() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
-        let key = SignedPublicKey::load_self(&t.ctx).await;
+        let key = SignedPublicKey::load_self(&t).await;
         assert!(key.is_ok());
     }
 
     #[async_std::test]
     async fn test_load_self_generate_secret() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
-        let key = SignedSecretKey::load_self(&t.ctx).await;
+        let key = SignedSecretKey::load_self(&t).await;
         assert!(key.is_ok());
     }
 
@@ -593,17 +589,17 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         use std::thread;
 
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
-        let ctx = t.ctx.clone();
-        let ctx0 = ctx.clone();
-        let thr0 =
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
-        let ctx1 = ctx;
-        let thr1 =
-            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
+        let thr0 = {
+            let ctx = t.clone();
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+        };
+        let thr1 = {
+            let ctx = t.clone();
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx)))
+        };
         let res0 = thr0.join().unwrap();
         let res1 = thr1.join().unwrap();
         assert_eq!(res0.unwrap(), res1.unwrap());
@@ -620,12 +616,11 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         // Saving the same key twice should result in only one row in
         // the keypairs table.
         let t = TestContext::new().await;
-        let ctx = Arc::new(t.ctx);
+        let ctx = Arc::new(t);
 
-        let ctx1 = ctx.clone();
         let nrows = || async {
-            ctx1.sql
-                .query_get_value::<u32>(&ctx1, "SELECT COUNT(*) FROM keypairs;", paramsv![])
+            ctx.sql
+                .query_get_value::<u32>(&ctx, "SELECT COUNT(*) FROM keypairs;", paramsv![])
                 .await
                 .unwrap()
         };

src/keyring.rs

@@ -62,7 +62,7 @@ where
 mod tests {
     use super::*;
     use crate::key::{SignedPublicKey, SignedSecretKey};
-    use crate::test_utils::*;
+    use crate::test_utils::{alice_keypair, TestContext};
 
     #[test]
     fn test_keyring_add_keys() {
@@ -83,10 +83,10 @@ mod tests {
         t.configure_alice().await;
         let alice = alice_keypair();
 
-        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t).await.unwrap();
         assert_eq!(pub_ring.keys(), [alice.public]);
 
-        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t).await.unwrap();
         assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 }

src/lib.rs

@@ -3,7 +3,9 @@
     clippy::correctness,
     missing_debug_implementations,
     clippy::all,
-    clippy::indexing_slicing
+    clippy::indexing_slicing,
+    clippy::wildcard_imports,
+    clippy::needless_borrow
 )]
 #![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
@@ -72,10 +74,13 @@ pub mod qr;
 pub mod securejoin;
 mod simplify;
 mod smtp;
-pub mod stock;
+pub mod stock_str;
 mod token;
 #[macro_use]
 mod dehtml;
+mod color;
+pub mod html;
+pub mod plaintext;
 
 pub mod dc_receive_imf;
 pub mod dc_tools;

src/location.rs

@@ -1,20 +1,20 @@
 //! Location handling
 
+use anyhow::{ensure, Error};
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::chat::{self, ChatId};
 use crate::config::Config;
-use crate::constants::*;
-use crate::context::*;
-use crate::dc_tools::*;
-use crate::error::{ensure, Error};
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+use crate::context::Context;
+use crate::dc_tools::time;
 use crate::events::EventType;
 use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::param::*;
-use crate::stock::StockMessage;
+use crate::param::Params;
+use crate::stock_str;
 
 /// Location record
 #[derive(Debug, Clone, Default)]
@@ -193,7 +193,8 @@ impl Kml {
 pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
     let now = time();
     if !(seconds < 0 || chat_id.is_special()) {
-        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id).await;
+        let is_sending_locations_before =
+            is_sending_locations_to_chat(context, Some(chat_id)).await;
         if context
             .sql
             .execute(
@@ -212,19 +213,13 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
         {
             if 0 != seconds && !is_sending_locations_before {
                 let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some(
-                    context
-                        .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
-                        .await,
-                );
+                msg.text = Some(stock_str::msg_location_enabled(context).await);
                 msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
                 chat::send_msg(context, chat_id, &mut msg)
                     .await
                     .unwrap_or_default();
             } else if 0 == seconds && is_sending_locations_before {
-                let stock_str = context
-                    .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
-                    .await;
+                let stock_str = stock_str::msg_location_disabled(context).await;
                 chat::add_info_msg(context, chat_id, stock_str).await;
             }
             context.emit_event(EventType::ChatModified(chat_id));
@@ -255,15 +250,29 @@ async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool)
     };
 }
 
-pub async fn is_sending_locations_to_chat(context: &Context, chat_id: ChatId) -> bool {
-    context
-        .sql
-        .exists(
-            "SELECT id  FROM chats  WHERE (? OR id=?)   AND locations_send_until>?;",
-            paramsv![if chat_id.is_unset() { 1 } else { 0 }, chat_id, time()],
-        )
-        .await
-        .unwrap_or_default()
+/// Returns whether `chat_id` or any chat is sending locations.
+///
+/// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
+/// is sending locations.
+pub async fn is_sending_locations_to_chat(context: &Context, chat_id: Option<ChatId>) -> bool {
+    match chat_id {
+        Some(chat_id) => context
+            .sql
+            .exists(
+                "SELECT id  FROM chats  WHERE id=?  AND locations_send_until>?;",
+                paramsv![chat_id, time()],
+            )
+            .await
+            .unwrap_or_default(),
+        None => context
+            .sql
+            .exists(
+                "SELECT id  FROM chats  WHERE locations_send_until>?;",
+                paramsv![time()],
+            )
+            .await
+            .unwrap_or_default(),
+    }
 }
 
 pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
@@ -311,14 +320,22 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
 
 pub async fn get_range(
     context: &Context,
-    chat_id: ChatId,
-    contact_id: u32,
+    chat_id: Option<ChatId>,
+    contact_id: Option<u32>,
     timestamp_from: i64,
     mut timestamp_to: i64,
 ) -> Vec<Location> {
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
+    let (disable_chat_id, chat_id) = match chat_id {
+        Some(chat_id) => (0, chat_id),
+        None => (1, ChatId::new(0)), // this ChatId is unused
+    };
+    let (disable_contact_id, contact_id) = match contact_id {
+        Some(contact_id) => (0, contact_id),
+        None => (1, 0), // this contact_id is unused
+    };
     context
         .sql
         .query_map(
@@ -329,9 +346,9 @@ pub async fn get_range(
              AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
              ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
             paramsv![
-                if chat_id.is_unset() { 1 } else { 0 },
+                disable_chat_id,
                 chat_id,
-                if contact_id == 0 { 1 } else { 0 },
+                disable_contact_id,
                 contact_id as i32,
                 timestamp_from,
                 timestamp_to,
@@ -372,7 +389,12 @@ pub async fn get_range(
 }
 
 fn is_marker(txt: &str) -> bool {
-    txt.len() == 1 && !txt.starts_with(' ')
+    let mut chars = txt.chars();
+    if let Some(c) = chars.next() {
+        !c.is_whitespace() && chars.next().is_none()
+    } else {
+        false
+    }
 }
 
 /// Deletes all locations from the database.
@@ -711,9 +733,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
                 paramsv![chat_id],
             ).await);
 
-            let stock_str = context
-                .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
-                .await;
+            let stock_str = stock_str::msg_location_disabled(context).await;
             chat::add_info_msg(context, chat_id, stock_str).await;
             context.emit_event(EventType::ChatModified(chat_id));
         }
@@ -777,4 +797,13 @@ mod tests {
         assert!(locations_ref[0].accuracy.abs() < f64::EPSILON);
         assert_eq!(locations_ref[0].timestamp, timestamp);
     }
+
+    #[test]
+    fn test_is_marker() {
+        assert!(is_marker("f"));
+        assert!(!is_marker("foo"));
+        assert!(is_marker("🏠"));
+        assert!(!is_marker(" "));
+        assert!(!is_marker("\t"));
+    }
 }

src/log.rs

@@ -1,4 +1,5 @@
-//! # Logging macros
+//! # Logging
+use crate::context::Context;
 
 #[macro_export]
 macro_rules! info {
@@ -58,3 +59,95 @@ macro_rules! emit_event {
         $ctx.emit_event($event);
     };
 }
+
+pub trait LogExt<T, E>
+where
+    Self: std::marker::Sized,
+{
+    #[track_caller]
+    fn log_err_inner(self, context: &Context, msg: Option<&str>) -> Result<T, E>;
+
+    /// Emits a warning if the receiver contains an Err value.
+    ///
+    /// Thanks to the [track_caller](https://blog.rust-lang.org/2020/08/27/Rust-1.46.0.html#track_caller)
+    /// feature, the location of the caller is printed to the log, just like with the warn!() macro.
+    ///
+    /// Unfortunately, the track_caller feature does not work on async functions (as of Rust 1.50).
+    /// Once it is, you can add `#[track_caller]` to helper functions that use one of the log helpers here
+    /// so that the location of the caller can be seen in the log. (this won't work with the macros,
+    /// like warn!(), since the file!() and line!() macros don't work with track_caller)  
+    /// See https://github.com/rust-lang/rust/issues/78840 for progress on this.
+    #[track_caller]
+    fn log_err(self, context: &Context, msg: &str) -> Result<T, E> {
+        self.log_err_inner(context, Some(msg))
+    }
+
+    /// Emits a warning if the receiver contains an Err value and returns an [`Option<T>`].
+    ///
+    /// Example:
+    /// ```text
+    /// if let Err(e) = do_something() {
+    ///     warn!(context, "{:#}", e);
+    /// }
+    /// ```
+    /// is equivalent to:
+    /// ```text
+    /// do_something().ok_or_log(context);
+    /// ```
+    ///
+    /// For a note on the `track_caller` feature, see the doc comment on `log_err()`.
+    #[track_caller]
+    fn ok_or_log(self, context: &Context) -> Option<T> {
+        self.log_err_inner(context, None).ok()
+    }
+
+    /// Like `ok_or_log()`, but you can pass an extra message that is prepended in the log.
+    ///
+    /// Example:
+    /// ```text
+    /// if let Err(e) = do_something() {
+    ///     warn!(context, "Something went wrong: {:#}", e);
+    /// }
+    /// ```
+    /// is equivalent to:
+    /// ```text
+    /// do_something().ok_or_log_msg(context, "Something went wrong");
+    /// ```
+    /// and is also equivalent to:
+    /// ```text
+    /// use anyhow::Context as _;
+    /// do_something().context("Something went wrong").ok_or_log(context);
+    /// ```
+    ///
+    /// For a note on the `track_caller` feature, see the doc comment on `log_err()`.
+    #[track_caller]
+    fn ok_or_log_msg(self, context: &Context, msg: &'static str) -> Option<T> {
+        self.log_err_inner(context, Some(msg)).ok()
+    }
+}
+
+impl<T: Default, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
+    #[track_caller]
+    fn log_err_inner(self, context: &Context, msg: Option<&str>) -> Result<T, E> {
+        if let Err(e) = &self {
+            let location = std::panic::Location::caller();
+
+            let separator = if msg.is_none() { "" } else { ": " };
+            let msg = msg.unwrap_or_default();
+
+            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+            let full = format!(
+                "{file}:{line}: {msg}{separator}{e:#}",
+                file = location.file(),
+                line = location.line(),
+                msg = msg,
+                separator = separator,
+                e = e
+            );
+            // We can't use the warn!() macro here as the file!() and line!() macros
+            // don't work with #[track_caller]
+            emit_event!(context, crate::EventType::Warning(full));
+        };
+        self
+    }
+}

src/login_param.rs

@@ -3,6 +3,7 @@
 use std::borrow::Cow;
 use std::fmt;
 
+use crate::provider::{get_provider_by_id, Provider};
 use crate::{context::Context, provider::Socket};
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
@@ -48,6 +49,7 @@ pub struct LoginParam {
     pub imap: ServerLoginParam,
     pub smtp: ServerLoginParam,
     pub server_flags: i32,
+    pub provider: Option<&'static Provider>,
 }
 
 impl LoginParam {
@@ -130,6 +132,12 @@ impl LoginParam {
             .await
             .unwrap_or_default();
 
+        let key = format!("{}provider", prefix);
+        let provider = sql
+            .get_raw_config(context, key)
+            .await
+            .and_then(|provider_id| get_provider_by_id(&provider_id));
+
         LoginParam {
             addr,
             imap: ServerLoginParam {
@@ -148,6 +156,7 @@ impl LoginParam {
                 security: send_security,
                 certificate_checks: smtp_certificate_checks,
             },
+            provider,
             server_flags,
         }
     }
@@ -216,6 +225,11 @@ impl LoginParam {
         sql.set_raw_config_int(context, key, self.server_flags)
             .await?;
 
+        if let Some(provider) = self.provider {
+            let key = format!("{}provider", prefix);
+            sql.set_raw_config(context, key, Some(provider.id)).await?;
+        }
+
         Ok(())
     }
 }

src/lot.rs

@@ -22,7 +22,9 @@ pub struct Lot {
 }
 
 #[repr(u8)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 pub enum Meaning {
     None = 0,
     Text1Draft = 1,
@@ -67,7 +69,9 @@ impl Lot {
 }
 
 #[repr(i32)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 pub enum LotState {
     // Default
     Undefined = 0,

src/oauth2.rs

@@ -1,14 +1,12 @@
 //! OAuth 2 module
 
-use regex::Regex;
 use std::collections::HashMap;
 
-use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::time;
 use crate::provider;
 use crate::provider::Oauth2Authorizer;
 
@@ -19,7 +17,6 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
-    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -29,11 +26,8 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
-    mx_pattern: None,
 };
 
-const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
-
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -41,7 +35,6 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
-    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -170,16 +163,14 @@ pub async fn dc_get_oauth2_access_token(
         }
 
         // ... and POST
-        let response = surf::post(post_url).body_form(&post_param);
-        if response.is_err() {
-            warn!(
-                context,
-                "Error calling OAuth2 at {}: {:?}", token_url, response
-            );
+        let mut req = surf::post(post_url).build();
+        if let Err(err) = req.body_form(&post_param) {
+            warn!(context, "Error calling OAuth2 at {}: {:?}", token_url, err);
             return None;
         }
 
-        let parsed: Result<Response, _> = response.unwrap().recv_json().await;
+        let client = surf::Client::new();
+        let parsed: Result<Response, _> = client.recv_json(req).await;
         if parsed.is_err() {
             warn!(
                 context,
@@ -278,52 +269,21 @@ impl Oauth2 {
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            if let Some(provider) = provider::get_provider_info(&addr_normalized) {
-                match &provider.oauth2_authorizer {
-                    Some(Oauth2Authorizer::Gmail) => Some(OAUTH2_GMAIL),
-                    Some(Oauth2Authorizer::Yandex) => Some(OAUTH2_YANDEX),
-                    None => None, // provider known to not support oauth2, no mx-lookup required
-                }
-            } else {
-                Oauth2::lookup_mx(domain).await
-            }
-        } else {
-            None
-        }
-    }
-
-    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
-        if let Ok(resolver) = resolver(
-            config::ResolverConfig::default(),
-            config::ResolverOpts::default(),
-        )
-        .await
-        {
-            for provider in OAUTH2_PROVIDERS.iter() {
-                if let Some(pattern) = provider.mx_pattern {
-                    let re = Regex::new(pattern).unwrap();
-
-                    let mut fqdn: String = String::from(domain.as_ref());
-                    if !fqdn.ends_with('.') {
-                        fqdn.push_str(".");
-                    }
-
-                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
-                        for rr in res.iter() {
-                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
-                                return Some(provider.clone());
-                            }
-                        }
-                    }
-                }
+            if let Some(oauth2_authorizer) = provider::get_provider_info(domain)
+                .await
+                .and_then(|provider| provider.oauth2_authorizer.as_ref())
+            {
+                return Some(match oauth2_authorizer {
+                    Oauth2Authorizer::Gmail => OAUTH2_GMAIL,
+                    Oauth2Authorizer::Yandex => OAUTH2_YANDEX,
+                });
             }
         }
-
         None
     }
 
     async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
-        let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
+        let userinfo_url = self.get_userinfo.unwrap_or("");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
 
         // should returns sth. as
@@ -388,7 +348,7 @@ fn normalize_addr(addr: &str) -> &str {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_normalize_addr() {

src/param.rs

@@ -2,13 +2,14 @@ use std::collections::BTreeMap;
 use std::fmt;
 use std::str;
 
+use anyhow::{bail, Error};
 use async_std::path::PathBuf;
+use itertools::Itertools;
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
-use crate::error::{self, bail, ensure};
 use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 
@@ -21,6 +22,11 @@ pub enum Param {
     /// For messages and jobs
     File = b'f',
 
+    /// For messages: This name should be shown instead of contact.get_display_name()
+    /// (used if this is a mailinglist
+    /// or explictly set using set_override_sender_name(), eg. by bots)
+    OverrideSenderDisplayname = b'O',
+
     /// For Messages
     Width = b'w',
 
@@ -33,6 +39,11 @@ pub enum Param {
     /// For Messages
     MimeType = b'm',
 
+    /// For Messages: HTML to be written to the database and to be send.
+    /// `SendHtml` param is not used for received messages.
+    /// Use `MsgId::get_html()` to get HTML of received messages.
+    SendHtml = b'T',
+
     /// For Messages: message is encrypted, outgoing: guarantee E2EE or the message is not send
     GuaranteeE2ee = b'c',
 
@@ -49,9 +60,13 @@ pub enum Param {
     /// For Messages
     WantsMdn = b'r',
 
-    /// For Messages
+    /// For Messages: unset or 0=not forwarded,
+    /// 1=forwarded from unknown msg_id, >9 forwarded from msg_id
     Forwarded = b'a',
 
+    /// For Messages: quoted text.
+    Quote = b'q',
+
     /// For Messages
     Cmd = b'S',
 
@@ -111,21 +126,14 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
-    /// For Chats: So that on sending a new message we can sent the subject to "Re: <last subject>"
+    /// For Chats: On sending a new message we set the subject to "Re: <last subject>".
+    /// Usually we just use the subject of the parent message, but if the parent message
+    /// is deleted, we use the LastSubject of the chat.
     LastSubject = b't',
 
     /// For Chats
     Devicetalk = b'D',
 
-    /// For QR
-    Auth = b's',
-
-    /// For QR
-    GroupId = b'x',
-
-    /// For QR
-    GroupName = b'g',
-
     /// For MDN-sending job
     MsgId = b'I',
 }
@@ -146,38 +154,44 @@ impl fmt::Display for Params {
             if i > 0 {
                 writeln!(f)?;
             }
-            write!(f, "{}={}", *key as u8 as char, value)?;
+            write!(
+                f,
+                "{}={}",
+                *key as u8 as char,
+                value.split('\n').join("\n\n")
+            )?;
         }
         Ok(())
     }
 }
 
 impl str::FromStr for Params {
-    type Err = error::Error;
+    type Err = Error;
 
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let mut inner = BTreeMap::new();
-        for pair in s.trim().lines() {
-            let pair = pair.trim();
-            if pair.is_empty() {
-                continue;
-            }
-            // TODO: probably nicer using a regex
-            ensure!(pair.len() > 1, "Invalid key pair: '{}'", pair);
-            let mut split = pair.splitn(2, '=');
-            let key = split.next();
-            let value = split.next();
+        let mut lines = s.lines().peekable();
 
-            ensure!(key.is_some(), "Missing key");
-            ensure!(value.is_some(), "Missing value");
+        while let Some(line) = lines.next() {
+            if let [key, value] = line.splitn(2, '=').collect::<Vec<_>>()[..] {
+                let key = key.to_string();
+                let mut value = value.to_string();
+                while let Some(s) = lines.peek() {
+                    if !s.is_empty() {
+                        break;
+                    }
+                    lines.next();
+                    value.push('\n');
+                    value += lines.next().unwrap_or_default();
+                }
 
-            let key = key.unwrap_or_default().trim();
-            let value = value.unwrap_or_default().trim();
-
-            if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
-                inner.insert(key, value.to_string());
+                if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
+                    inner.insert(key, value);
+                } else {
+                    bail!("Unknown key: {}", key);
+                }
             } else {
-                bail!("Unknown key: {}", key);
+                bail!("Not a key-value pair: {:?}", line);
             }
         }
 
@@ -369,11 +383,11 @@ mod tests {
     use async_std::fs;
     use async_std::path::Path;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_dc_param() {
-        let mut p1: Params = "\r\n\r\na=1\nf=2\n\nc = 3 ".parse().unwrap();
+        let mut p1: Params = "a=1\nf=2\nc=3".parse().unwrap();
 
         assert_eq!(p1.get_int(Param::Forwarded), Some(1));
         assert_eq!(p1.get_int(Param::File), Some(2));
@@ -408,17 +422,17 @@ mod tests {
     }
 
     #[test]
-    fn test_regression() {
-        let p1: Params = "a=cli%40deltachat.de\nn=\ni=TbnwJ6lSvD5\ns=0ejvbdFSQxB"
-            .parse()
-            .unwrap();
-        assert_eq!(p1.get(Param::Forwarded).unwrap(), "cli%40deltachat.de");
+    fn test_roundtrip() {
+        let mut params = Params::new();
+        params.set(Param::Height, "foo\nbar=baz\nquux");
+        params.set(Param::Width, "\n\n\na=\n=");
+        assert_eq!(params.to_string().parse::<Params>().unwrap(), params);
     }
 
     #[async_std::test]
     async fn test_params_file_fs_path() {
         let t = TestContext::new().await;
-        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
+        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t, "/foo/bar/baz").unwrap() {
             assert_eq!(p, Path::new("/foo/bar/baz"));
         } else {
             panic!("Wrong enum variant");
@@ -428,7 +442,7 @@ mod tests {
     #[async_std::test]
     async fn test_params_file_blob() {
         let t = TestContext::new().await;
-        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
+        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t, "$BLOBDIR/foo").unwrap() {
             assert_eq!(b.as_name(), "$BLOBDIR/foo");
         } else {
             panic!("Wrong enum variant");
@@ -443,51 +457,33 @@ mod tests {
         let mut p = Params::new();
         p.set(Param::File, fname.to_str().unwrap());
 
-        let file = p.get_file(Param::File, &t.ctx).unwrap().unwrap();
+        let file = p.get_file(Param::File, &t).unwrap().unwrap();
         assert_eq!(file, ParamsFile::FsPath(fname.clone().into()));
 
-        let path: PathBuf = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        let path: PathBuf = p.get_path(Param::File, &t).unwrap().unwrap();
         let fname: PathBuf = fname.into();
         assert_eq!(path, fname);
 
         // Blob does not exist yet, expect BlobError.
-        let err = p.get_blob(Param::File, &t.ctx, false).await.unwrap_err();
+        let err = p.get_blob(Param::File, &t, false).await.unwrap_err();
         match err {
             BlobError::WrongBlobdir { .. } => (),
             _ => panic!("wrong error type/variant: {:?}", err),
         }
 
         fs::write(fname, b"boo").await.unwrap();
-        let blob = p
-            .get_blob(Param::File, &t.ctx, true)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(
-            blob,
-            BlobObject::from_name(&t.ctx, "foo".to_string()).unwrap()
-        );
+        let blob = p.get_blob(Param::File, &t, true).await.unwrap().unwrap();
+        assert_eq!(blob, BlobObject::from_name(&t, "foo".to_string()).unwrap());
 
         // Blob in blobdir, expect blob.
-        let bar_path = t.ctx.get_blobdir().join("bar");
+        let bar_path = t.get_blobdir().join("bar");
         p.set(Param::File, bar_path.to_str().unwrap());
-        let blob = p
-            .get_blob(Param::File, &t.ctx, false)
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(
-            blob,
-            BlobObject::from_name(&t.ctx, "bar".to_string()).unwrap()
-        );
+        let blob = p.get_blob(Param::File, &t, false).await.unwrap().unwrap();
+        assert_eq!(blob, BlobObject::from_name(&t, "bar".to_string()).unwrap());
 
         p.remove(Param::File);
-        assert!(p.get_file(Param::File, &t.ctx).unwrap().is_none());
-        assert!(p.get_path(Param::File, &t.ctx).unwrap().is_none());
-        assert!(p
-            .get_blob(Param::File, &t.ctx, false)
-            .await
-            .unwrap()
-            .is_none());
+        assert!(p.get_file(Param::File, &t).unwrap().is_none());
+        assert!(p.get_path(Param::File, &t).unwrap().is_none());
+        assert!(p.get_blob(Param::File, &t, false).await.unwrap().is_none());
     }
 }

src/peerstate.rs

@@ -1,18 +1,19 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
+
 use std::collections::HashSet;
 use std::fmt;
 
+use anyhow::{bail, Result};
 use num_traits::FromPrimitive;
 
-use crate::aheader::*;
+use crate::aheader::{Aheader, EncryptPreference};
 use crate::chat;
 use crate::constants::Blocked;
 use crate::context::Context;
-use crate::error::{bail, Result};
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 #[derive(Debug)]
 pub enum PeerstateKeyType {
@@ -29,8 +30,7 @@ pub enum PeerstateVerifiedStatus {
 }
 
 /// Peerstate represents the state of an Autocrypt peer.
-pub struct Peerstate<'a> {
-    pub context: &'a Context,
+pub struct Peerstate {
     pub addr: String,
     pub last_seen: i64,
     pub last_seen_autocrypt: i64,
@@ -46,7 +46,7 @@ pub struct Peerstate<'a> {
     pub fingerprint_changed: bool,
 }
 
-impl<'a> PartialEq for Peerstate<'a> {
+impl PartialEq for Peerstate {
     fn eq(&self, other: &Peerstate) -> bool {
         self.addr == other.addr
             && self.last_seen == other.last_seen
@@ -64,9 +64,9 @@ impl<'a> PartialEq for Peerstate<'a> {
     }
 }
 
-impl<'a> Eq for Peerstate<'a> {}
+impl Eq for Peerstate {}
 
-impl<'a> fmt::Debug for Peerstate<'a> {
+impl fmt::Debug for Peerstate {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         f.debug_struct("Peerstate")
             .field("addr", &self.addr)
@@ -93,10 +93,9 @@ pub enum ToSave {
     All = 0x02,
 }
 
-impl<'a> Peerstate<'a> {
-    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
+impl Peerstate {
+    pub fn from_header(header: &Aheader, message_time: i64) -> Self {
         Peerstate {
-            context,
             addr: header.addr.clone(),
             last_seen: message_time,
             last_seen_autocrypt: message_time,
@@ -113,13 +112,20 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub fn from_gossip(context: &'a Context, gossip_header: &Aheader, message_time: i64) -> Self {
+    pub fn from_gossip(gossip_header: &Aheader, message_time: i64) -> Self {
         Peerstate {
-            context,
             addr: gossip_header.addr.clone(),
             last_seen: 0,
             last_seen_autocrypt: 0,
-            prefer_encrypt: Default::default(),
+
+            // Non-standard extension. According to Autocrypt 1.1.0 gossip headers SHOULD NOT
+            // contain encryption preference.
+            //
+            // Delta Chat includes encryption preference to ensure new users introduced to a group
+            // learn encryption preferences of other members immediately and don't send unencrypted
+            // messages to a group where everyone prefers encryption.
+            prefer_encrypt: gossip_header.prefer_encrypt,
+
             public_key: None,
             public_key_fingerprint: None,
             gossip_key: Some(gossip_header.public_key.clone()),
@@ -132,7 +138,7 @@ impl<'a> Peerstate<'a> {
         }
     }
 
-    pub async fn from_addr(context: &'a Context, addr: &str) -> Result<Option<Peerstate<'a>>> {
+    pub async fn from_addr(context: &Context, addr: &str) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint \
@@ -142,10 +148,10 @@ impl<'a> Peerstate<'a> {
     }
 
     pub async fn from_fingerprint(
-        context: &'a Context,
+        context: &Context,
         _sql: &Sql,
         fingerprint: &Fingerprint,
-    ) -> Result<Option<Peerstate<'a>>> {
+    ) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint \
@@ -158,10 +164,10 @@ impl<'a> Peerstate<'a> {
     }
 
     async fn from_stmt(
-        context: &'a Context,
+        context: &Context,
         query: &str,
         params: Vec<&dyn crate::ToSql>,
-    ) -> Result<Option<Peerstate<'a>>> {
+    ) -> Result<Option<Peerstate>> {
         let peerstate = context
             .sql
             .query_row_optional(query, params, |row| {
@@ -172,7 +178,6 @@ impl<'a> Peerstate<'a> {
                 */
 
                 let res = Peerstate {
-                    context,
                     addr: row.get(0)?,
                     last_seen: row.get(1)?,
                     last_seen_autocrypt: row.get(2)?,
@@ -272,9 +277,7 @@ impl<'a> Peerstate<'a> {
                         .await
                         .unwrap_or_default();
 
-                let msg = context
-                    .stock_string_repl_str(StockMessage::ContactSetupChanged, self.addr.clone())
-                    .await;
+                let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
 
                 chat::add_info_msg(context, contact_chat_id, msg).await;
                 emit_event!(context, EventType::ChatModified(contact_chat_id));
@@ -427,31 +430,31 @@ impl<'a> Peerstate<'a> {
         if self.to_save == Some(ToSave::All) || create {
             sql.execute(
                 if create {
-                "INSERT INTO acpeerstates (last_seen, last_seen_autocrypt, prefer_encrypted, \
+                    "INSERT INTO acpeerstates (last_seen, last_seen_autocrypt, prefer_encrypted, \
                  public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                  verified_key, verified_key_fingerprint, addr \
                 ) VALUES(?,?,?,?,?,?,?,?,?,?,?)"
                 } else {
-                "UPDATE acpeerstates \
+                    "UPDATE acpeerstates \
                  SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
                  public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
                  verified_key=?, verified_key_fingerprint=? \
                  WHERE addr=?"
                 },
                 paramsv![
-                    self.last_seen,
-                    self.last_seen_autocrypt,
-                    self.prefer_encrypt as i64,
-                    self.public_key.as_ref().map(|k| k.to_bytes()),
-                    self.gossip_timestamp,
-                    self.gossip_key.as_ref().map(|k| k.to_bytes()),
-                    self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
-                    self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
-                    self.verified_key.as_ref().map(|k| k.to_bytes()),
-                    self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
-                    self.addr,
+                self.last_seen,
+                self.last_seen_autocrypt,
+                self.prefer_encrypt as i64,
+                self.public_key.as_ref().map(|k| k.to_bytes()),
+                self.gossip_timestamp,
+                self.gossip_key.as_ref().map(|k| k.to_bytes()),
+                self.public_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                self.verified_key.as_ref().map(|k| k.to_bytes()),
+                self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
+                self.addr,
                 ],
-            ).await?;
+                ).await?;
         } else if self.to_save == Some(ToSave::Timestamps) {
             sql.execute(
                 "UPDATE acpeerstates SET last_seen=?, last_seen_autocrypt=?, gossip_timestamp=? \
@@ -487,7 +490,7 @@ impl From<crate::key::FingerprintError> for rusqlite::Error {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::alice_keypair;
     use pretty_assertions::assert_eq;
 
     #[async_std::test]
@@ -498,7 +501,6 @@ mod tests {
         let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -542,7 +544,6 @@ mod tests {
         let pub_key = alice_keypair().public;
 
         let peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -576,7 +577,6 @@ mod tests {
         let pub_key = alice_keypair().public;
 
         let mut peerstate = Peerstate {
-            context: &ctx.ctx,
             addr: addr.into(),
             last_seen: 10,
             last_seen_autocrypt: 11,
@@ -637,13 +637,11 @@ mod tests {
 
     #[async_std::test]
     async fn test_peerstate_degrade_reordering() {
-        let context = crate::test_utils::TestContext::new().await.ctx;
         let addr = "example@example.org";
         let pub_key = alice_keypair().public;
         let header = Aheader::new(addr.to_string(), pub_key, EncryptPreference::Mutual);
 
         let mut peerstate = Peerstate {
-            context: &context,
             addr: addr.to_string(),
             last_seen: 0,
             last_seen_autocrypt: 0,

src/pgp.rs

@@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashSet};
 use std::io;
 use std::io::Cursor;
 
+use anyhow::{bail, ensure, format_err, Result};
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
@@ -17,7 +18,6 @@ use rand::{thread_rng, CryptoRng, Rng};
 
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
-use crate::error::{bail, ensure, format_err, Result};
 use crate::key::{DcKey, Fingerprint};
 use crate::keyring::Keyring;
 
@@ -380,8 +380,8 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::*;
-    use lazy_static::lazy_static;
+    use crate::test_utils::{alice_keypair, bob_keypair};
+    use once_cell::sync::Lazy;
 
     #[test]
     fn test_split_armored_data_1() {
@@ -449,26 +449,29 @@ mod tests {
     /// The original text of [CTEXT_SIGNED]
     static CLEARTEXT: &[u8] = b"This is a test";
 
-    lazy_static! {
-        /// Initialised [TestKeys] for tests.
-        static ref KEYS: TestKeys = TestKeys::new();
+    /// Initialised [TestKeys] for tests.
+    static KEYS: Lazy<TestKeys> = Lazy::new(TestKeys::new);
 
-        /// A cyphertext encrypted to Alice & Bob, signed by Alice.
-        static ref CTEXT_SIGNED: String = {
-            let mut keyring = Keyring::new();
-            keyring.add(KEYS.alice_public.clone());
-            keyring.add(KEYS.bob_public.clone());
-            smol::block_on(pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))).unwrap()
-        };
+    /// A cyphertext encrypted to Alice & Bob, signed by Alice.
+    static CTEXT_SIGNED: Lazy<String> = Lazy::new(|| {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_public.clone());
+        keyring.add(KEYS.bob_public.clone());
+        futures_lite::future::block_on(pk_encrypt(
+            CLEARTEXT,
+            keyring,
+            Some(KEYS.alice_secret.clone()),
+        ))
+        .unwrap()
+    });
 
-        /// A cyphertext encrypted to Alice & Bob, not signed.
-        static ref CTEXT_UNSIGNED: String = {
-            let mut keyring = Keyring::new();
-            keyring.add(KEYS.alice_public.clone());
-            keyring.add(KEYS.bob_public.clone());
-            smol::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
-        };
-    }
+    /// A cyphertext encrypted to Alice & Bob, not signed.
+    static CTEXT_UNSIGNED: Lazy<String> = Lazy::new(|| {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_public.clone());
+        keyring.add(KEYS.bob_public.clone());
+        futures_lite::future::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
+    });
 
     #[test]
     fn test_encrypt_signed() {

src/plaintext.rs

@@ -0,0 +1,252 @@
+///! Handle plain text together with some attributes.
+use crate::simplify::split_lines;
+use once_cell::sync::Lazy;
+
+#[derive(Debug)]
+pub struct PlainText {
+    pub text: String,
+
+    /// Text may "flowed" as defined in [RFC 2646](https://tools.ietf.org/html/rfc2646).
+    /// At a glance, that means, if a line ends with a space, it is merged with the next one
+    /// and the first leading spaces is ignored
+    /// (to allow lines starting with `>` that normally indicates a quote)
+    pub flowed: bool,
+
+    /// If set together with "flowed",
+    /// The space indicating merging two lines is removed.
+    pub delsp: bool,
+}
+
+impl PlainText {
+    /// Convert plain text to HTML.
+    /// The function handles quotes, links, fixed and floating text paragraphs.
+    pub async fn to_html(&self) -> String {
+        static LINKIFY_MAIL_RE: Lazy<regex::Regex> =
+            Lazy::new(|| regex::Regex::new(r#"\b([\w.\-+]+@[\w.\-]+)\b"#).unwrap());
+
+        static LINKIFY_URL_RE: Lazy<regex::Regex> = Lazy::new(|| {
+            regex::Regex::new(r#"\b((http|https|ftp|ftps):[\w.,:;$/@!?&%\-~=#+]+)"#).unwrap()
+        });
+
+        let lines = split_lines(&self.text);
+
+        let mut ret =
+            "<!DOCTYPE html>\n<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" /></head><body>\n".to_string();
+
+        for line in lines {
+            let is_quote = line.starts_with('>');
+
+            // we need to do html-entity-encoding after linkify, as otherwise encapsulated links
+            // as <http://example.org> cannot be handled correctly
+            // (they would become &lt;http://example.org&gt; where the trailing &gt; would become a valid url part).
+            // to avoid double encoding, we escape our html-entities by \r that must not be used in the string elsewhere.
+            let line = line.to_string().replace("\r", "");
+
+            let mut line = LINKIFY_MAIL_RE
+                .replace_all(&*line, "\rLTa href=\rQUOTmailto:$1\rQUOT\rGT$1\rLT/a\rGT")
+                .as_ref()
+                .to_string();
+
+            line = LINKIFY_URL_RE
+                .replace_all(&*line, "\rLTa href=\rQUOT$1\rQUOT\rGT$1\rLT/a\rGT")
+                .as_ref()
+                .to_string();
+
+            // encode html-entities after linkify the raw string
+            line = escaper::encode_minimal(&line);
+
+            // make our escaped html-entities real after encoding all others
+            line = line.replace("\rLT", "<");
+            line = line.replace("\rGT", ">");
+            line = line.replace("\rQUOT", "\"");
+
+            if self.flowed {
+                // flowed text as of RFC 3676 -
+                // a leading space shall be removed
+                // and is only there to allow > at the beginning of a line that is no quote.
+                line = line.strip_prefix(" ").unwrap_or(&line).to_string();
+                if is_quote {
+                    line = "<em>".to_owned() + &line + "</em>";
+                }
+
+                // a trailing space indicates that the line can be merged with the next one;
+                // for sake of simplicity, we skip merging for quotes (quotes may be combined with
+                // delsp, so `> >` is different from `>>` etc. see RFC 3676 for details)
+                if line.ends_with(' ') && !is_quote {
+                    if self.delsp {
+                        line.pop();
+                    }
+                } else {
+                    line += "<br/>\n";
+                }
+            } else {
+                // normal, fixed text
+                if is_quote {
+                    line = "<em>".to_owned() + &line + "</em>";
+                }
+                line += "<br/>\n";
+            }
+
+            ret += &*line;
+        }
+        ret += "</body></html>\n";
+        ret
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[async_std::test]
+    async fn test_plain_to_html() {
+        let html = PlainText {
+            text: r##"line 1
+line 2
+line with https://link-mid-of-line.org and http://link-end-of-line.com/file?foo=bar%20
+http://link-at-start-of-line.org
+"##
+            .to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line 1<br/>
+line 2<br/>
+line with <a href="https://link-mid-of-line.org">https://link-mid-of-line.org</a> and <a href="http://link-end-of-line.com/file?foo=bar%20">http://link-end-of-line.com/file?foo=bar%20</a><br/>
+<a href="http://link-at-start-of-line.org">http://link-at-start-of-line.org</a><br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_encapsulated() {
+        let html = PlainText {
+            text: r#"line with <http://encapsulated.link/?foo=_bar> here!"#.to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line with &lt;<a href="http://encapsulated.link/?foo=_bar">http://encapsulated.link/?foo=_bar</a>&gt; here!<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_nolink() {
+        let html = PlainText {
+            text: r#"line with nohttp://no.link here"#.to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line with nohttp://no.link here<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_mailto() {
+        let html = PlainText {
+            text: r#"just an address: foo@bar.org another@one.de"#.to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+just an address: <a href="mailto:foo@bar.org">foo@bar.org</a> <a href="mailto:another@one.de">another@one.de</a><br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_flowed() {
+        let html = PlainText {
+            text: "line \nstill line\n>quote \n>still quote\n >no quote".to_string(),
+            flowed: true,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line still line<br/>
+<em>&gt;quote </em><br/>
+<em>&gt;still quote</em><br/>
+&gt;no quote<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_flowed_delsp() {
+        let html = PlainText {
+            text: "line \nstill line\n>quote \n>still quote\n >no quote".to_string(),
+            flowed: true,
+            delsp: true,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+linestill line<br/>
+<em>&gt;quote </em><br/>
+<em>&gt;still quote</em><br/>
+&gt;no quote<br/>
+</body></html>
+"#
+        );
+    }
+
+    #[async_std::test]
+    async fn test_plain_to_html_fixed() {
+        let html = PlainText {
+            text: "line \nstill line\n>quote \n>still quote\n >no quote".to_string(),
+            flowed: false,
+            delsp: false,
+        }
+        .to_html()
+        .await;
+        assert_eq!(
+            html,
+            r#"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+line <br/>
+still line<br/>
+<em>&gt;quote </em><br/>
+<em>&gt;still quote</em><br/>
+ &gt;no quote<br/>
+</body></html>
+"#
+        );
+    }
+}

src/provider/mod.rs

@@ -3,8 +3,9 @@
 mod data;
 
 use crate::config::Config;
-use crate::dc_tools::EmailAddress;
-use crate::provider::data::PROVIDER_DATA;
+use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED};
+use async_std_resolver::{config, resolver};
+use chrono::{NaiveDateTime, NaiveTime};
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
@@ -67,6 +68,8 @@ pub struct ConfigDefault {
 
 #[derive(Debug)]
 pub struct Provider {
+    /// Unique ID, corresponding to provider database filename.
+    pub id: &'static str,
     pub status: Status,
     pub before_login_hint: &'static str,
     pub after_login_hint: &'static str,
@@ -74,48 +77,117 @@ pub struct Provider {
     pub server: Vec<Server>,
     pub config_defaults: Option<Vec<ConfigDefault>>,
     pub strict_tls: bool,
+    pub max_smtp_rcpt_to: Option<u16>,
     pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
-pub fn get_provider_info(addr: &str) -> Option<&Provider> {
-    let domain = match addr.parse::<EmailAddress>() {
-        Ok(addr) => addr.domain,
-        Err(_err) => return None,
-    }
-    .to_lowercase();
+/// Returns provider for the given domain.
+///
+/// This function looks up domain in offline database first. If not
+/// found, it queries MX record for the domain and looks up offline
+/// database for MX domains.
+///
+/// For compatibility, email address can be passed to this function
+/// instead of the domain.
+pub async fn get_provider_info(domain: &str) -> Option<&'static Provider> {
+    let domain = domain.rsplitn(2, '@').next()?;
 
-    if let Some(provider) = PROVIDER_DATA.get(domain.as_str()) {
+    if let Some(provider) = get_provider_by_domain(domain) {
+        return Some(provider);
+    }
+
+    if let Some(provider) = get_provider_by_mx(domain).await {
+        return Some(provider);
+    }
+
+    None
+}
+
+/// Finds a provider in offline database based on domain.
+pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
+    if let Some(provider) = PROVIDER_DATA.get(domain.to_lowercase().as_str()) {
         return Some(*provider);
     }
 
     None
 }
 
+/// Finds a provider based on MX record for the given domain.
+///
+/// For security reasons, only Gmail can be configured this way.
+pub async fn get_provider_by_mx(domain: impl AsRef<str>) -> Option<&'static Provider> {
+    if let Ok(resolver) = resolver(
+        config::ResolverConfig::default(),
+        config::ResolverOpts::default(),
+    )
+    .await
+    {
+        let mut fqdn: String = String::from(domain.as_ref());
+        if !fqdn.ends_with('.') {
+            fqdn.push('.');
+        }
+
+        if let Ok(mx_domains) = resolver.mx_lookup(fqdn).await {
+            for (provider_domain, provider) in PROVIDER_DATA.iter() {
+                if provider.id != "gmail" {
+                    // MX lookup is limited to Gmail for security reasons
+                    continue;
+                }
+
+                let provider_fqdn = provider_domain.to_string() + ".";
+                let provider_fqdn_dot = ".".to_string() + &provider_fqdn;
+
+                for mx_domain in mx_domains.iter() {
+                    let mx_domain = mx_domain.exchange().to_lowercase().to_utf8();
+
+                    if mx_domain == provider_fqdn || mx_domain.ends_with(&provider_fqdn_dot) {
+                        return Some(provider);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+pub fn get_provider_by_id(id: &str) -> Option<&'static Provider> {
+    if let Some(provider) = PROVIDER_IDS.get(id) {
+        Some(provider)
+    } else {
+        None
+    }
+}
+
+// returns update timestamp in seconds, UTC, compatible for comparison with time() and database times
+pub fn get_provider_update_timestamp() -> i64 {
+    NaiveDateTime::new(*PROVIDER_UPDATED, NaiveTime::from_hms(0, 0, 0)).timestamp_millis() / 1_000
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
+    use crate::dc_tools::time;
+    use chrono::NaiveDate;
 
     #[test]
-    fn test_get_provider_info_unexistant() {
-        let provider = get_provider_info("user@unexistant.org");
+    fn test_get_provider_by_domain_unexistant() {
+        let provider = get_provider_by_domain("unexistant.org");
         assert!(provider.is_none());
     }
 
     #[test]
-    fn test_get_provider_info_mixed_case() {
-        let provider = get_provider_info("uSer@nAUta.Cu").unwrap();
+    fn test_get_provider_by_domain_mixed_case() {
+        let provider = get_provider_by_domain("nAUta.Cu").unwrap();
         assert!(provider.status == Status::OK);
     }
 
     #[test]
-    fn test_get_provider_info() {
-        let provider = get_provider_info("nauta.cu"); // this is no email address
-        assert!(provider.is_none());
-
-        let addr = "user@nauta.cu";
-        let provider = get_provider_info(addr).unwrap();
+    fn test_get_provider_by_domain() {
+        let addr = "nauta.cu";
+        let provider = get_provider_by_domain(addr).unwrap();
         assert!(provider.status == Status::OK);
         let server = &provider.server[0];
         assert_eq!(server.protocol, Protocol::IMAP);
@@ -130,12 +202,39 @@ mod tests {
         assert_eq!(server.port, 25);
         assert_eq!(server.username_pattern, UsernamePattern::EMAIL);
 
-        let provider = get_provider_info("user@gmail.com").unwrap();
+        let provider = get_provider_by_domain("gmail.com").unwrap();
         assert!(provider.status == Status::PREPARATION);
         assert!(!provider.before_login_hint.is_empty());
         assert!(!provider.overview_page.is_empty());
 
-        let provider = get_provider_info("user@googlemail.com").unwrap();
+        let provider = get_provider_by_domain("googlemail.com").unwrap();
         assert!(provider.status == Status::PREPARATION);
     }
+
+    #[test]
+    fn test_get_provider_by_id() {
+        let provider = get_provider_by_id("gmail").unwrap();
+        assert!(provider.id == "gmail");
+    }
+
+    #[async_std::test]
+    async fn test_get_provider_info() {
+        assert!(get_provider_info("").await.is_none());
+        assert!(get_provider_info("google.com").await.unwrap().id == "gmail");
+
+        // get_provider_info() accepts email addresses for backwards compatibility
+        assert!(get_provider_info("example@google.com").await.unwrap().id == "gmail");
+    }
+
+    #[test]
+    fn test_get_provider_update_timestamp() {
+        let timestamp_past = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 9, 9),
+            NaiveTime::from_hms(0, 0, 0),
+        )
+        .timestamp_millis()
+            / 1_000;
+        assert!(get_provider_update_timestamp() <= time());
+        assert!(get_provider_update_timestamp() > timestamp_past);
+    }
 }

src/provider/update.py

@@ -4,10 +4,12 @@
 import sys
 import os
 import yaml
+import datetime
 
 out_all = ""
 out_domains = ""
-domains_dict = {}
+out_ids = ""
+domains_set = set()
 
 def camel(name):
     words = name.split("_")
@@ -21,15 +23,19 @@ def cleanstr(s):
     return s
 
 
+def file2id(f):
+    return os.path.basename(f).replace(".md", "")
+
+
 def file2varname(f):
-    f = f[f.rindex("/")+1:].replace(".md", "")
+    f = file2id(f)
     f = f.replace(".", "_")
     f = f.replace("-", "_")
     return "P_" + f.upper()
 
 
 def file2url(f):
-    f = f[f.rindex("/")+1:].replace(".md", "")
+    f = file2id(f)
     f = f.replace(".", "-")
     return "https://providers.delta.chat/" + f
 
@@ -41,8 +47,8 @@ def process_config_defaults(data):
     config_defaults = data.get("config_defaults", "")
     for key in config_defaults:
         value = str(config_defaults[key])
-        defaults += "            ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
-    defaults += "        ])"
+        defaults += "        ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
+    defaults += "    ])"
     return defaults
 
 
@@ -60,14 +66,16 @@ def process_data(data, file):
         if domain == "" or domain.count(".") < 1 or domain.lower() != domain:
             raise TypeError("bad domain: " + domain)
 
-        global domains_dict
-        if domains_dict.get(domain, False):
+        global domains_set
+        if domain in domains_set:
             raise TypeError("domain used twice: " + domain)
-        domains_dict[domain] = True
+        domains_set.add(domain)
 
-        domains += "        (\"" + domain + "\", &*" + file2varname(file) + "),\n"
+        domains += "    (\"" + domain + "\", &*" + file2varname(file) + "),\n"
         comment += domain + ", "
 
+    ids = ""
+    ids += "    (\"" + file2id(file) + "\", &*" + file2varname(file) + "),\n"
 
     server = ""
     has_imap = False
@@ -95,14 +103,17 @@ def process_data(data, file):
             if username_pattern != "EMAIL" and username_pattern != "EMAILLOCALPART":
                 raise TypeError("bad username pattern")
 
-            server += ("            Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
+            server += ("        Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
             + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern + " },\n")
 
     config_defaults = process_config_defaults(data)
 
-    strict_tls = data.get("strict_tls", False)
+    strict_tls = data.get("strict_tls", True)
     strict_tls = "true" if strict_tls else "false"
 
+    max_smtp_rcpt_to = data.get("max_smtp_rcpt_to", 0)
+    max_smtp_rcpt_to = "Some(" + str(max_smtp_rcpt_to) + ")" if max_smtp_rcpt_to != 0 else "None"
+
     oauth2 = data.get("oauth2", "")
     oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
 
@@ -110,16 +121,18 @@ def process_data(data, file):
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
     if (not has_imap and not has_smtp) or (has_imap and has_smtp):
-        provider += "    static ref " + file2varname(file) + ": Provider = Provider {\n"
-        provider += "        status: Status::" + status + ",\n"
-        provider += "        before_login_hint: \"" + before_login_hint + "\",\n"
-        provider += "        after_login_hint: \"" + after_login_hint + "\",\n"
-        provider += "        overview_page: \"" + file2url(file) + "\",\n"
-        provider += "        server: vec![\n" + server + "        ],\n"
-        provider += "        config_defaults: " + config_defaults + ",\n"
-        provider += "        strict_tls: " + strict_tls + ",\n"
-        provider += "        oauth2_authorizer: " + oauth2 + ",\n"
-        provider += "    };\n\n"
+        provider += "static " + file2varname(file) + ": Lazy<Provider> = Lazy::new(|| Provider {\n"
+        provider += "    id: \"" + file2id(file) + "\",\n"
+        provider += "    status: Status::" + status + ",\n"
+        provider += "    before_login_hint: \"" + before_login_hint + "\",\n"
+        provider += "    after_login_hint: \"" + after_login_hint + "\",\n"
+        provider += "    overview_page: \"" + file2url(file) + "\",\n"
+        provider += "    server: vec![\n" + server + "    ],\n"
+        provider += "    config_defaults: " + config_defaults + ",\n"
+        provider += "    strict_tls: " + strict_tls + ",\n"
+        provider += "    max_smtp_rcpt_to: " + max_smtp_rcpt_to + ",\n"
+        provider += "    oauth2_authorizer: " + oauth2 + ",\n"
+        provider += "});\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
 
@@ -127,13 +140,14 @@ def process_data(data, file):
         raise TypeError("status PREPARATION or BROKEN requires before_login_hint: " + file)
 
     # finally, add the provider
-    global out_all, out_domains
-    out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
+    global out_all, out_domains, out_ids
+    out_all += "// " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
 
     # also add provider with no special things to do -
     # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
     out_all += provider
     out_domains += domains
+    out_ids += ids
 
 
 def process_file(file):
@@ -163,12 +177,20 @@ if __name__ == "__main__":
     "use crate::provider::UsernamePattern::*;\n"
     "use crate::provider::*;\n"
     "use std::collections::HashMap;\n\n"
-    "lazy_static::lazy_static! {\n\n")
+    "use once_cell::sync::Lazy;\n\n")
 
     process_dir(sys.argv[1])
 
-    out_all += "    pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [\n"
+    out_all += "pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
     out_all += out_domains;
-    out_all += "    ].iter().copied().collect();\n}"
+    out_all += "].iter().copied().collect());\n\n"
+
+    out_all += "pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
+    out_all += out_ids;
+    out_all += "].iter().copied().collect());\n\n"
+
+    now = datetime.datetime.utcnow()
+    out_all += "pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> = "\
+               "Lazy::new(|| chrono::NaiveDate::from_ymd("+str(now.year)+", "+str(now.month)+", "+str(now.day)+"));\n"
 
     print(out_all)

src/qr.rs

@@ -1,20 +1,20 @@
 //! # QR code module
 
-use lazy_static::lazy_static;
+use anyhow::{bail, ensure, format_err, Error};
+use once_cell::sync::Lazy;
 use percent_encoding::percent_decode_str;
 use serde::Deserialize;
+use std::collections::BTreeMap;
 
 use crate::chat;
-use crate::config::*;
+use crate::config::Config;
 use crate::constants::Blocked;
-use crate::contact::*;
+use crate::contact::{addr_normalize, may_be_valid_addr, Contact, Origin};
 use crate::context::Context;
-use crate::error::{bail, ensure, format_err, Error};
 use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::message::Message;
-use crate::param::*;
-use crate::peerstate::*;
+use crate::peerstate::Peerstate;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
@@ -93,16 +93,18 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         }
     };
 
-    // replace & with \n to match expected param format
-    let fragment = fragment.replace('&', "\n");
+    let param: BTreeMap<&str, &str> = fragment
+        .split('&')
+        .filter_map(|s| {
+            if let [key, value] = s.splitn(2, '=').collect::<Vec<_>>()[..] {
+                Some((key, value))
+            } else {
+                None
+            }
+        })
+        .collect();
 
-    // Then parse the parameters
-    let param: Params = match fragment.parse() {
-        Ok(params) => params,
-        Err(err) => return err.into(),
-    };
-
-    let addr = if let Some(addr) = param.get(Param::Forwarded) {
+    let addr = if let Some(addr) = param.get("a") {
         match normalize_address(addr) {
             Ok(addr) => Some(addr),
             Err(err) => return err.into(),
@@ -112,7 +114,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     };
 
     // what is up with that param name?
-    let name = if let Some(encoded_name) = param.get(Param::SetLongitude) {
+    let name = if let Some(encoded_name) = param.get("n") {
         let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
         match percent_decode_str(&encoded_name).decode_utf8() {
             Ok(name) => name.to_string(),
@@ -122,12 +124,12 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         "".to_string()
     };
 
-    let invitenumber = param.get(Param::ProfileImage).map(|s| s.to_string());
-    let auth = param.get(Param::Auth).map(|s| s.to_string());
-    let grpid = param.get(Param::GroupId).map(|s| s.to_string());
+    let invitenumber = param.get("i").map(|s| s.to_string());
+    let auth = param.get("s").map(|s| s.to_string());
+    let grpid = param.get("x").map(|s| s.to_string());
 
     let grpname = if grpid.is_some() {
-        if let Some(encoded_name) = param.get(Param::GroupName) {
+        if let Some(encoded_name) = param.get("g") {
             let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
             match percent_decode_str(&encoded_name).decode_utf8() {
                 Ok(name) => Some(name.to_string()),
@@ -358,16 +360,14 @@ async fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     Lot::from_address(context, name, addr).await
 }
 
-lazy_static! {
-    static ref VCARD_NAME_RE: regex::Regex =
-        regex::Regex::new(r"(?m)^N:([^;]*);([^;\n]*)").unwrap();
-    static ref VCARD_EMAIL_RE: regex::Regex =
-        regex::Regex::new(r"(?m)^EMAIL([^:\n]*):([^;\n]*)").unwrap();
-}
+static VCARD_NAME_RE: Lazy<regex::Regex> =
+    Lazy::new(|| regex::Regex::new(r"(?m)^N:([^;]*);([^;\n]*)").unwrap());
+static VCARD_EMAIL_RE: Lazy<regex::Regex> =
+    Lazy::new(|| regex::Regex::new(r"(?m)^EMAIL([^:\n]*):([^;\n]*)").unwrap());
 
-/// Extract address for the matmsg scheme.
+/// Extract address for the vcard scheme.
 ///
-/// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;
+/// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;`
 #[allow(clippy::indexing_slicing)]
 async fn decode_vcard(context: &Context, qr: &str) -> Lot {
     let name = VCARD_NAME_RE
@@ -427,7 +427,7 @@ fn normalize_address(addr: &str) -> Result<String, Error> {
     let new_addr = percent_decode_str(addr).decode_utf8()?;
     let new_addr = addr_normalize(&new_addr);
 
-    ensure!(may_be_valid_addr(&new_addr), "Bad e-mail address");
+    ensure!(may_be_valid_addr(new_addr), "Bad e-mail address");
 
     Ok(new_addr.to_string())
 }
@@ -490,6 +490,8 @@ mod tests {
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
         assert_eq!(contact.get_name(), "First Last");
+        assert_eq!(contact.get_authname(), "");
+        assert_eq!(contact.get_display_name(), "First Last");
     }
 
     #[async_std::test]
@@ -565,6 +567,7 @@ mod tests {
         assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
         // Test it again with lowercased "openpgp4fpr:" uri scheme
+        let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
             "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
@@ -605,6 +608,21 @@ mod tests {
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
         assert_eq!(contact.get_name(), "Jörn P. P.");
+
+        // Regression test
+        let ctx = TestContext::new().await;
+        let res = check_qr(
+            &ctx.ctx,
+            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
+        ).await;
+
+        println!("{:?}", res);
+        assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
+        assert_ne!(res.get_id(), 0);
+
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
+        assert_eq!(contact.get_addr(), "cli@deltachat.de");
+        assert_eq!(contact.get_name(), "");
     }
 
     #[async_std::test]

src/scheduler.rs

@@ -1,11 +1,16 @@
 use async_std::prelude::*;
-use async_std::sync::{channel, Receiver, Sender};
-use async_std::task;
+use async_std::{
+    channel::{self, Receiver, Sender},
+    task,
+};
 
+use crate::config::Config;
 use crate::context::Context;
+use crate::dc_tools::maybe_add_time_based_warnings;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
-use crate::{config::Config, message::MsgId, smtp::Smtp};
+use crate::message::MsgId;
+use crate::smtp::Smtp;
 
 pub(crate) struct StopToken;
 
@@ -53,7 +58,10 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started.send(()).await;
+        started
+            .send(())
+            .await
+            .expect("inbox loop, missing started receiver");
         let ctx = ctx1;
 
         // track number of continously executed jobs
@@ -68,9 +76,11 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                 }
                 Some(job) => {
                     // Let the fetch run, but return back to the job afterwards.
-                    info!(ctx, "postponing imap-job {} to run fetch...", job);
                     jobs_loaded = 0;
-                    fetch(&ctx, &mut connection).await;
+                    if ctx.get_config_bool(Config::InboxWatch).await {
+                        info!(ctx, "postponing imap-job {} to run fetch...", job);
+                        fetch(&ctx, &mut connection).await;
+                    }
                 }
                 None => {
                     jobs_loaded = 0;
@@ -81,9 +91,14 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                         warn!(ctx, "failed to close folder: {:?}", err);
                     }
 
+                    maybe_add_time_based_warnings(&ctx).await;
+
                     info = if ctx.get_config_bool(Config::InboxWatch).await {
                         fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
                     } else {
+                        if let Err(err) = connection.scan_folders(&ctx).await {
+                            warn!(ctx, "{}", err);
+                        }
                         connection.fake_idle(&ctx, None).await
                     };
                 }
@@ -98,26 +113,29 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         })
         .race(fut)
         .await;
-    shutdown_sender.send(()).await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("inbox loop, missing shutdown receiver");
 }
 
 async fn fetch(ctx: &Context, connection: &mut Imap) {
     match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
-            if let Err(err) = connection.connect_configured(&ctx).await {
+            if let Err(err) = connection.connect_configured(ctx).await {
                 error_network!(ctx, "{}", err);
                 return;
             }
 
             // fetch
-            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+            if let Err(err) = connection.fetch(ctx, &watch_folder).await {
                 connection.trigger_reconnect();
-                warn!(ctx, "{}", err);
+                warn!(ctx, "{:#}", err);
             }
         }
         None => {
             warn!(ctx, "Can not fetch inbox folder, not set");
-            connection.fake_idle(&ctx, None).await;
+            connection.fake_idle(ctx, None).await;
         }
     }
 }
@@ -126,21 +144,30 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
     match ctx.get_config(folder).await {
         Some(watch_folder) => {
             // connect and fake idle if unable to connect
-            if let Err(err) = connection.connect_configured(&ctx).await {
+            if let Err(err) = connection.connect_configured(ctx).await {
                 warn!(ctx, "imap connection failed: {}", err);
-                return connection.fake_idle(&ctx, None).await;
+                return connection.fake_idle(ctx, Some(watch_folder)).await;
             }
 
             // fetch
-            if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
+            if let Err(err) = connection.fetch(ctx, &watch_folder).await {
                 connection.trigger_reconnect();
-                warn!(ctx, "{}", err);
+                warn!(ctx, "{:#}", err);
+            }
+
+            if folder == Config::ConfiguredInboxFolder {
+                // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
+                if let Err(err) = connection.scan_folders(ctx).await {
+                    // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
+                    // but maybe just one folder can't be selected or something
+                    warn!(ctx, "{}", err);
+                }
             }
 
             // idle
             if connection.can_idle() {
                 connection
-                    .idle(&ctx, Some(watch_folder))
+                    .idle(ctx, Some(watch_folder))
                     .await
                     .unwrap_or_else(|err| {
                         connection.trigger_reconnect();
@@ -148,12 +175,12 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                         InterruptInfo::new(false, None)
                     })
             } else {
-                connection.fake_idle(&ctx, Some(watch_folder)).await
+                connection.fake_idle(ctx, Some(watch_folder)).await
             }
         }
         None => {
             warn!(ctx, "Can not watch {} folder, not set", folder);
-            connection.fake_idle(&ctx, None).await
+            connection.fake_idle(ctx, None).await
         }
     }
 }
@@ -176,7 +203,10 @@ async fn simple_imap_loop(
     let ctx1 = ctx.clone();
 
     let fut = async move {
-        started.send(()).await;
+        started
+            .send(())
+            .await
+            .expect("simple imap loop, missing started receive");
         let ctx = ctx1;
 
         loop {
@@ -191,7 +221,10 @@ async fn simple_imap_loop(
         })
         .race(fut)
         .await;
-    shutdown_sender.send(()).await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("simple imap loop, missing shutdown receiver");
 }
 
 async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnectionHandlers) {
@@ -207,7 +240,10 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started.send(()).await;
+        started
+            .send(())
+            .await
+            .expect("smtp loop, missing started receiver");
         let ctx = ctx1;
 
         let mut interrupt_info = Default::default();
@@ -235,7 +271,10 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
         })
         .race(fut)
         .await;
-    shutdown_sender.send(()).await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("smtp loop, missing shutdown receiver");
 }
 
 impl Scheduler {
@@ -246,23 +285,25 @@ impl Scheduler {
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
         let (inbox, inbox_handlers) = ImapConnectionState::new();
 
-        let (inbox_start_send, inbox_start_recv) = channel(1);
-        let (mvbox_start_send, mvbox_start_recv) = channel(1);
+        let (inbox_start_send, inbox_start_recv) = channel::bounded(1);
+        let (mvbox_start_send, mvbox_start_recv) = channel::bounded(1);
         let mut mvbox_handle = None;
-        let (sentbox_start_send, sentbox_start_recv) = channel(1);
+        let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1);
         let mut sentbox_handle = None;
-        let (smtp_start_send, smtp_start_recv) = channel(1);
+        let (smtp_start_send, smtp_start_recv) = channel::bounded(1);
 
-        let ctx1 = ctx.clone();
-        let inbox_handle = Some(task::spawn(async move {
-            inbox_loop(ctx1, inbox_start_send, inbox_handlers).await
-        }));
+        let inbox_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                inbox_loop(ctx, inbox_start_send, inbox_handlers).await
+            }))
+        };
 
         if ctx.get_config_bool(Config::MvboxWatch).await {
-            let ctx1 = ctx.clone();
+            let ctx = ctx.clone();
             mvbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
-                    ctx1,
+                    ctx,
                     mvbox_start_send,
                     mvbox_handlers,
                     Config::ConfiguredMvboxFolder,
@@ -270,14 +311,17 @@ impl Scheduler {
                 .await
             }));
         } else {
-            mvbox_start_send.send(()).await;
+            mvbox_start_send
+                .send(())
+                .await
+                .expect("mvbox start send, missing receiver");
         }
 
         if ctx.get_config_bool(Config::SentboxWatch).await {
-            let ctx1 = ctx.clone();
+            let ctx = ctx.clone();
             sentbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
-                    ctx1,
+                    ctx,
                     sentbox_start_send,
                     sentbox_handlers,
                     Config::ConfiguredSentboxFolder,
@@ -285,13 +329,18 @@ impl Scheduler {
                 .await
             }));
         } else {
-            sentbox_start_send.send(()).await;
+            sentbox_start_send
+                .send(())
+                .await
+                .expect("sentbox start send, missing receiver");
         }
 
-        let ctx1 = ctx.clone();
-        let smtp_handle = Some(task::spawn(async move {
-            smtp_loop(ctx1, smtp_start_send, smtp_handlers).await
-        }));
+        let smtp_handle = {
+            let ctx = ctx.clone();
+            Some(task::spawn(async move {
+                smtp_loop(ctx, smtp_start_send, smtp_handlers).await
+            }))
+        };
 
         *self = Scheduler::Running {
             inbox,
@@ -362,17 +411,27 @@ impl Scheduler {
             }
             Scheduler::Running {
                 inbox,
+                inbox_handle,
                 mvbox,
+                mvbox_handle,
                 sentbox,
+                sentbox_handle,
                 smtp,
+                smtp_handle,
                 ..
             } => {
-                inbox
-                    .stop()
-                    .join(mvbox.stop())
-                    .join(sentbox.stop())
-                    .join(smtp.stop())
-                    .await;
+                if inbox_handle.is_some() {
+                    inbox.stop().await;
+                }
+                if mvbox_handle.is_some() {
+                    mvbox.stop().await;
+                }
+                if sentbox_handle.is_some() {
+                    sentbox.stop().await;
+                }
+                if smtp_handle.is_some() {
+                    smtp.stop().await;
+                }
 
                 StopToken
             }
@@ -412,10 +471,7 @@ impl Scheduler {
 
     /// Check if the scheduler is running.
     pub fn is_running(&self) -> bool {
-        match self {
-            Scheduler::Running { .. } => true,
-            _ => false,
-        }
+        matches!(self, Scheduler::Running { .. })
     }
 }
 
@@ -434,7 +490,10 @@ impl ConnectionState {
     /// Shutdown this connection completely.
     async fn stop(&self) {
         // Trigger shutdown of the run loop.
-        self.stop_sender.send(()).await;
+        self.stop_sender
+            .send(())
+            .await
+            .expect("stop, missing receiver");
         // Wait for a notification that the run loop has been shutdown.
         self.shutdown_receiver.recv().await.ok();
     }
@@ -452,9 +511,9 @@ pub(crate) struct SmtpConnectionState {
 
 impl SmtpConnectionState {
     fn new() -> (Self, SmtpConnectionHandlers) {
-        let (stop_sender, stop_receiver) = channel(1);
-        let (shutdown_sender, shutdown_receiver) = channel(1);
-        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+        let (stop_sender, stop_receiver) = channel::bounded(1);
+        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = SmtpConnectionHandlers {
             connection: Smtp::new(),
@@ -500,9 +559,9 @@ pub(crate) struct ImapConnectionState {
 impl ImapConnectionState {
     /// Construct a new connection.
     fn new() -> (Self, ImapConnectionHandlers) {
-        let (stop_sender, stop_receiver) = channel(1);
-        let (shutdown_sender, shutdown_receiver) = channel(1);
-        let (idle_interrupt_sender, idle_interrupt_receiver) = channel(1);
+        let (stop_sender, stop_receiver) = channel::bounded(1);
+        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
+        let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = ImapConnectionHandlers {
             connection: Imap::new(idle_interrupt_receiver),

src/securejoin/bobstate.rs

@@ -0,0 +1,525 @@
+//! Secure-Join protocol state machine for Bob, the joiner-side.
+//!
+//! This module contains the state machine to run the Secure-Join handshake for Bob and does
+//! not do any user interaction required by the protocol.  Instead the state machine
+//! provides all the information to its driver so it can perform the correct interactions.
+//!
+//! The [`BobState`] is only directly used to initially create it when starting the
+//! protocol.  Afterwards it must be stored in a mutex and the [`BobStateHandle`] should be
+//! used to work with the state.
+
+use anyhow::{Error, Result};
+use async_std::sync::MutexGuard;
+
+use crate::chat::{self, ChatId};
+use crate::constants::{Blocked, Viewtype};
+use crate::contact::{Contact, Origin};
+use crate::context::Context;
+use crate::events::EventType;
+use crate::headerdef::HeaderDef;
+use crate::key::{DcKey, SignedPublicKey};
+use crate::message::Message;
+use crate::mimeparser::{MimeMessage, SystemMessage};
+use crate::param::Param;
+
+use super::qrinvite::QrInvite;
+use super::{
+    encrypted_and_signed, fingerprint_equals_sender, mark_peer_as_verified, JoinError, SendMsgError,
+};
+
+/// The stage of the [`BobState`] securejoin handshake protocol state machine.
+///
+/// This does not concern itself with user interactions, only represents what happened to
+/// the protocol state machine from handling this message.
+#[derive(Clone, Copy, Debug, Display)]
+pub enum BobHandshakeStage {
+    /// Step 2 completed: (vc|vg)-request message sent.
+    ///
+    /// Note that this is only ever returned by [`BobState::start_protocol`] and never by
+    /// [`BobState::handle_message`].
+    RequestSent,
+    /// Step 4 completed: (vc|vg)-request-with-auth message sent.
+    RequestWithAuthSent,
+    /// The protocol completed successfully.
+    Completed,
+    /// The protocol prematurely terminated with given reason.
+    Terminated(&'static str),
+}
+
+/// A handle to work with the [`BobState`] of Bob's securejoin protocol.
+///
+/// This handle can only be created for when an underlying [`BobState`] exists.  It keeps
+/// open a lock which guarantees unique access to the state and this struct must be dropped
+/// to return the lock.
+pub struct BobStateHandle<'a> {
+    guard: MutexGuard<'a, Option<BobState>>,
+    bobstate: BobState,
+    clear_state_on_drop: bool,
+}
+
+impl<'a> BobStateHandle<'a> {
+    /// Creates a new instance, upholding the guarantee that [`BobState`] must exist.
+    pub fn from_guard(mut guard: MutexGuard<'a, Option<BobState>>) -> Option<Self> {
+        match guard.take() {
+            Some(bobstate) => Some(Self {
+                guard,
+                bobstate,
+                clear_state_on_drop: false,
+            }),
+            None => None,
+        }
+    }
+
+    /// Returns the [`ChatId`] of the 1:1 chat with the inviter (Alice).
+    pub fn chat_id(&self) -> ChatId {
+        self.bobstate.chat_id
+    }
+
+    /// Returns a reference to the [`QrInvite`] of the joiner process.
+    pub fn invite(&self) -> &QrInvite {
+        &self.bobstate.invite
+    }
+
+    /// Handles the given message for the securejoin handshake for Bob.
+    ///
+    /// This proxies to [`BobState::handle_message`] and makes sure to clear the state when
+    /// the protocol state is terminal.  It returns `Some` if the message successfully
+    /// advanced the state of the protocol state machine, `None` otherwise.
+    pub async fn handle_message(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Option<BobHandshakeStage> {
+        info!(context, "Handling securejoin message for BobStateHandle");
+        match self.bobstate.handle_message(context, mime_message).await {
+            Ok(Some(stage)) => {
+                if matches!(
+                    stage,
+                    BobHandshakeStage::Completed | BobHandshakeStage::Terminated(_)
+                ) {
+                    self.finish_protocol(context).await;
+                }
+                Some(stage)
+            }
+            Ok(None) => None,
+            Err(err) => {
+                warn!(
+                    context,
+                    "Error handling handshake message, aborting handshake: {}", err
+                );
+                self.finish_protocol(context).await;
+                None
+            }
+        }
+    }
+
+    /// Marks the bob handshake as finished.
+    ///
+    /// This will clear the state on [`InnerContext::bob`] once this handle is dropped,
+    /// allowing a new handshake to be started from [`Bob`].
+    ///
+    /// Note that the state is only cleared on Drop since otherwise the invariant that the
+    /// state is always consistent is violated.  However the "ongoing" process is released
+    /// here a little bit earlier as this requires access to the Context, which we do not
+    /// have on Drop (Drop can not run asynchronous code).  Stopping the "ongoing" process
+    /// will release [`securejoin`](super::securejoin) which in turn will finally free the
+    /// ongoing process using [`Context::free_ongoing`].
+    ///
+    /// [`InnerContext::bob`]: crate::context::InnerContext::bob
+    /// [`Bob`]: super::Bob
+    async fn finish_protocol(&mut self, context: &Context) {
+        info!(context, "Finishing securejoin handshake protocol for Bob");
+        self.clear_state_on_drop = true;
+        if let QrInvite::Group { .. } = self.bobstate.invite {
+            context.stop_ongoing().await;
+        }
+    }
+}
+
+impl<'a> Drop for BobStateHandle<'a> {
+    fn drop(&mut self) {
+        if self.clear_state_on_drop {
+            // The Option should already be empty because we take it out in the ctor,
+            // however the typesystem doesn't guarantee this so do it again anyway.
+            self.guard.take();
+        } else {
+            // Make sure to put back the BobState into the Option of the Mutex, it was taken
+            // out by the constructor.
+            self.guard.replace(self.bobstate.clone());
+        }
+    }
+}
+
+/// The securejoin state kept in-memory while Bob is joining.
+///
+/// This is currently stored in [`Bob`] which is stored on the [`Context`], thus Bob can
+/// only run one securejoin joiner protocol at a time.
+///
+/// This purposefully has nothing optional, the state is always fully valid.  See
+/// [`Bob::state`] to get access to this state.
+///
+/// # Conducting the securejoin handshake
+///
+/// The methods on this struct allow you to interact with the state and thus conduct the
+/// securejoin handshake for Bob.  The methods only concern themselves with the protocol
+/// state and explicitly avoid performing any user interactions required by securejoin.
+/// This simplifies the concerns and logic required in both the callers and in the state
+/// management.  The return values can be used to understand what user interactions need to
+/// happen.
+///
+/// [`Bob`]: super::Bob
+/// [`Bob::state`]: super::Bob::state
+#[derive(Debug, Clone)]
+pub struct BobState {
+    /// The QR Invite code.
+    invite: QrInvite,
+    /// The next expected message from Alice.
+    next: SecureJoinStep,
+    /// The [`ChatId`] of the 1:1 chat with Alice, matching [`QrInvite::contact_id`].
+    chat_id: ChatId,
+}
+
+impl BobState {
+    /// Starts the securejoin protocol and creates a new [`BobState`].
+    ///
+    /// # Bob - the joiner's side
+    /// ## Step 2 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    pub async fn start_protocol(
+        context: &Context,
+        invite: QrInvite,
+    ) -> Result<(Self, BobHandshakeStage), JoinError> {
+        let chat_id = chat::create_by_contact_id(context, invite.contact_id())
+            .await
+            .map_err(JoinError::UnknownContact)?;
+        if fingerprint_equals_sender(context, invite.fingerprint(), chat_id).await {
+            // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
+            info!(context, "Taking securejoin protocol shortcut");
+            let state = Self {
+                invite,
+                next: SecureJoinStep::ContactConfirm,
+                chat_id,
+            };
+            state
+                .send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
+                .await?;
+            Ok((state, BobHandshakeStage::RequestWithAuthSent))
+        } else {
+            let state = Self {
+                invite,
+                next: SecureJoinStep::AuthRequired,
+                chat_id,
+            };
+            state
+                .send_handshake_message(context, BobHandshakeMsg::Request)
+                .await?;
+            Ok((state, BobHandshakeStage::RequestSent))
+        }
+    }
+
+    /// Returns the [`QrInvite`] used to create this [`BobState`].
+    pub fn invite(&self) -> &QrInvite {
+        &self.invite
+    }
+
+    /// Handles the given message for the securejoin handshake for Bob.
+    ///
+    /// If the message was not used for this handshake `None` is returned, otherwise the new
+    /// stage is returned.  Once [`BobHandshakeStage::Completed`] or
+    /// [`BobHandshakeStage::Terminated`] are reached this [`BobState`] should be destroyed,
+    /// further calling it will just result in the messages being unused by this handshake.
+    ///
+    /// # Errors
+    ///
+    /// Under normal operation this should never return an error, regardless of what kind of
+    /// message it is called with.  Any errors therefore should be treated as fatal internal
+    /// errors and this entire [`BobState`] should be thrown away as the state machine can
+    /// no longer be considered consistent.
+    async fn handle_message(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Result<Option<BobHandshakeStage>> {
+        let step = match mime_message.get(HeaderDef::SecureJoin) {
+            Some(step) => step,
+            None => {
+                warn!(
+                    context,
+                    "Message has no Secure-Join header: {}",
+                    mime_message.get_rfc724_mid().unwrap_or_default()
+                );
+                return Ok(None);
+            }
+        };
+        if !self.is_msg_expected(context, step.as_str()) {
+            info!(context, "{} message out of sync for BobState", step);
+            return Ok(None);
+        }
+        match step.as_str() {
+            "vg-auth-required" | "vc-auth-required" => {
+                self.step_auth_required(context, mime_message).await
+            }
+            "vg-member-added" | "vc-contact-confirm" => {
+                self.step_contact_confirm(context, mime_message).await
+            }
+            _ => {
+                warn!(context, "Invalid step for BobState: {}", step);
+                Ok(None)
+            }
+        }
+    }
+
+    /// Returns `true` if the message is expected according to the protocol.
+    fn is_msg_expected(&self, context: &Context, step: &str) -> bool {
+        let variant_matches = match self.invite {
+            QrInvite::Contact { .. } => step.starts_with("vc-"),
+            QrInvite::Group { .. } => step.starts_with("vg-"),
+        };
+        let step_matches = self.next.matches(context, step);
+        variant_matches && step_matches
+    }
+
+    /// Handles a *vc-auth-required* or *vg-auth-required* message.
+    ///
+    /// # Bob - the joiner's side
+    /// ## Step 4 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    async fn step_auth_required(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Result<Option<BobHandshakeStage>> {
+        info!(
+            context,
+            "Bob Step 4 - handling vc-auth-require/vg-auth-required message"
+        );
+        if !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint())) {
+            let reason = if mime_message.was_encrypted() {
+                "Valid signature missing"
+            } else {
+                "Required encryption missing"
+            };
+            self.next = SecureJoinStep::Terminated;
+            return Ok(Some(BobHandshakeStage::Terminated(reason)));
+        }
+        if !fingerprint_equals_sender(context, self.invite.fingerprint(), self.chat_id).await {
+            self.next = SecureJoinStep::Terminated;
+            return Ok(Some(BobHandshakeStage::Terminated("Fingerprint mismatch")));
+        }
+        info!(context, "Fingerprint verified.",);
+        self.next = SecureJoinStep::ContactConfirm;
+        self.send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
+            .await?;
+        Ok(Some(BobHandshakeStage::RequestWithAuthSent))
+    }
+
+    /// Handles a *vc-contact-confirm* or *vg-member-added* message.
+    ///
+    /// # Bob - the joiner's side
+    /// ## Step 7 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+    ///
+    /// This deviates from the protocol by also sending a confirmation message in response
+    /// to the *vc-contact-confirm* message.  This has no specific value to the protocol and
+    /// is only done out of symmerty with *vg-member-added* handling.
+    async fn step_contact_confirm(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Result<Option<BobHandshakeStage>> {
+        info!(
+            context,
+            "Bob Step 7 - handling vc-contact-confirm/vg-member-added message"
+        );
+        let vg_expect_encrypted = match self.invite {
+            QrInvite::Contact { .. } => {
+                // setup-contact is always encrypted
+                true
+            }
+            QrInvite::Group { ref grpid, .. } => {
+                // This is buggy, is_verified_group will always be
+                // false since the group is created by receive_imf for
+                // the very handshake message we're handling now.  But
+                // only after we have returned.  It does not impact
+                // the security invariants of secure-join however.
+                let (_, is_verified_group, _) = chat::get_chat_id_by_grpid(context, grpid)
+                    .await
+                    .unwrap_or((ChatId::new(0), false, Blocked::Not));
+                // when joining a non-verified group
+                // the vg-member-added message may be unencrypted
+                // when not all group members have keys or prefer encryption.
+                // So only expect encryption if this is a verified group
+                is_verified_group
+            }
+        };
+        if vg_expect_encrypted
+            && !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint()))
+        {
+            self.next = SecureJoinStep::Terminated;
+            return Ok(Some(BobHandshakeStage::Terminated(
+                "Contact confirm message not encrypted",
+            )));
+        }
+        mark_peer_as_verified(context, self.invite.fingerprint()).await?;
+        Contact::scaleup_origin_by_id(context, self.invite.contact_id(), Origin::SecurejoinJoined)
+            .await;
+        emit_event!(context, EventType::ContactsChanged(None));
+
+        if let QrInvite::Group { .. } = self.invite {
+            let member_added = mime_message
+                .get(HeaderDef::ChatGroupMemberAdded)
+                .map(|s| s.as_str())
+                .ok_or_else(|| Error::msg("Missing Chat-Group-Member-Added header"))?;
+            if !context.is_self_addr(member_added).await? {
+                info!(context, "Message belongs to a different handshake (scaled up contact anyway to allow creation of group).");
+                return Ok(None);
+            }
+        }
+
+        self.send_handshake_message(context, BobHandshakeMsg::ContactConfirmReceived)
+            .await
+            .map_err(|_| {
+                warn!(
+                    context,
+                    "Failed to send vc-contact-confirm-received/vg-member-added-received"
+                );
+            })
+            // This is not an error affecting the protocol outcome.
+            .ok();
+
+        self.next = SecureJoinStep::Completed;
+        Ok(Some(BobHandshakeStage::Completed))
+    }
+
+    /// Sends the requested handshake message to Alice.
+    ///
+    /// This takes care of adding the required headers for the step.
+    async fn send_handshake_message(
+        &self,
+        context: &Context,
+        step: BobHandshakeMsg,
+    ) -> Result<(), SendMsgError> {
+        let mut msg = Message {
+            viewtype: Viewtype::Text,
+            text: Some(step.body_text(&self.invite)),
+            hidden: true,
+            ..Default::default()
+        };
+        msg.param.set_cmd(SystemMessage::SecurejoinMessage);
+
+        // Sends the step in Secure-Join header.
+        msg.param
+            .set(Param::Arg, step.securejoin_header(&self.invite));
+
+        match step {
+            BobHandshakeMsg::Request => {
+                // Sends the Secure-Join-Invitenumber header in mimefactory.rs.
+                msg.param.set(Param::Arg2, self.invite.invitenumber());
+                msg.param.set_int(Param::ForcePlaintext, 1);
+            }
+            BobHandshakeMsg::RequestWithAuth => {
+                // Sends the Secure-Join-Auth header in mimefactory.rs.
+                msg.param.set(Param::Arg2, self.invite.authcode());
+                msg.param.set_int(Param::GuaranteeE2ee, 1);
+            }
+            BobHandshakeMsg::ContactConfirmReceived => {
+                msg.param.set_int(Param::GuaranteeE2ee, 1);
+            }
+        };
+
+        // Sends our own fingerprint in the Secure-Join-Fingerprint header.
+        let bob_fp = SignedPublicKey::load_self(context).await?.fingerprint();
+        msg.param.set(Param::Arg3, bob_fp.hex());
+
+        // Sends the grpid in the Secure-Join-Group header.
+        if let QrInvite::Group { ref grpid, .. } = self.invite {
+            msg.param.set(Param::Arg4, grpid);
+        }
+
+        chat::send_msg(context, self.chat_id, &mut msg).await?;
+        Ok(())
+    }
+}
+
+/// Identifies the SecureJoin handshake messages Bob can send.
+enum BobHandshakeMsg {
+    /// vc-request or vg-request
+    Request,
+    /// vc-request-with-auth or vg-request-with-auth
+    RequestWithAuth,
+    /// vc-contact-confirm-received or vg-member-added-received
+    ContactConfirmReceived,
+}
+
+impl BobHandshakeMsg {
+    /// Returns the text to send in the body of the handshake message.
+    ///
+    /// This text has no significance to the protocol, but would be visible if users see
+    /// this email message directly, e.g. when accessing their email without using
+    /// DeltaChat.
+    fn body_text(&self, invite: &QrInvite) -> String {
+        format!("Secure-Join: {}", self.securejoin_header(invite))
+    }
+
+    /// Returns the `Secure-Join` header value.
+    ///
+    /// This identifies the step this message is sending information about.  Most protocol
+    /// steps include additional information into other headers, see
+    /// [`BobState::send_handshake_message`] for these.
+    fn securejoin_header(&self, invite: &QrInvite) -> &'static str {
+        match self {
+            Self::Request => match invite {
+                QrInvite::Contact { .. } => "vc-request",
+                QrInvite::Group { .. } => "vg-request",
+            },
+            Self::RequestWithAuth => match invite {
+                QrInvite::Contact { .. } => "vc-request-with-auth",
+                QrInvite::Group { .. } => "vg-request-with-auth",
+            },
+            Self::ContactConfirmReceived => match invite {
+                QrInvite::Contact { .. } => "vc-contact-confirm-received",
+                QrInvite::Group { .. } => "vg-member-added-received",
+            },
+        }
+    }
+}
+
+/// The next message expected by [`BobState`] in the setup-contact/secure-join protocol.
+#[derive(Debug, Clone, PartialEq)]
+enum SecureJoinStep {
+    /// Expecting the auth-required message.
+    ///
+    /// This corresponds to the `vc-auth-required` or `vg-auth-required` message of step 3d.
+    AuthRequired,
+    /// Expecting the contact-confirm message.
+    ///
+    /// This corresponds to the `vc-contact-confirm` or `vg-member-added` message of step
+    /// 6b.
+    ContactConfirm,
+    /// The protocol terminated because of an error.
+    ///
+    /// The securejoin protocol terminated, this exists to ensure [`BobState`] can detect
+    /// when it earlier signalled that is should be terminated.  It is an error to call with
+    /// this state.
+    Terminated,
+    /// The protocol completed.
+    ///
+    /// This exists to ensure [`BobState`] can detect when it earlier signalled that it is
+    /// complete.  It is an error to call with this state.
+    Completed,
+}
+
+impl SecureJoinStep {
+    /// Compares the legacy string representation of a step to a [`SecureJoinStep`] variant.
+    fn matches(&self, context: &Context, step: &str) -> bool {
+        match self {
+            Self::AuthRequired => step == "vc-auth-required" || step == "vg-auth-required",
+            Self::ContactConfirm => step == "vc-contact-confirm" || step == "vg-member-added",
+            SecureJoinStep::Terminated => {
+                warn!(context, "Terminated state for next securejoin step");
+                false
+            }
+            SecureJoinStep::Completed => {
+                warn!(context, "Complted state for next securejoin step");
+                false
+            }
+        }
+    }
+}

src/securejoin/qrinvite.rs

@@ -0,0 +1,120 @@
+//! Supporting code for the QR-code invite.
+//!
+//! QR-codes are decoded into a more general-purpose [`Lot`] struct normally, this struct is
+//! so general it is not even specific to QR-codes.  This makes working with it rather hard,
+//! so here we have a wrapper type that specifically deals with Secure-Join QR-codes so
+//! that the Secure-Join code can have many more guarantees when dealing with this.
+
+use std::convert::TryFrom;
+
+use anyhow::Result;
+
+use crate::key::{Fingerprint, FingerprintError};
+use crate::lot::{Lot, LotState};
+
+/// Represents the data from a QR-code scan.
+///
+/// There are methods to conveniently access fields present in both variants.
+#[derive(Debug, Clone)]
+pub enum QrInvite {
+    Contact {
+        contact_id: u32,
+        fingerprint: Fingerprint,
+        invitenumber: String,
+        authcode: String,
+    },
+    Group {
+        contact_id: u32,
+        fingerprint: Fingerprint,
+        name: String,
+        grpid: String,
+        invitenumber: String,
+        authcode: String,
+    },
+}
+
+impl QrInvite {
+    /// The contact ID of the inviter.
+    ///
+    /// The actual QR-code contains a URL-encoded email address, but upon scanning this is
+    /// translated to a contact ID.
+    pub fn contact_id(&self) -> u32 {
+        match self {
+            Self::Contact { contact_id, .. } | Self::Group { contact_id, .. } => *contact_id,
+        }
+    }
+
+    /// The fingerprint of the inviter.
+    pub fn fingerprint(&self) -> &Fingerprint {
+        match self {
+            Self::Contact { fingerprint, .. } | Self::Group { fingerprint, .. } => fingerprint,
+        }
+    }
+
+    /// The `INVITENUMBER` of the setup-contact/secure-join protocol.
+    pub fn invitenumber(&self) -> &str {
+        match self {
+            Self::Contact { invitenumber, .. } | Self::Group { invitenumber, .. } => invitenumber,
+        }
+    }
+
+    /// The `AUTH` code of the setup-contact/secure-join protocol.
+    pub fn authcode(&self) -> &str {
+        match self {
+            Self::Contact { authcode, .. } | Self::Group { authcode, .. } => authcode,
+        }
+    }
+}
+
+impl TryFrom<Lot> for QrInvite {
+    type Error = QrError;
+
+    fn try_from(lot: Lot) -> Result<Self, Self::Error> {
+        if lot.state != LotState::QrAskVerifyContact && lot.state != LotState::QrAskVerifyGroup {
+            return Err(QrError::UnsupportedProtocol);
+        }
+        if lot.id == 0 {
+            return Err(QrError::MissingContactId);
+        }
+        let fingerprint = lot.fingerprint.ok_or(QrError::MissingFingerprint)?;
+        let invitenumber = lot.invitenumber.ok_or(QrError::MissingInviteNumber)?;
+        let authcode = lot.auth.ok_or(QrError::MissingAuthCode)?;
+        match lot.state {
+            LotState::QrAskVerifyContact => Ok(QrInvite::Contact {
+                contact_id: lot.id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            }),
+            LotState::QrAskVerifyGroup => Ok(QrInvite::Group {
+                contact_id: lot.id,
+                fingerprint,
+                name: lot.text1.ok_or(QrError::MissingGroupName)?,
+                grpid: lot.text2.ok_or(QrError::MissingGroupId)?,
+                invitenumber,
+                authcode,
+            }),
+            _ => Err(QrError::UnsupportedProtocol),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum QrError {
+    #[error("Unsupported protocol in QR-code")]
+    UnsupportedProtocol,
+    #[error("Failed to read fingerprint")]
+    InvalidFingerprint(#[from] FingerprintError),
+    #[error("Missing fingerprint")]
+    MissingFingerprint,
+    #[error("Missing invitenumber")]
+    MissingInviteNumber,
+    #[error("Missing auth code")]
+    MissingAuthCode,
+    #[error("Missing group name")]
+    MissingGroupName,
+    #[error("Missing group id")]
+    MissingGroupId,
+    #[error("Missing contact id")]
+    MissingContactId,
+}

src/simplify.rs

@@ -1,3 +1,5 @@
+use itertools::Itertools;
+
 // protect lines starting with `--` against being treated as a footer.
 // for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B);
 // this should be invisible on most systems and there is no need to unescape it again
@@ -15,18 +17,23 @@ pub fn escape_message_footer_marks(text: &str) -> String {
 }
 
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
+/// Returns `(lines, footer_lines)` tuple;
+/// `footer_lines` is set to `Some` if the footer was actually removed from `lines`
+/// (which is equal to the input array otherwise).
 #[allow(clippy::indexing_slicing)]
-fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
+fn remove_message_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<&'a [&'a str]>) {
     let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
         match line {
             // some providers encode `-- ` to `-- =20` which results in `--  `
-            "-- " | "--  " => return &lines[..ix],
+            "-- " | "--  " => return (&lines[..ix], lines.get(ix + 1..)),
             // some providers encode `-- ` to `=2D-` which results in only `--`;
             // use that only when no other footer is found
             // and if the line before is empty and the line after is not empty
             "--" => {
-                if (ix == 0 || lines[ix - 1] == "") && ix != lines.len() - 1 && lines[ix + 1] != ""
+                if (ix == 0 || lines[ix - 1].is_empty())
+                    && ix != lines.len() - 1
+                    && !lines[ix + 1].is_empty()
                 {
                     nearly_standard_footer = Some(ix);
                 }
@@ -35,13 +42,15 @@ fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
         }
     }
     if let Some(ix) = nearly_standard_footer {
-        return &lines[..ix];
+        return (&lines[..ix], lines.get(ix + 1..));
     }
-    lines
+    (lines, None)
 }
 
-/// Remove nonstandard footer and a boolean indicating whether such
-/// footer was removed.
+/// Remove nonstandard footer and a boolean indicating whether such footer was removed.
+/// Returns `(lines, is_footer_removed)` tuple;
+/// `is_footer_removed` is set to `true` if the footer was actually removed from `lines`
+/// (which is equal to the input array otherwise).
 #[allow(clippy::indexing_slicing)]
 fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     for (ix, &line) in lines.iter().enumerate() {
@@ -58,39 +67,58 @@ fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     (lines, false)
 }
 
-fn split_lines(buf: &str) -> Vec<&str> {
+pub(crate) fn split_lines(buf: &str) -> Vec<&str> {
     buf.split('\n').collect()
 }
 
 /// Simplify message text for chat display.
 /// Remove quotes, signatures, trailing empty lines etc.
-pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
+/// Returns `(text, is_forwarded, is_cut, quote, footer)` tuple,
+/// returning the simplified text and some additional information gained from the input.
+pub fn simplify(
+    mut input: String,
+    is_chat_message: bool,
+) -> (String, bool, bool, Option<String>, Option<String>) {
+    let mut is_cut = false;
+
     input.retain(|c| c != '\r');
     let lines = split_lines(&input);
     let (lines, is_forwarded) = skip_forward_header(&lines);
 
+    let (lines, mut top_quote) = remove_top_quote(lines);
     let original_lines = &lines;
-
-    let lines = remove_message_footer(lines);
+    let (lines, footer_lines) = remove_message_footer(lines);
+    let footer = footer_lines.map(|footer_lines| render_message(footer_lines, false));
+    is_cut = is_cut || footer.is_some();
 
     let text = if is_chat_message {
-        render_message(lines, false, false)
+        render_message(lines, false)
     } else {
         let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
-        let (lines, has_bottom_quote) = remove_bottom_quote(lines);
-        let (lines, has_top_quote) = remove_top_quote(lines);
+        let (lines, mut bottom_quote) = remove_bottom_quote(lines);
+
+        if top_quote.is_none() && bottom_quote.is_some() {
+            std::mem::swap(&mut top_quote, &mut bottom_quote);
+        }
 
         if lines.iter().all(|it| it.trim().is_empty()) {
-            render_message(original_lines, false, false)
+            render_message(original_lines, false)
         } else {
-            render_message(
-                lines,
-                has_top_quote,
-                has_nonstandard_footer || has_bottom_quote,
-            )
+            is_cut = is_cut || has_nonstandard_footer || bottom_quote.is_some();
+            render_message(lines, has_nonstandard_footer || bottom_quote.is_some())
         }
     };
-    (text, is_forwarded)
+
+    if !is_chat_message {
+        top_quote = top_quote.map(|quote| {
+            let quote_lines = split_lines(&quote);
+            let (quote_lines, quote_footer_lines) = remove_message_footer(&quote_lines);
+            is_cut = is_cut || quote_footer_lines.is_some();
+
+            render_message(quote_lines, false)
+        });
+    }
+    (text, is_forwarded, is_cut, top_quote, footer)
 }
 
 /// Skips "forwarded message" header.
@@ -108,16 +136,27 @@ fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
 }
 
 #[allow(clippy::indexing_slicing)]
-fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<String>) {
+    let mut first_quoted_line = lines.len();
     let mut last_quoted_line = None;
     for (l, line) in lines.iter().enumerate().rev() {
         if is_plain_quote(line) {
+            if last_quoted_line.is_none() {
+                first_quoted_line = l + 1;
+            }
             last_quoted_line = Some(l)
         } else if !is_empty_line(line) {
             break;
         }
     }
     if let Some(mut l_last) = last_quoted_line {
+        let quoted_text = lines[l_last..first_quoted_line]
+            .iter()
+            .map(|s| {
+                s.strip_prefix(">")
+                    .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u))
+            })
+            .join("\n");
         if l_last > 1 && is_empty_line(lines[l_last - 1]) {
             l_last -= 1
         }
@@ -127,18 +166,22 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
                 l_last -= 1
             }
         }
-        (&lines[..l_last], true)
+        (&lines[..l_last], Some(quoted_text))
     } else {
-        (lines, false)
+        (lines, None)
     }
 }
 
 #[allow(clippy::indexing_slicing)]
-fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<String>) {
+    let mut first_quoted_line = 0;
     let mut last_quoted_line = None;
     let mut has_quoted_headline = false;
     for (l, line) in lines.iter().enumerate() {
         if is_plain_quote(line) {
+            if last_quoted_line.is_none() {
+                first_quoted_line = l;
+            }
             last_quoted_line = Some(l)
         } else if !is_empty_line(line) {
             if is_quoted_headline(line) && !has_quoted_headline && last_quoted_line.is_none() {
@@ -150,17 +193,25 @@ fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
         }
     }
     if let Some(last_quoted_line) = last_quoted_line {
-        (&lines[last_quoted_line + 1..], true)
+        (
+            &lines[last_quoted_line + 1..],
+            Some(
+                lines[first_quoted_line..last_quoted_line + 1]
+                    .iter()
+                    .map(|s| {
+                        s.strip_prefix(">")
+                            .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u))
+                    })
+                    .join("\n"),
+            ),
+        )
     } else {
-        (lines, false)
+        (lines, None)
     }
 }
 
-fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) -> String {
+fn render_message(lines: &[&str], is_cut_at_end: bool) -> String {
     let mut ret = String::new();
-    if is_cut_at_begin {
-        ret += "[...]";
-    }
     /* we write empty lines only in case and non-empty line follows */
     let mut pending_linebreaks = 0;
     let mut empty_body = true;
@@ -183,7 +234,7 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) ->
             pending_linebreaks = 1
         }
     }
-    if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
+    if is_cut_at_end && !empty_body {
         ret += " [...]";
     }
     // redo escaping done by escape_message_footer_marks()
@@ -194,19 +245,10 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) ->
  * Tools
  */
 fn is_empty_line(buf: &str) -> bool {
-    // XXX: can it be simplified to buf.chars().all(|c| c.is_whitespace())?
-    //
-    // Strictly speaking, it is not equivalent (^A is not whitespace, but less than ' '),
-    // but having control sequences in email body?!
-    //
-    // See discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
-    for c in buf.chars() {
-        if c > ' ' {
-            return false;
-        }
-    }
-
-    true
+    buf.chars().all(char::is_whitespace)
+    // for some time, this checked for `char <= ' '`,
+    // see discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
+    // and https://github.com/deltachat/deltachat-core-rust/pull/2104/files#r538973613
 }
 
 fn is_quoted_headline(buf: &str) -> bool {
@@ -231,7 +273,7 @@ mod tests {
         #[test]
         // proptest does not support [[:graphical:][:space:]] regex.
         fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let (output, _is_forwarded) = simplify(input, true);
+            let (output, _is_forwarded, _, _, _) = simplify(input, true);
             assert!(output.split('\n').all(|s| s != "-- "));
         }
     }
@@ -239,38 +281,47 @@ mod tests {
     #[test]
     fn test_dont_remove_whole_message() {
         let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, is_cut, _, _) = simplify(input, false);
         assert_eq!(
             plain,
             "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
         );
         assert!(!is_forwarded);
+        assert!(!is_cut);
     }
 
     #[test]
     fn test_chat_message() {
         let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
-        let (plain, is_forwarded) = simplify(input, true);
+        let (plain, is_forwarded, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
         assert!(!is_forwarded);
+        assert!(is_cut);
+        assert_eq!(
+            footer.unwrap(),
+            "Sent with my Delta Chat Messenger: https://delta.chat"
+        );
     }
 
     #[test]
     fn test_simplify_trim() {
         let input = "line1\n\r\r\rline2".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, is_cut, _, _) = simplify(input, false);
 
         assert_eq!(plain, "line1\nline2");
         assert!(!is_forwarded);
+        assert!(!is_cut);
     }
 
     #[test]
     fn test_simplify_forwarded_message() {
         let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, is_cut, _, footer) = simplify(input, false);
 
         assert_eq!(plain, "Forwarded message");
         assert!(is_forwarded);
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "Signature goes here");
     }
 
     #[test]
@@ -287,17 +338,17 @@ mod tests {
 
     #[test]
     fn test_remove_top_quote() {
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second"]);
+        let (lines, top_quote) = remove_top_quote(&["> first", "> second"]);
         assert!(lines.is_empty());
-        assert!(has_top_quote);
+        assert_eq!(top_quote.unwrap(), "first\nsecond");
 
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
+        let (lines, top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
         assert_eq!(lines, &["not a quote"]);
-        assert!(has_top_quote);
+        assert_eq!(top_quote.unwrap(), "first\nsecond");
 
-        let (lines, has_top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
+        let (lines, top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
         assert_eq!(lines, &["not a quote", "> first", "> second"]);
-        assert!(!has_top_quote);
+        assert!(top_quote.is_none());
     }
 
     #[test]
@@ -312,41 +363,60 @@ mod tests {
     #[test]
     fn test_remove_message_footer() {
         let input = "text\n--\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n--\nno footer");
+        assert_eq!(footer, None);
+        assert!(!is_cut);
 
         let input = "text\n\n--\n\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\n\nno footer");
+        assert_eq!(footer, None);
+        assert!(!is_cut);
 
         let input = "text\n\n-- no footer\n\n".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n\n-- no footer");
+        assert_eq!(footer, None);
 
         let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, is_cut, _, footer) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\nno footer");
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "footer");
 
         let input = "text\n\n--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), true);
         assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "treated as footer when unescaped");
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
+        let (plain, _, is_cut, _, footer) = simplify(escaped, true);
         assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
+        assert!(!is_cut);
+        assert_eq!(footer, None);
 
         // Nonstandard footer sent by https://siju.es/
         let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
-        let (plain, _) = simplify(input.clone(), false);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), false);
         assert_eq!(plain, "Message text here [...]");
-        let (plain, _) = simplify(input.clone(), true);
+        assert!(is_cut);
+        assert_eq!(footer, None);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), true);
         assert_eq!(plain, input);
+        assert!(!is_cut);
+        assert_eq!(footer, None);
 
         let input = "--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, is_cut, _, footer) = simplify(input.clone(), true);
         assert_eq!(plain, ""); // see remove_message_footer() for some explanations
+        assert!(is_cut);
+        assert_eq!(footer.unwrap(), "treated as footer when unescaped");
 
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
+        let (plain, _, is_cut, _, footer) = simplify(escaped, true);
         assert_eq!(plain, "--\ntreated as footer when unescaped");
+        assert!(!is_cut);
+        assert_eq!(footer, None);
     }
 }

src/smtp/mod.rs

@@ -4,16 +4,16 @@ pub mod send;
 
 use std::time::{Duration, SystemTime};
 
-use async_smtp::smtp::client::net::*;
-use async_smtp::*;
+use async_smtp::smtp::client::net::ClientTlsParameters;
+use async_smtp::{error, smtp, EmailAddress};
 
-use crate::constants::*;
+use crate::constants::DC_LP_AUTH_OAUTH2;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam};
-use crate::oauth2::*;
-use crate::provider::{get_provider_info, Socket};
-use crate::stock::StockMessage;
+use crate::oauth2::dc_get_oauth2_access_token;
+use crate::provider::Socket;
+use crate::stock_str;
 
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
@@ -30,7 +30,7 @@ pub enum Error {
         error: error::Error,
     },
 
-    #[error("SMTP: failed to connect: {0:?}")]
+    #[error("SMTP: failed to connect: {0}")]
     ConnectionFailure(#[source] smtp::error::Error),
 
     #[error("SMTP: failed to setup connection {0:?}")]
@@ -107,16 +107,16 @@ impl Smtp {
                 &lp.smtp,
                 &lp.addr,
                 lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+                lp.provider.map_or(false, |provider| provider.strict_tls),
             )
             .await;
         if let Err(ref err) = res {
-            let message = context
-                .stock_string_repl_str2(
-                    StockMessage::ServerResponse,
-                    format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
-                    err.to_string(),
-                )
-                .await;
+            let message = stock_str::server_response(
+                context,
+                format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
+                err.to_string(),
+            )
+            .await;
 
             context.emit_event(EventType::ErrorNetwork(message));
         };
@@ -130,6 +130,7 @@ impl Smtp {
         lp: &ServerLoginParam,
         addr: &str,
         oauth2: bool,
+        provider_strict_tls: bool,
     ) -> Result<()> {
         if self.is_connected().await {
             warn!(context, "SMTP already connected.");
@@ -151,9 +152,8 @@ impl Smtp {
         let domain = &lp.server;
         let port = lp.port;
 
-        let provider = get_provider_info(addr);
         let strict_tls = match lp.certificate_checks {
-            CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
+            CertificateChecks::Automatic => provider_strict_tls,
             CertificateChecks::Strict => true,
             CertificateChecks::AcceptInvalidCertificates
             | CertificateChecks::AcceptInvalidCertificates2 => false,
@@ -192,7 +192,8 @@ impl Smtp {
         };
 
         let security = match lp.security {
-            Socket::STARTTLS | Socket::Plain => smtp::ClientSecurity::Opportunistic(tls_parameters),
+            Socket::Plain => smtp::ClientSecurity::None,
+            Socket::STARTTLS => smtp::ClientSecurity::Required(tls_parameters),
             _ => smtp::ClientSecurity::Wrapper(tls_parameters),
         };
 

src/smtp/send.rs

@@ -1,8 +1,9 @@
 //! # SMTP message sending
 
 use super::Smtp;
-use async_smtp::*;
+use async_smtp::{EmailAddress, Envelope, SendableEmail, Transport};
 
+use crate::constants::DEFAULT_MAX_SMTP_RCPT_TO;
 use crate::context::Context;
 use crate::events::EventType;
 use itertools::Itertools;
@@ -34,37 +35,46 @@ impl Smtp {
     ) -> Result<()> {
         let message_len_bytes = message.len();
 
-        let recipients_display = recipients.iter().map(|x| x.to_string()).join(",");
-
-        let envelope =
-            Envelope::new(self.from.clone(), recipients).map_err(Error::EnvelopeError)?;
-        let mail = SendableEmail::new(
-            envelope,
-            format!("{}", job_id), // only used for internal logging
-            message,
-        );
-
-        if let Some(ref mut transport) = self.transport {
-            // The timeout is 1min + 3min per MB.
-            let timeout = 60 + (180 * message_len_bytes / 1_000_000) as u64;
-            transport
-                .send_with_timeout(mail, Some(&Duration::from_secs(timeout)))
-                .await
-                .map_err(Error::SendError)?;
-
-            context.emit_event(EventType::SmtpMessageSent(format!(
-                "Message len={} was smtp-sent to {}",
-                message_len_bytes, recipients_display
-            )));
-            self.last_success = Some(std::time::SystemTime::now());
-
-            Ok(())
-        } else {
-            warn!(
-                context,
-                "uh? SMTP has no transport, failed to send to {}", recipients_display
-            );
-            Err(Error::NoTransport)
+        let mut chunk_size = DEFAULT_MAX_SMTP_RCPT_TO;
+        if let Some(provider) = context.get_configured_provider().await {
+            if let Some(max_smtp_rcpt_to) = provider.max_smtp_rcpt_to {
+                chunk_size = max_smtp_rcpt_to as usize;
+            }
         }
+
+        for recipients_chunk in recipients.chunks(chunk_size).into_iter() {
+            let recipients = recipients_chunk.to_vec();
+            let recipients_display = recipients.iter().map(|x| x.to_string()).join(",");
+
+            let envelope =
+                Envelope::new(self.from.clone(), recipients).map_err(Error::EnvelopeError)?;
+            let mail = SendableEmail::new(
+                envelope,
+                format!("{}", job_id), // only used for internal logging
+                &message,
+            );
+
+            if let Some(ref mut transport) = self.transport {
+                // The timeout is 1min + 3min per MB.
+                let timeout = 60 + (180 * message_len_bytes / 1_000_000) as u64;
+                transport
+                    .send_with_timeout(mail, Some(&Duration::from_secs(timeout)))
+                    .await
+                    .map_err(Error::SendError)?;
+
+                context.emit_event(EventType::SmtpMessageSent(format!(
+                    "Message len={} was smtp-sent to {}",
+                    message_len_bytes, recipients_display
+                )));
+                self.last_success = Some(std::time::SystemTime::now());
+            } else {
+                warn!(
+                    context,
+                    "uh? SMTP has no transport, failed to send to {}", recipients_display
+                );
+                return Err(Error::NoTransport);
+            }
+        }
+        Ok(())
     }
 }

src/sql.rs

@@ -7,15 +7,23 @@ use std::collections::HashSet;
 use std::path::Path;
 use std::time::Duration;
 
+use anyhow::format_err;
+use anyhow::Context as _;
 use rusqlite::{Connection, Error as SqlError, OpenFlags};
 
-use crate::chat::{update_device_icon, update_saved_messages_icon};
-use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH};
+use crate::chat::{add_device_msg, update_device_icon, update_saved_messages_icon};
+use crate::config::Config;
+use crate::config::Config::DeleteServerAfter;
+use crate::constants::{ShowEmails, Viewtype, DC_CHAT_ID_TRASH};
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_delete_file, time, EmailAddress};
 use crate::ephemeral::start_ephemeral_timers;
-use crate::param::*;
-use crate::peerstate::*;
+use crate::imap;
+use crate::message::Message;
+use crate::param::{Param, Params};
+use crate::peerstate::Peerstate;
+use crate::provider::get_provider_by_domain;
+use crate::stock_str;
 
 #[macro_export]
 macro_rules! paramsv {
@@ -44,7 +52,7 @@ pub enum Error {
     #[error("{0:?}")]
     BlobError(#[from] crate::blob::BlobError),
     #[error("{0}")]
-    Other(#[from] crate::error::Error),
+    Other(#[from] anyhow::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -77,18 +85,29 @@ impl Sql {
         // drop closes the connection
     }
 
-    // return true on success, false on failure
-    pub async fn open<T: AsRef<Path>>(&self, context: &Context, dbfile: T, readonly: bool) -> bool {
-        match open(context, self, dbfile, readonly).await {
-            Ok(_) => true,
-            Err(err) => match err.downcast_ref::<Error>() {
-                Some(Error::SqlAlreadyOpen) => false,
+    pub async fn open<T: AsRef<Path>>(
+        &self,
+        context: &Context,
+        dbfile: T,
+        readonly: bool,
+    ) -> anyhow::Result<()> {
+        let res = open(context, self, &dbfile, readonly).await;
+        if let Err(err) = &res {
+            match err.downcast_ref::<Error>() {
+                Some(Error::SqlAlreadyOpen) => {}
                 _ => {
                     self.close().await;
-                    false
                 }
-            },
+            }
         }
+        res.map_err(|e| {
+            format_err!(
+                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                "Could not open db file {}: {:#}",
+                dbfile.as_ref().to_string_lossy(),
+                e
+            )
+        })
     }
 
     pub async fn execute<S: AsRef<str>>(
@@ -130,7 +149,7 @@ impl Sql {
         &self,
     ) -> Result<r2d2::PooledConnection<r2d2_sqlite::SqliteConnectionManager>> {
         let lock = self.pool.read().await;
-        let pool = lock.as_ref().ok_or_else(|| Error::SqlNoConnection)?;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
         let conn = pool.get()?;
 
         Ok(conn)
@@ -144,7 +163,7 @@ impl Sql {
             + FnOnce(r2d2::PooledConnection<r2d2_sqlite::SqliteConnectionManager>) -> Result<H>,
     {
         let lock = self.pool.read().await;
-        let pool = lock.as_ref().ok_or_else(|| Error::SqlNoConnection)?;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
         let conn = pool.get()?;
 
         g(conn)
@@ -156,7 +175,7 @@ impl Sql {
         Fut: Future<Output = Result<H>> + Send,
     {
         let lock = self.pool.read().await;
-        let pool = lock.as_ref().ok_or_else(|| Error::SqlNoConnection)?;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
 
         let conn = pool.get()?;
         g(conn).await
@@ -208,6 +227,31 @@ impl Sql {
         .await
     }
 
+    /// Check if a column exists in a given table.
+    pub async fn col_exists(
+        &self,
+        table_name: impl AsRef<str>,
+        col_name: impl AsRef<str>,
+    ) -> Result<bool> {
+        let table_name = table_name.as_ref().to_string();
+        let col_name = col_name.as_ref().to_string();
+        self.with_conn(move |conn| {
+            let mut exists = false;
+            // `PRAGMA table_info` returns one row per column,
+            // each row containing 0=cid, 1=name, 2=type, 3=notnull, 4=dflt_value
+            conn.pragma(None, "table_info", &table_name, |row| {
+                let curr_name: String = row.get(1)?;
+                if col_name == curr_name {
+                    exists = true;
+                }
+                Ok(())
+            })?;
+
+            Ok(exists)
+        })
+        .await
+    }
+
     /// Execute a query which is expected to return zero or one row.
     pub async fn query_row_optional<T, F>(
         &self,
@@ -281,7 +325,7 @@ impl Sql {
         }
 
         let key = key.as_ref();
-        let res = if let Some(ref value) = value {
+        let res = if let Some(value) = value {
             let exists = self
                 .exists("SELECT value FROM config WHERE keyname=?;", paramsv![key])
                 .await?;
@@ -452,7 +496,11 @@ pub fn get_rowid2(
     )
 }
 
-pub async fn housekeeping(context: &Context) {
+pub async fn housekeeping(context: &Context) -> anyhow::Result<()> {
+    if let Err(err) = crate::ephemeral::delete_expired_messages(context).await {
+        warn!(context, "Failed to delete expired messages: {}", err);
+    }
+
     let mut files_in_use = HashSet::new();
     let mut unreferenced_count = 0;
 
@@ -463,28 +511,28 @@ pub async fn housekeeping(context: &Context) {
         "SELECT param FROM msgs  WHERE chat_id!=3   AND type!=10;",
         Param::File,
     )
-    .await;
+    .await?;
     maybe_add_from_param(
         context,
         &mut files_in_use,
         "SELECT param FROM jobs;",
         Param::File,
     )
-    .await;
+    .await?;
     maybe_add_from_param(
         context,
         &mut files_in_use,
         "SELECT param FROM chats;",
         Param::ProfileImage,
     )
-    .await;
+    .await?;
     maybe_add_from_param(
         context,
         &mut files_in_use,
         "SELECT param FROM contacts;",
         Param::ProfileImage,
     )
-    .await;
+    .await?;
 
     context
         .sql
@@ -500,9 +548,7 @@ pub async fn housekeeping(context: &Context) {
             },
         )
         .await
-        .unwrap_or_else(|err| {
-            warn!(context, "sql: failed query: {}", err);
-        });
+        .context("housekeeping: failed to SELECT value FROM config")?;
 
     info!(context, "{} files in use.", files_in_use.len(),);
     /* go through directory and delete unused files */
@@ -583,7 +629,14 @@ pub async fn housekeeping(context: &Context) {
         );
     }
 
-    info!(context, "Housekeeping done.",);
+    if let Err(e) = context
+        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
+        .await
+    {
+        warn!(context, "Can't set config: {}", e);
+    }
+    info!(context, "Housekeeping done.");
+    Ok(())
 }
 
 #[allow(clippy::indexing_slicing)]
@@ -612,7 +665,7 @@ async fn maybe_add_from_param(
     files_in_use: &mut HashSet<String>,
     query: &str,
     param_id: Param,
-) {
+) -> anyhow::Result<()> {
     context
         .sql
         .query_map(
@@ -630,9 +683,7 @@ async fn maybe_add_from_param(
             },
         )
         .await
-        .unwrap_or_else(|err| {
-            warn!(context, "sql: failed to add_from_param: {}", err);
-        });
+        .context(format!("housekeeping: failed to add_from_param {}", query))
 }
 
 #[allow(clippy::cognitive_complexity)]
@@ -641,7 +692,7 @@ async fn open(
     sql: &Sql,
     dbfile: impl AsRef<Path>,
     readonly: bool,
-) -> crate::error::Result<()> {
+) -> anyhow::Result<()> {
     if sql.is_open().await {
         error!(
             context,
@@ -666,7 +717,10 @@ async fn open(
         .with_flags(open_flags)
         .with_init(|c| {
             c.execute_batch(&format!(
-                "PRAGMA secure_delete=on; PRAGMA busy_timeout = {};",
+                "PRAGMA secure_delete=on;
+                 PRAGMA busy_timeout = {};
+                 PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
+                 ",
                 Duration::from_secs(10).as_millis()
             ))?;
             Ok(())
@@ -693,156 +747,208 @@ async fn open(
             .ok();
 
         let mut exists_before_update = false;
-        let mut dbversion_before_update: i32 = 0;
-        /* Init tables to dbversion=0 */
+        // Init tables to dbversion=68
+        let mut dbversion_before_update: i32 = 68;
         if !sql.table_exists("config").await? {
             info!(
                 context,
                 "First time init: creating tables in {:?}.",
                 dbfile.as_ref(),
             );
-            sql.execute(
-                "CREATE TABLE config (id INTEGER PRIMARY KEY, keyname TEXT, value TEXT);",
-                paramsv![],
-            )
+            sql.with_conn(move |mut conn| {
+                let tx = conn.transaction()?;
+                tx.execute_batch(
+                    r#"
+CREATE TABLE config (id INTEGER PRIMARY KEY, keyname TEXT, value TEXT);
+CREATE INDEX config_index1 ON config (keyname);
+CREATE TABLE contacts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT DEFAULT '',
+    addr TEXT DEFAULT '' COLLATE NOCASE,
+    origin INTEGER DEFAULT 0,
+    blocked INTEGER DEFAULT 0,
+    last_seen INTEGER DEFAULT 0,
+    param TEXT DEFAULT '',
+    authname TEXT DEFAULT '',
+    selfavatar_sent INTEGER DEFAULT 0
+);
+CREATE INDEX contacts_index1 ON contacts (name COLLATE NOCASE);
+CREATE INDEX contacts_index2 ON contacts (addr COLLATE NOCASE);
+INSERT INTO contacts (id,name,origin) VALUES
+(1,'self',262144), (2,'info',262144), (3,'rsvd',262144),
+(4,'rsvd',262144), (5,'device',262144), (6,'rsvd',262144),
+(7,'rsvd',262144), (8,'rsvd',262144), (9,'rsvd',262144);
+
+CREATE TABLE chats (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    type INTEGER DEFAULT 0,
+    name TEXT DEFAULT '',
+    draft_timestamp INTEGER DEFAULT 0,
+    draft_txt TEXT DEFAULT '',
+    blocked INTEGER DEFAULT 0,
+    grpid TEXT DEFAULT '',
+    param TEXT DEFAULT '',
+    archived INTEGER DEFAULT 0,
+    gossiped_timestamp INTEGER DEFAULT 0,
+    locations_send_begin INTEGER DEFAULT 0,
+    locations_send_until INTEGER DEFAULT 0,
+    locations_last_sent INTEGER DEFAULT 0,
+    created_timestamp INTEGER DEFAULT 0,
+    muted_until INTEGER DEFAULT 0,
+    ephemeral_timer INTEGER
+);
+CREATE INDEX chats_index1 ON chats (grpid);
+CREATE INDEX chats_index2 ON chats (archived);
+CREATE INDEX chats_index3 ON chats (locations_send_until);
+INSERT INTO chats (id,type,name) VALUES
+(1,120,'deaddrop'), (2,120,'rsvd'), (3,120,'trash'),
+(4,120,'msgs_in_creation'), (5,120,'starred'), (6,120,'archivedlink'),
+(7,100,'rsvd'), (8,100,'rsvd'), (9,100,'rsvd');
+
+CREATE TABLE chats_contacts (chat_id INTEGER, contact_id INTEGER);
+CREATE INDEX chats_contacts_index1 ON chats_contacts (chat_id);
+CREATE INDEX chats_contacts_index2 ON chats_contacts (contact_id);
+
+CREATE TABLE msgs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    rfc724_mid TEXT DEFAULT '',
+    server_folder TEXT DEFAULT '',
+    server_uid INTEGER DEFAULT 0,
+    chat_id INTEGER DEFAULT 0,
+    from_id INTEGER DEFAULT 0,
+    to_id INTEGER DEFAULT 0,
+    timestamp INTEGER DEFAULT 0,
+    type INTEGER DEFAULT 0,
+    state INTEGER DEFAULT 0,
+    msgrmsg INTEGER DEFAULT 1,
+    bytes INTEGER DEFAULT 0,
+    txt TEXT DEFAULT '',
+    txt_raw TEXT DEFAULT '',
+    param TEXT DEFAULT '',
+    starred INTEGER DEFAULT 0,
+    timestamp_sent INTEGER DEFAULT 0,
+    timestamp_rcvd INTEGER DEFAULT 0,
+    hidden INTEGER DEFAULT 0,
+    mime_headers TEXT,
+    mime_in_reply_to TEXT,
+    mime_references TEXT,
+    move_state INTEGER DEFAULT 1,
+    location_id INTEGER DEFAULT 0,
+    error TEXT DEFAULT '',
+
+-- Timer value in seconds. For incoming messages this
+-- timer starts when message is read, so we want to have
+-- the value stored here until the timer starts.
+    ephemeral_timer INTEGER DEFAULT 0,
+
+-- Timestamp indicating when the message should be
+-- deleted. It is convenient to store it here because UI
+-- needs this value to display how much time is left until
+-- the message is deleted.
+    ephemeral_timestamp INTEGER DEFAULT 0
+);
+
+CREATE INDEX msgs_index1 ON msgs (rfc724_mid);
+CREATE INDEX msgs_index2 ON msgs (chat_id);
+CREATE INDEX msgs_index3 ON msgs (timestamp);
+CREATE INDEX msgs_index4 ON msgs (state);
+CREATE INDEX msgs_index5 ON msgs (starred);
+CREATE INDEX msgs_index6 ON msgs (location_id);
+CREATE INDEX msgs_index7 ON msgs (state, hidden, chat_id);
+INSERT INTO msgs (id,msgrmsg,txt) VALUES
+(1,0,'marker1'), (2,0,'rsvd'), (3,0,'rsvd'),
+(4,0,'rsvd'), (5,0,'rsvd'), (6,0,'rsvd'), (7,0,'rsvd'),
+(8,0,'rsvd'), (9,0,'daymarker');
+
+CREATE TABLE jobs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    added_timestamp INTEGER,
+    desired_timestamp INTEGER DEFAULT 0,
+    action INTEGER,
+    foreign_id INTEGER,
+    param TEXT DEFAULT '',
+    thread INTEGER DEFAULT 0,
+    tries INTEGER DEFAULT 0
+);
+CREATE INDEX jobs_index1 ON jobs (desired_timestamp);
+
+CREATE TABLE leftgrps (
+    id INTEGER PRIMARY KEY,
+    grpid TEXT DEFAULT ''
+);
+CREATE INDEX leftgrps_index1 ON leftgrps (grpid);
+
+CREATE TABLE keypairs (
+    id INTEGER PRIMARY KEY,
+    addr TEXT DEFAULT '' COLLATE NOCASE,
+    is_default INTEGER DEFAULT 0,
+    private_key,
+    public_key,
+    created INTEGER DEFAULT 0
+);
+
+CREATE TABLE acpeerstates (
+    id INTEGER PRIMARY KEY,
+    addr TEXT DEFAULT '' COLLATE NOCASE,
+    last_seen INTEGER DEFAULT 0,
+    last_seen_autocrypt INTEGER DEFAULT 0,
+    public_key,
+    prefer_encrypted INTEGER DEFAULT 0,
+    gossip_timestamp INTEGER DEFAULT 0,
+    gossip_key,
+    public_key_fingerprint TEXT DEFAULT '',
+    gossip_key_fingerprint TEXT DEFAULT '',
+    verified_key,
+    verified_key_fingerprint TEXT DEFAULT ''
+);
+CREATE INDEX acpeerstates_index1 ON acpeerstates (addr);
+CREATE INDEX acpeerstates_index3 ON acpeerstates (public_key_fingerprint);
+CREATE INDEX acpeerstates_index4 ON acpeerstates (gossip_key_fingerprint);
+CREATE INDEX acpeerstates_index5 ON acpeerstates (verified_key_fingerprint);
+
+CREATE TABLE msgs_mdns (
+    msg_id INTEGER,
+    contact_id INTEGER,
+    timestamp_sent INTEGER DEFAULT 0
+);
+CREATE INDEX msgs_mdns_index1 ON msgs_mdns (msg_id);
+
+CREATE TABLE tokens (
+    id INTEGER PRIMARY KEY,
+    namespc INTEGER DEFAULT 0,
+    foreign_id INTEGER DEFAULT 0,
+    token TEXT DEFAULT '',
+    timestamp INTEGER DEFAULT 0
+);
+
+CREATE TABLE locations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    latitude REAL DEFAULT 0.0,
+    longitude REAL DEFAULT 0.0,
+    accuracy REAL DEFAULT 0.0,
+    timestamp INTEGER DEFAULT 0,
+    chat_id INTEGER DEFAULT 0,
+    from_id INTEGER DEFAULT 0,
+    independent INTEGER DEFAULT 0
+);
+CREATE INDEX locations_index1 ON locations (from_id);
+CREATE INDEX locations_index2 ON locations (timestamp);
+
+CREATE TABLE devmsglabels (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    label TEXT,
+    msg_id INTEGER DEFAULT 0
+);
+CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
+"#,
+                )?;
+                tx.commit()?;
+                Ok(())
+            })
             .await?;
-            sql.execute(
-                "CREATE INDEX config_index1 ON config (keyname);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE contacts (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT, \
-                 name TEXT DEFAULT '', \
-                 addr TEXT DEFAULT '' COLLATE NOCASE, \
-                 origin INTEGER DEFAULT 0, \
-                 blocked INTEGER DEFAULT 0, \
-                 last_seen INTEGER DEFAULT 0, \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX contacts_index1 ON contacts (name COLLATE NOCASE);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX contacts_index2 ON contacts (addr COLLATE NOCASE);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "INSERT INTO contacts (id,name,origin) VALUES \
-                 (1,'self',262144), (2,'info',262144), (3,'rsvd',262144), \
-                 (4,'rsvd',262144), (5,'device',262144), (6,'rsvd',262144), \
-                 (7,'rsvd',262144), (8,'rsvd',262144), (9,'rsvd',262144);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE chats (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT,  \
-                 type INTEGER DEFAULT 0, \
-                 name TEXT DEFAULT '', \
-                 draft_timestamp INTEGER DEFAULT 0, \
-                 draft_txt TEXT DEFAULT '', \
-                 blocked INTEGER DEFAULT 0, \
-                 grpid TEXT DEFAULT '', \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute("CREATE INDEX chats_index1 ON chats (grpid);", paramsv![])
+
+            sql.set_raw_config_int(context, "dbversion", dbversion_before_update)
                 .await?;
-            sql.execute(
-                "CREATE TABLE chats_contacts (chat_id INTEGER, contact_id INTEGER);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX chats_contacts_index1 ON chats_contacts (chat_id);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "INSERT INTO chats (id,type,name) VALUES \
-                 (1,120,'deaddrop'), (2,120,'rsvd'), (3,120,'trash'), \
-                 (4,120,'msgs_in_creation'), (5,120,'starred'), (6,120,'archivedlink'), \
-                 (7,100,'rsvd'), (8,100,'rsvd'), (9,100,'rsvd');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE msgs (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT, \
-                 rfc724_mid TEXT DEFAULT '', \
-                 server_folder TEXT DEFAULT '', \
-                 server_uid INTEGER DEFAULT 0, \
-                 chat_id INTEGER DEFAULT 0, \
-                 from_id INTEGER DEFAULT 0, \
-                 to_id INTEGER DEFAULT 0, \
-                 timestamp INTEGER DEFAULT 0, \
-                 type INTEGER DEFAULT 0, \
-                 state INTEGER DEFAULT 0, \
-                 msgrmsg INTEGER DEFAULT 1, \
-                 bytes INTEGER DEFAULT 0, \
-                 txt TEXT DEFAULT '', \
-                 txt_raw TEXT DEFAULT '', \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute("CREATE INDEX msgs_index1 ON msgs (rfc724_mid);", paramsv![])
-                .await?;
-            sql.execute("CREATE INDEX msgs_index2 ON msgs (chat_id);", paramsv![])
-                .await?;
-            sql.execute("CREATE INDEX msgs_index3 ON msgs (timestamp);", paramsv![])
-                .await?;
-            sql.execute("CREATE INDEX msgs_index4 ON msgs (state);", paramsv![])
-                .await?;
-            sql.execute(
-                "INSERT INTO msgs (id,msgrmsg,txt) VALUES \
-                 (1,0,'marker1'), (2,0,'rsvd'), (3,0,'rsvd'), \
-                 (4,0,'rsvd'), (5,0,'rsvd'), (6,0,'rsvd'), (7,0,'rsvd'), \
-                 (8,0,'rsvd'), (9,0,'daymarker');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE jobs (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT, \
-                 added_timestamp INTEGER, \
-                 desired_timestamp INTEGER DEFAULT 0, \
-                 action INTEGER, \
-                 foreign_id INTEGER, \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX jobs_index1 ON jobs (desired_timestamp);",
-                paramsv![],
-            )
-            .await?;
-            if !sql.table_exists("config").await?
-                || !sql.table_exists("contacts").await?
-                || !sql.table_exists("chats").await?
-                || !sql.table_exists("chats_contacts").await?
-                || !sql.table_exists("msgs").await?
-                || !sql.table_exists("jobs").await?
-            {
-                error!(
-                    context,
-                    "Cannot create tables in new database \"{:?}\".",
-                    dbfile.as_ref(),
-                );
-                // cannot create the tables - maybe we cannot write?
-                return Err(Error::SqlFailedToOpen.into());
-            } else {
-                sql.set_raw_config_int(context, "dbversion", 0).await?;
-            }
         } else {
             exists_before_update = true;
             dbversion_before_update = sql
@@ -858,7 +964,8 @@ async fn open(
 
         let mut dbversion = dbversion_before_update;
         let mut recalc_fingerprints = false;
-        let mut update_icons = false;
+        let mut update_icons = !exists_before_update;
+        let mut disable_server_delete = false;
 
         if dbversion < 1 {
             info!(context, "[migration] v1");
@@ -872,7 +979,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 1;
             sql.set_raw_config_int(context, "dbversion", 1).await?;
         }
         if dbversion < 2 {
@@ -882,7 +988,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 2;
             sql.set_raw_config_int(context, "dbversion", 2).await?;
         }
         if dbversion < 7 {
@@ -898,7 +1003,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 7;
             sql.set_raw_config_int(context, "dbversion", 7).await?;
         }
         if dbversion < 10 {
@@ -919,7 +1023,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 10;
             sql.set_raw_config_int(context, "dbversion", 10).await?;
         }
         if dbversion < 12 {
@@ -934,7 +1037,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 12;
             sql.set_raw_config_int(context, "dbversion", 12).await?;
         }
         if dbversion < 17 {
@@ -946,6 +1048,8 @@ async fn open(
             .await?;
             sql.execute("CREATE INDEX chats_index2 ON chats (archived);", paramsv![])
                 .await?;
+            // 'starred' column is not used currently
+            // (dropping is not easily doable and stop adding it will make reusing it complicated)
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN starred INTEGER DEFAULT 0;",
                 paramsv![],
@@ -953,7 +1057,6 @@ async fn open(
             .await?;
             sql.execute("CREATE INDEX msgs_index5 ON msgs (starred);", paramsv![])
                 .await?;
-            dbversion = 17;
             sql.set_raw_config_int(context, "dbversion", 17).await?;
         }
         if dbversion < 18 {
@@ -968,7 +1071,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 18;
             sql.set_raw_config_int(context, "dbversion", 18).await?;
         }
         if dbversion < 27 {
@@ -992,7 +1094,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 27;
             sql.set_raw_config_int(context, "dbversion", 27).await?;
         }
         if dbversion < 34 {
@@ -1028,7 +1129,6 @@ async fn open(
             )
             .await?;
             recalc_fingerprints = true;
-            dbversion = 34;
             sql.set_raw_config_int(context, "dbversion", 34).await?;
         }
         if dbversion < 39 {
@@ -1052,7 +1152,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 39;
             sql.set_raw_config_int(context, "dbversion", 39).await?;
         }
         if dbversion < 40 {
@@ -1062,14 +1161,12 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 40;
             sql.set_raw_config_int(context, "dbversion", 40).await?;
         }
         if dbversion < 44 {
             info!(context, "[migration] v44");
             sql.execute("ALTER TABLE msgs ADD COLUMN mime_headers TEXT;", paramsv![])
                 .await?;
-            dbversion = 44;
             sql.set_raw_config_int(context, "dbversion", 44).await?;
         }
         if dbversion < 46 {
@@ -1094,7 +1191,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 47;
             sql.set_raw_config_int(context, "dbversion", 47).await?;
         }
         if dbversion < 48 {
@@ -1105,8 +1201,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-
-            dbversion = 48;
             sql.set_raw_config_int(context, "dbversion", 48).await?;
         }
         if dbversion < 49 {
@@ -1116,7 +1210,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 49;
             sql.set_raw_config_int(context, "dbversion", 49).await?;
         }
         if dbversion < 50 {
@@ -1128,7 +1221,6 @@ async fn open(
                 sql.set_raw_config_int(context, "show_emails", ShowEmails::All as i32)
                     .await?;
             }
-            dbversion = 50;
             sql.set_raw_config_int(context, "dbversion", 50).await?;
         }
         if dbversion < 53 {
@@ -1169,7 +1261,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 53;
             sql.set_raw_config_int(context, "dbversion", 53).await?;
         }
         if dbversion < 54 {
@@ -1184,7 +1275,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 54;
             sql.set_raw_config_int(context, "dbversion", 54).await?;
         }
         if dbversion < 55 {
@@ -1264,18 +1354,11 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            // Timer value in seconds. For incoming messages this
-            // timer starts when message is read, so we want to have
-            // the value stored here until the timer starts.
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN ephemeral_timer INTEGER DEFAULT 0",
                 paramsv![],
             )
             .await?;
-            // Timestamp indicating when the message should be
-            // deleted. It is convenient to store it here because UI
-            // needs this value to display how much time is left until
-            // the message is deleted.
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0",
                 paramsv![],
@@ -1317,7 +1400,7 @@ async fn open(
         }
         if dbversion < 68 {
             info!(context, "[migration] v68");
-            // the index is used to speed up get_fresh_msg_cnt(), see comment there for more details
+            // the index is used to speed up get_fresh_msg_cnt() (see comment there for more details) and marknoticed_chat()
             sql.execute(
                 "CREATE INDEX IF NOT EXISTS msgs_index7 ON msgs (state, hidden, chat_id);",
                 paramsv![],
@@ -1325,6 +1408,111 @@ async fn open(
             .await?;
             sql.set_raw_config_int(context, "dbversion", 68).await?;
         }
+        if dbversion < 69 {
+            info!(context, "[migration] v69");
+            sql.execute(
+                "ALTER TABLE chats ADD COLUMN protected INTEGER DEFAULT 0;",
+                paramsv![],
+            )
+            .await?;
+            sql.execute(
+                "UPDATE chats SET protected=1, type=120 WHERE type=130;", // 120=group, 130=old verified group
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 69).await?;
+        }
+        if dbversion < 71 {
+            info!(context, "[migration] v71");
+            if let Some(addr) = context.get_config(Config::ConfiguredAddr).await {
+                if let Ok(domain) = addr.parse::<EmailAddress>().map(|email| email.domain) {
+                    context
+                        .set_config(
+                            Config::ConfiguredProvider,
+                            get_provider_by_domain(&domain).map(|provider| provider.id),
+                        )
+                        .await?;
+                } else {
+                    warn!(context, "Can't parse configured address: {:?}", addr);
+                }
+            }
+
+            sql.set_raw_config_int(context, "dbversion", 71).await?;
+        }
+        if dbversion < 72 {
+            info!(context, "[migration] v72");
+            if !sql.col_exists("msgs", "mime_modified").await? {
+                sql.execute(
+                    "ALTER TABLE msgs ADD COLUMN mime_modified INTEGER DEFAULT 0;",
+                    paramsv![],
+                )
+                .await?;
+            }
+            sql.set_raw_config_int(context, "dbversion", 72).await?;
+        }
+        if dbversion < 73 {
+            use Config::*;
+            info!(context, "[migration] v73");
+            sql.execute(
+                "CREATE TABLE imap_sync (folder TEXT PRIMARY KEY, uidvalidity INTEGER DEFAULT 0, uid_next INTEGER DEFAULT 0);",
+                paramsv![],
+            )
+            .await?;
+            for c in &[
+                ConfiguredInboxFolder,
+                ConfiguredSentboxFolder,
+                ConfiguredMvboxFolder,
+            ] {
+                if let Some(folder) = context.get_config(*c).await {
+                    let (uid_validity, last_seen_uid) =
+                        imap::get_config_last_seen_uid(context, &folder).await;
+                    if last_seen_uid > 0 {
+                        imap::set_uid_next(context, &folder, last_seen_uid + 1).await?;
+                        imap::set_uidvalidity(context, &folder, uid_validity).await?;
+                    }
+                }
+            }
+            if exists_before_update {
+                disable_server_delete = true;
+
+                // Don't disable server delete if it was on by default (Nauta):
+                if let Some(provider) = context.get_configured_provider().await {
+                    if let Some(defaults) = &provider.config_defaults {
+                        if defaults.iter().any(|d| d.key == Config::DeleteServerAfter) {
+                            disable_server_delete = false;
+                        }
+                    }
+                }
+            }
+            sql.set_raw_config_int(context, "dbversion", 73).await?;
+        }
+        if dbversion < 74 {
+            info!(context, "[migration] v74");
+            sql.execute(
+                "UPDATE contacts SET name='' WHERE name=authname",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 74).await?;
+        }
+        if dbversion < 75 {
+            info!(context, "[migration] v75");
+            sql.execute(
+                "ALTER TABLE contacts ADD COLUMN status TEXT DEFAULT '';",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 75).await?;
+        }
+        if dbversion < 76 {
+            info!(context, "[migration] v76");
+            sql.execute(
+                "ALTER TABLE msgs ADD COLUMN subject TEXT DEFAULT '';",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 76).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)
@@ -1355,6 +1543,16 @@ async fn open(
             update_saved_messages_icon(context).await?;
             update_device_icon(context).await?;
         }
+        if disable_server_delete {
+            // We now always watch all folders and delete messages there if delete_server is enabled.
+            // So, for people who have delete_server enabled, disable it and add a hint to the devicechat:
+            if context.get_config_delete_server_after().await.is_some() {
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text = Some(stock_str::delete_server_turned_off(context).await);
+                add_device_msg(context, None, Some(&mut msg)).await?;
+                context.set_config(DeleteServerAfter, Some("0")).await?;
+            }
+        }
     }
 
     info!(context, "Opened {:?}.", dbfile.as_ref(),);
@@ -1379,7 +1577,10 @@ async fn prune_tombstones(context: &Context) -> Result<()> {
 
 #[cfg(test)]
 mod test {
+    use async_std::fs::File;
+
     use super::*;
+    use crate::{test_utils::TestContext, Event, EventType};
 
     #[test]
     fn test_maybe_add_file() {
@@ -1406,4 +1607,59 @@ mod test {
         assert!(!is_file_in_use(&files, Some(".txt"), "hello"));
         assert!(is_file_in_use(&files, Some("-suffix"), "world.txt-suffix"));
     }
+
+    #[async_std::test]
+    async fn test_table_exists() {
+        let t = TestContext::new().await;
+        assert!(t.ctx.sql.table_exists("msgs").await.unwrap());
+        assert!(!t.ctx.sql.table_exists("foobar").await.unwrap());
+    }
+
+    #[async_std::test]
+    async fn test_col_exists() {
+        let t = TestContext::new().await;
+        assert!(t.ctx.sql.col_exists("msgs", "mime_modified").await.unwrap());
+        assert!(!t.ctx.sql.col_exists("msgs", "foobar").await.unwrap());
+        assert!(!t.ctx.sql.col_exists("foobar", "foobar").await.unwrap());
+    }
+
+    #[async_std::test]
+    async fn test_housekeeping_db_closed() {
+        let t = TestContext::new().await;
+
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+
+        t.add_event_sink(move |event: Event| async move {
+            match event.typ {
+                EventType::Info(s) => assert!(
+                    !s.contains("Keeping new unreferenced file"),
+                    "File {} was almost deleted, only reason it was kept is that it was created recently (as the tests don't run for a long time)",
+                    s
+                ),
+                EventType::Error(s) => panic!(s),
+                _ => {}
+            }
+        })
+        .await;
+
+        let a = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_bytes, &async_std::fs::read(&a).await.unwrap()[..]);
+
+        t.sql.close().await;
+        housekeeping(&t).await.unwrap_err(); // housekeeping should fail as the db is closed
+        t.sql.open(&t, &t.get_dbfile(), false).await.unwrap();
+
+        let a = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_bytes, &async_std::fs::read(&a).await.unwrap()[..]);
+    }
 }

src/stock.rs

@@ -1,631 +0,0 @@
-//! Module to work with translatable stock strings
-
-use std::borrow::Cow;
-
-use strum::EnumProperty;
-use strum_macros::EnumProperty;
-
-use crate::blob::BlobObject;
-use crate::chat;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
-use crate::contact::*;
-use crate::context::Context;
-use crate::error::{bail, Error};
-use crate::message::Message;
-use crate::param::Param;
-use crate::stock::StockMessage::{DeviceMessagesHint, WelcomeMessage};
-
-/// Stock strings
-///
-/// These identify the string to return in [Context.stock_str].  The
-/// numbers must stay in sync with `deltachat.h` `DC_STR_*` constants.
-///
-/// See the `stock_*` methods on [Context] to use these.
-///
-/// [Context]: crate::context::Context
-#[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, EnumProperty)]
-#[repr(u32)]
-pub enum StockMessage {
-    #[strum(props(fallback = "No messages."))]
-    NoMessages = 1,
-
-    #[strum(props(fallback = "Me"))]
-    SelfMsg = 2,
-
-    #[strum(props(fallback = "Draft"))]
-    Draft = 3,
-
-    #[strum(props(fallback = "Voice message"))]
-    VoiceMessage = 7,
-
-    #[strum(props(fallback = "Contact requests"))]
-    DeadDrop = 8,
-
-    #[strum(props(fallback = "Image"))]
-    Image = 9,
-
-    #[strum(props(fallback = "Video"))]
-    Video = 10,
-
-    #[strum(props(fallback = "Audio"))]
-    Audio = 11,
-
-    #[strum(props(fallback = "File"))]
-    File = 12,
-
-    #[strum(props(fallback = "Sent with my Delta Chat Messenger: https://delta.chat"))]
-    StatusLine = 13,
-
-    #[strum(props(fallback = "Hello, I\'ve just created the group \"%1$s\" for us."))]
-    NewGroupDraft = 14,
-
-    #[strum(props(fallback = "Group name changed from \"%1$s\" to \"%2$s\"."))]
-    MsgGrpName = 15,
-
-    #[strum(props(fallback = "Group image changed."))]
-    MsgGrpImgChanged = 16,
-
-    #[strum(props(fallback = "Member %1$s added."))]
-    MsgAddMember = 17,
-
-    #[strum(props(fallback = "Member %1$s removed."))]
-    MsgDelMember = 18,
-
-    #[strum(props(fallback = "Group left."))]
-    MsgGroupLeft = 19,
-
-    #[strum(props(fallback = "GIF"))]
-    Gif = 23,
-
-    #[strum(props(fallback = "Encrypted message"))]
-    EncryptedMsg = 24,
-
-    #[strum(props(fallback = "End-to-end encryption available."))]
-    E2eAvailable = 25,
-
-    #[strum(props(fallback = "Transport-encryption."))]
-    EncrTransp = 27,
-
-    #[strum(props(fallback = "No encryption."))]
-    EncrNone = 28,
-
-    #[strum(props(fallback = "This message was encrypted for another setup."))]
-    CantDecryptMsgBody = 29,
-
-    #[strum(props(fallback = "Fingerprints"))]
-    FingerPrints = 30,
-
-    #[strum(props(fallback = "Return receipt"))]
-    ReadRcpt = 31,
-
-    #[strum(props(fallback = "This is a return receipt for the message \"%1$s\"."))]
-    ReadRcptMailBody = 32,
-
-    #[strum(props(fallback = "Group image deleted."))]
-    MsgGrpImgDeleted = 33,
-
-    #[strum(props(fallback = "End-to-end encryption preferred."))]
-    E2ePreferred = 34,
-
-    #[strum(props(fallback = "%1$s verified."))]
-    ContactVerified = 35,
-
-    #[strum(props(fallback = "Cannot verify %1$s"))]
-    ContactNotVerified = 36,
-
-    #[strum(props(fallback = "Changed setup for %1$s"))]
-    ContactSetupChanged = 37,
-
-    #[strum(props(fallback = "Archived chats"))]
-    ArchivedChats = 40,
-
-    #[strum(props(fallback = "Starred messages"))]
-    StarredMsgs = 41,
-
-    #[strum(props(fallback = "Autocrypt Setup Message"))]
-    AcSetupMsgSubject = 42,
-
-    #[strum(props(
-        fallback = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\nTo decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device."
-    ))]
-    AcSetupMsgBody = 43,
-
-    #[strum(props(
-        fallback = "Cannot login as \"%1$s\". Please check if the email address and the password are correct."
-    ))]
-    CannotLogin = 60,
-
-    #[strum(props(fallback = "Could not connect to %1$s: %2$s"))]
-    ServerResponse = 61,
-
-    #[strum(props(fallback = "%1$s by %2$s."))]
-    MsgActionByUser = 62,
-
-    #[strum(props(fallback = "%1$s by me."))]
-    MsgActionByMe = 63,
-
-    #[strum(props(fallback = "Location streaming enabled."))]
-    MsgLocationEnabled = 64,
-
-    #[strum(props(fallback = "Location streaming disabled."))]
-    MsgLocationDisabled = 65,
-
-    #[strum(props(fallback = "Location"))]
-    Location = 66,
-
-    #[strum(props(fallback = "Sticker"))]
-    Sticker = 67,
-
-    #[strum(props(fallback = "Device messages"))]
-    DeviceMessages = 68,
-
-    #[strum(props(fallback = "Saved messages"))]
-    SavedMessages = 69,
-
-    #[strum(props(
-        fallback = "Messages in this chat are generated locally by your Delta Chat app. \
-                    Its makers use it to inform about app updates and problems during usage."
-    ))]
-    DeviceMessagesHint = 70,
-
-    #[strum(props(fallback = "Welcome to Delta Chat! – \
-                    Delta Chat looks and feels like other popular messenger apps, \
-                    but does not involve centralized control, \
-                    tracking or selling you, friends, colleagues or family out to large organizations.\n\n\
-                    Technically, Delta Chat is an email application with a modern chat interface. \
-                    Email in a new dress if you will 👻\n\n\
-                    Use Delta Chat with anyone out of billions of people: just use their e-mail address. \
-                    Recipients don't need to install Delta Chat, visit websites or sign up anywhere - \
-                    however, of course, if they like, you may point them to 👉 https://get.delta.chat"))]
-    WelcomeMessage = 71,
-
-    #[strum(props(fallback = "Unknown sender for this chat. See 'info' for more details."))]
-    UnknownSenderForChat = 72,
-
-    #[strum(props(fallback = "Message from %1$s"))]
-    SubjectForNewContact = 73,
-
-    #[strum(props(fallback = "Failed to send message to %1$s."))]
-    FailedSendingTo = 74,
-
-    #[strum(props(fallback = "Message deletion timer is disabled."))]
-    MsgEphemeralTimerDisabled = 75,
-
-    // A fallback message for unknown timer values.
-    // "s" stands for "second" SI unit here.
-    #[strum(props(fallback = "Message deletion timer is set to %1$s s."))]
-    MsgEphemeralTimerEnabled = 76,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 minute."))]
-    MsgEphemeralTimerMinute = 77,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 hour."))]
-    MsgEphemeralTimerHour = 78,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 day."))]
-    MsgEphemeralTimerDay = 79,
-
-    #[strum(props(fallback = "Message deletion timer is set to 1 week."))]
-    MsgEphemeralTimerWeek = 80,
-
-    #[strum(props(fallback = "Message deletion timer is set to 4 weeks."))]
-    MsgEphemeralTimerFourWeeks = 81,
-
-    #[strum(props(fallback = "Video chat invitation"))]
-    VideochatInvitation = 82,
-
-    #[strum(props(fallback = "You are invited to a video chat, click %1$s to join."))]
-    VideochatInviteMsgBody = 83,
-
-    #[strum(props(fallback = "Configuration failed. Error: “%1$s”"))]
-    ConfigurationFailed = 84,
-}
-
-/*
-"
-*/
-
-impl StockMessage {
-    /// Default untranslated strings for stock messages.
-    ///
-    /// These could be used in logging calls, so no logging here.
-    fn fallback(self) -> &'static str {
-        self.get_str("fallback").unwrap_or_default()
-    }
-}
-
-impl Context {
-    /// Set the stock string for the [StockMessage].
-    ///
-    pub async fn set_stock_translation(
-        &self,
-        id: StockMessage,
-        stockstring: String,
-    ) -> Result<(), Error> {
-        if stockstring.contains("%1") && !id.fallback().contains("%1") {
-            bail!(
-                "translation {} contains invalid %1 placeholder, default is {}",
-                stockstring,
-                id.fallback()
-            );
-        }
-        if stockstring.contains("%2") && !id.fallback().contains("%2") {
-            bail!(
-                "translation {} contains invalid %2 placeholder, default is {}",
-                stockstring,
-                id.fallback()
-            );
-        }
-        self.translated_stockstrings
-            .write()
-            .await
-            .insert(id as usize, stockstring);
-        Ok(())
-    }
-
-    /// Return the stock string for the [StockMessage].
-    ///
-    /// Return a translation (if it was set with set_stock_translation before)
-    /// or a default (English) string.
-    pub async fn stock_str(&self, id: StockMessage) -> Cow<'_, str> {
-        match self
-            .translated_stockstrings
-            .read()
-            .await
-            .get(&(id as usize))
-        {
-            Some(ref x) => Cow::Owned((*x).to_string()),
-            None => Cow::Borrowed(id.fallback()),
-        }
-    }
-
-    /// Return stock string, replacing placeholders with provided string.
-    ///
-    /// This replaces both the *first* `%1$s`, `%1$d` and `%1$@`
-    /// placeholders with the provided string.
-    /// (the `%1$@` variant is used on iOS, the other are used on Android and Desktop)
-    pub async fn stock_string_repl_str(&self, id: StockMessage, insert: impl AsRef<str>) -> String {
-        self.stock_str(id)
-            .await
-            .replacen("%1$s", insert.as_ref(), 1)
-            .replacen("%1$d", insert.as_ref(), 1)
-            .replacen("%1$@", insert.as_ref(), 1)
-    }
-
-    /// Return stock string, replacing placeholders with provided int.
-    ///
-    /// Like [Context::stock_string_repl_str] but substitute the placeholders
-    /// with an integer.
-    pub async fn stock_string_repl_int(&self, id: StockMessage, insert: i32) -> String {
-        self.stock_string_repl_str(id, format!("{}", insert).as_str())
-            .await
-    }
-
-    /// Return stock string, replacing 2 placeholders with provided string.
-    ///
-    /// This replaces both the *first* `%1$s`, `%1$d` and `%1$@`
-    /// placeholders with the string in `insert` and does the same for
-    /// `%2$s`, `%2$d` and `%2$@` for `insert2`.
-    /// (the `%1$@` variant is used on iOS, the other are used on Android and Desktop)
-    pub async fn stock_string_repl_str2(
-        &self,
-        id: StockMessage,
-        insert: impl AsRef<str>,
-        insert2: impl AsRef<str>,
-    ) -> String {
-        self.stock_str(id)
-            .await
-            .replacen("%1$s", insert.as_ref(), 1)
-            .replacen("%1$d", insert.as_ref(), 1)
-            .replacen("%1$@", insert.as_ref(), 1)
-            .replacen("%2$s", insert2.as_ref(), 1)
-            .replacen("%2$d", insert2.as_ref(), 1)
-            .replacen("%2$@", insert2.as_ref(), 1)
-    }
-
-    /// Return some kind of stock message
-    ///
-    /// If the `id` is [StockMessage::MsgAddMember] or
-    /// [StockMessage::MsgDelMember] then `param1` is considered to be the
-    /// contact address and will be replaced by that contact's display
-    /// name.
-    ///
-    /// If `from_id` is not `0`, any trailing dot is removed from the
-    /// first stock string created so far.  If the `from_id` contact is
-    /// the user itself, i.e. `DC_CONTACT_ID_SELF` the string is used
-    /// itself as param to the [StockMessage::MsgActionByMe] stock string
-    /// resulting in a string like "Member Alice added by me." (for
-    /// [StockMessage::MsgAddMember] as `id`).  If the `from_id` contact
-    /// is any other user than the contact's display name is looked up and
-    /// used as the second parameter to [StockMessage::MsgActionByUser] with
-    /// again the original stock string being used as the first parameter,
-    /// resulting in a string like "Member Alice added by Bob.".
-    pub async fn stock_system_msg(
-        &self,
-        id: StockMessage,
-        param1: impl AsRef<str>,
-        param2: impl AsRef<str>,
-        from_id: u32,
-    ) -> String {
-        let insert1 = if id == StockMessage::MsgAddMember || id == StockMessage::MsgDelMember {
-            let contact_id =
-                Contact::lookup_id_by_addr(self, param1.as_ref(), Origin::Unknown).await;
-            if contact_id != 0 {
-                Contact::get_by_id(self, contact_id)
-                    .await
-                    .map(|contact| contact.get_name_n_addr())
-                    .unwrap_or_default()
-            } else {
-                param1.as_ref().to_string()
-            }
-        } else {
-            param1.as_ref().to_string()
-        };
-
-        let action = self
-            .stock_string_repl_str2(id, insert1, param2.as_ref().to_string())
-            .await;
-        let action1 = action.trim_end_matches('.');
-        match from_id {
-            0 => action,
-            DC_CONTACT_ID_SELF => {
-                self.stock_string_repl_str(StockMessage::MsgActionByMe, action1)
-                    .await
-            }
-            _ => {
-                let displayname = Contact::get_by_id(self, from_id)
-                    .await
-                    .map(|contact| contact.get_name_n_addr())
-                    .unwrap_or_default();
-
-                self.stock_string_repl_str2(StockMessage::MsgActionByUser, action1, &displayname)
-                    .await
-            }
-        }
-    }
-
-    pub async fn update_device_chats(&self) -> Result<(), Error> {
-        // check for the LAST added device message - if it is present, we can skip message creation.
-        // this is worthwhile as this function is typically called
-        // by the ui on every probram start or even on every opening of the chatlist.
-        if chat::was_device_msg_ever_added(&self, "core-welcome").await? {
-            return Ok(());
-        }
-
-        // create saved-messages chat;
-        // we do this only once, if the user has deleted the chat, he can recreate it manually.
-        if !self.sql.get_raw_config_bool(&self, "self-chat-added").await {
-            self.sql
-                .set_raw_config_bool(&self, "self-chat-added", true)
-                .await?;
-            chat::create_by_contact_id(&self, DC_CONTACT_ID_SELF).await?;
-        }
-
-        // add welcome-messages. by the label, this is done only once,
-        // if the user has deleted the message or the chat, it is not added again.
-        let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(self.stock_str(DeviceMessagesHint).await.to_string());
-        chat::add_device_msg(&self, Some("core-about-device-chat"), Some(&mut msg)).await?;
-
-        let image = include_bytes!("../assets/welcome-image.jpg");
-        let blob = BlobObject::create(&self, "welcome-image.jpg".to_string(), image).await?;
-        let mut msg = Message::new(Viewtype::Image);
-        msg.param.set(Param::File, blob.as_name());
-        chat::add_device_msg(&self, Some("core-welcome-image"), Some(&mut msg)).await?;
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(self.stock_str(WelcomeMessage).await.to_string());
-        chat::add_device_msg(&self, Some("core-welcome"), Some(&mut msg)).await?;
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::test_utils::*;
-
-    use crate::constants::DC_CONTACT_ID_SELF;
-
-    use crate::chatlist::Chatlist;
-    use num_traits::ToPrimitive;
-
-    #[test]
-    fn test_enum_mapping() {
-        assert_eq!(StockMessage::NoMessages.to_usize().unwrap(), 1);
-        assert_eq!(StockMessage::SelfMsg.to_usize().unwrap(), 2);
-    }
-
-    #[test]
-    fn test_fallback() {
-        assert_eq!(StockMessage::NoMessages.fallback(), "No messages.");
-    }
-
-    #[async_std::test]
-    async fn test_set_stock_translation() {
-        let t = TestContext::new().await;
-        t.ctx
-            .set_stock_translation(StockMessage::NoMessages, "xyz".to_string())
-            .await
-            .unwrap();
-        assert_eq!(t.ctx.stock_str(StockMessage::NoMessages).await, "xyz")
-    }
-
-    #[async_std::test]
-    async fn test_set_stock_translation_wrong_replacements() {
-        let t = TestContext::new().await;
-        assert!(t
-            .ctx
-            .set_stock_translation(StockMessage::NoMessages, "xyz %1$s ".to_string())
-            .await
-            .is_err());
-        assert!(t
-            .ctx
-            .set_stock_translation(StockMessage::NoMessages, "xyz %2$s ".to_string())
-            .await
-            .is_err());
-    }
-
-    #[async_std::test]
-    async fn test_stock_str() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx.stock_str(StockMessage::NoMessages).await,
-            "No messages."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_string_repl_str() {
-        let t = TestContext::new().await;
-        // uses %1$s substitution
-        assert_eq!(
-            t.ctx
-                .stock_string_repl_str(StockMessage::MsgAddMember, "Foo")
-                .await,
-            "Member Foo added."
-        );
-        // We have no string using %1$d to test...
-    }
-
-    #[async_std::test]
-    async fn test_stock_string_repl_int() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_string_repl_int(StockMessage::MsgAddMember, 42)
-                .await,
-            "Member 42 added."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_string_repl_str2() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_string_repl_str2(StockMessage::ServerResponse, "foo", "bar")
-                .await,
-            "Could not connect to foo: bar"
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_simple() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0)
-                .await,
-            "Location streaming enabled."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_add_member_by_me() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgAddMember,
-                    "alice@example.com",
-                    "",
-                    DC_CONTACT_ID_SELF
-                )
-                .await,
-            "Member alice@example.com added by me."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_add_member_by_me_with_displayname() {
-        let t = TestContext::new().await;
-        Contact::create(&t.ctx, "Alice", "alice@example.com")
-            .await
-            .expect("failed to create contact");
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgAddMember,
-                    "alice@example.com",
-                    "",
-                    DC_CONTACT_ID_SELF
-                )
-                .await,
-            "Member Alice (alice@example.com) added by me."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_add_member_by_other_with_displayname() {
-        let t = TestContext::new().await;
-        let contact_id = {
-            Contact::create(&t.ctx, "Alice", "alice@example.com")
-                .await
-                .expect("Failed to create contact Alice");
-            Contact::create(&t.ctx, "Bob", "bob@example.com")
-                .await
-                .expect("failed to create bob")
-        };
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgAddMember,
-                    "alice@example.com",
-                    "",
-                    contact_id,
-                )
-                .await,
-            "Member Alice (alice@example.com) added by Bob (bob@example.com)."
-        );
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_grp_name() {
-        let t = TestContext::new().await;
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(
-                    StockMessage::MsgGrpName,
-                    "Some chat",
-                    "Other chat",
-                    DC_CONTACT_ID_SELF
-                )
-                .await,
-            "Group name changed from \"Some chat\" to \"Other chat\" by me."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_stock_system_msg_grp_name_other() {
-        let t = TestContext::new().await;
-        let id = Contact::create(&t.ctx, "Alice", "alice@example.com")
-            .await
-            .expect("failed to create contact");
-
-        assert_eq!(
-            t.ctx
-                .stock_system_msg(StockMessage::MsgGrpName, "Some chat", "Other chat", id)
-                .await,
-            "Group name changed from \"Some chat\" to \"Other chat\" by Alice (alice@example.com)."
-        )
-    }
-
-    #[async_std::test]
-    async fn test_update_device_chats() {
-        let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.ok();
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 2);
-
-        chats.get_chat_id(0).delete(&t.ctx).await.ok();
-        chats.get_chat_id(1).delete(&t.ctx).await.ok();
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 0);
-
-        // a subsequent call to update_device_chats() must not re-add manally deleted messages or chats
-        t.ctx.update_device_chats().await.ok();
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 0);
-    }
-}