bump version to 1.49

It is already done for voice messages and makes saving attachments to
one folder easier.

.circleci/config.yml

@@ -138,6 +138,12 @@ jobs:
             - py-docs
             - wheelhouse
 
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+
   remote_tests_python:
     machine: true
     steps:
@@ -172,14 +178,17 @@ workflows:
     jobs:
       # - cargo_fetch
 
+      - remote_tests_rust:
+          filters:
+            tags:
+              only: /.*/
+
       - remote_tests_python:
           filters:
             tags:
               only: /.*/
 
       - remote_python_packaging:
-          requires: 
-             - remote_tests_python
           filters:
             branches:
               only: master

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.43.1
+          toolchain: 1.45.0
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -32,21 +32,36 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: 1.43.1
+            toolchain: 1.45.0
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          args: --workspace --tests --examples
 
 
   build_and_test:
     name: Build and test
     runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [nightly, 1.43.1]
+        # macOS disabled due to random failures related to caching
+        #os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest]
+        rust: [1.45.0]
+        experimental: [false]
+#        include:
+#          - os: ubuntu-latest
+#            rust: nightly
+#            experimental: true
+#          - os: windows-latest
+#            rust: nightly
+#            experimental: true
+#          - os: macOS-latest
+#            rust: nightly
+#            experimental: true
 
     steps:
     - uses: actions/checkout@master
@@ -79,11 +94,10 @@ jobs:
       uses: actions-rs/cargo@v1
       with:
         command:  check
-        args: --workspace --all --bins --examples --tests
+        args: --all --bins --examples --tests --features repl
 
     - name: tests
       uses: actions-rs/cargo@v1
       with:
         command: test
-        args: --workspace
-          
+        args: --all

ASYNC-API-TODO.txt

@@ -1,54 +0,0 @@
-
-Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
-
-- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
-
-APIs:
-    dc_context_new                 # opens the database
-    dc_open                             # FFI only
-      -> drop it and move parameters to dc_context_new()
-
-    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
-    dc_imex NEVER goes through the job system
-    dc_imex import_backup needs to ensure dc_stop_jobs()
-
-    dc_start_io                # start smtp/imap and job handling subsystems
-    dc_stop_io                 # stop smtp/imap and job handling subsystems
-    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
-
-    dc_close                            # FFI only
-      -> can be dropped
-    dc_context_unref
-
-    for ios share-extension:
-    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
-        0: message was sent
-        1: message failed to go out, is queued as a job to be retried later
-        2: message permanently failed?
-
-EVENT handling:
-    start a callback thread and call get_next_event() which is BLOCKING
-    it's fine to start this callback thread later, it will see all events.
-    Note that the core infinitely fills the internal queue if you never drain it.
-
-    FFI-get_next_event() returns NULL if the context is unrefed already?
-
-   sidenote: how python's callback thread does it currently:
-         CB-thread runs this while loop:
-         while not QUITFLAG:
-             ev = context.get_next_event( )
-             ...
-         So in order to shutdown properly one has to set QUITFLAG
-         before calling dc_stop_jobs() and dc_context_unref
-
-    event API:
-        get_data1_int
-        get_data2_int
-        get_data3_str
-
-
-- userdata likely only used for the callbacks, likely can be dropped, needs verification
-
-
-- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
-

CHANGELOG.md

@@ -1,5 +1,287 @@
 # Changelog 
 
+## 1.49.0
+
+- add timestamps to image and video filenames #2068
+
+- forbid quoting messages from another context #2069
+
+- fix: preserve quotes in messages with attachments #2070
+
+
+## 1.48.0
+
+- `fetch_existing` renamed to `fetch_existing_msgs` and disabled by default
+  #2035 #2042
+
+- skip fetch existing messages/contacts if config-option `bot` set #2017
+
+- always log why a message is sorted to trash #2045
+
+- display a quote if top posting is detected #2047
+
+- add ephemeral task cancellation to `dc_stop_io()`;
+  before, there was no way to quickly terminate pending ephemeral tasks #2051
+
+- when saved-messages chat is deleted,
+  a device-message about recreation is added #2050
+
+- use `max_smtp_rcpt_to` from provider-db,
+  sending messages to many recipients in configurable chunks #2056
+
+- fix handling of empty autoconfigure files #2027
+
+- fix adding saved messages to wrong chats on multi-device #2034 #2039
+
+- fix hang on android4.4 and other systems
+  by adding a workaround to executer-blocking-handling bug #2040
+
+- fix secret key export/import roundtrip #2048
+
+- fix mistakenly unarchived chats #2057
+
+- fix outdated-reminder test that fails only 7 days a year,
+  including halloween :) #2059
+
+- improve python bindings #2021 #2036 #2038
+
+- update provider-database #2037
+
+
+## 1.47.0
+
+- breaking change: `dc_update_device_chats()` removed;
+  this is now done automatically during configure
+  unless the new config-option `bot` is set #1957
+
+- breaking change: split `DC_EVENT_MSGS_NOTICED` off `DC_EVENT_MSGS_CHANGED`
+  and remove `dc_marknoticed_all_chats()` #1942 #1981
+
+- breaking change: remove unused starring options #1965
+
+- breaking change: `DC_CHAT_TYPE_VERIFIED_GROUP` replaced by
+  `dc_chat_is_protected()`; also single-chats may be protected now, this may
+  happen over the wire even if the UI do not offer an option for that #1968
+
+- breaking change: split quotes off message text,
+  UIs should use at least `dc_msg_get_quoted_text()` to show quotes now #1975
+
+- new api for quote handling: `dc_msg_set_quote()`, `dc_msg_get_quoted_text()`,
+  `dc_msg_get_quoted_msg()` #1975 #1984 #1985 #1987 #1989 #2004
+
+- require quorum to enable encryption #1946
+
+- speed up and clean up account creation #1912 #1927 #1960 #1961
+
+- configure now collects recent contacts and fetches last messages
+  unless disabled by `fetch_existing` config-option #1913 #2003
+  EDIT: `fetch_existing` renamed to `fetch_existing_msgs` in 1.48.0 #2042
+
+- emit `DC_EVENT_CHAT_MODIFIED` on contact rename
+  and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
+
+- add `dc_set_chat_protection()`; the `protect` parameter in
+  `dc_create_group_chat()` will be removed in an upcoming release;
+  up to then, UIs using the "verified group" paradigm
+  should not use `dc_set_chat_protection()` #1968 #2014 #2001 #2012 #2007
+
+- remove unneeded `DC_STR_COUNT` #1991
+
+- mark all failed messages as failed when receiving an NDN #1993
+
+- check some easy cases for bad system clock and outdated app #1901
+
+- fix import temporary directory usage #1929
+
+- fix forcing encryption for reset peers #1998
+
+- fix: do not allow to save drafts in non-writeable chats #1997
+
+- fix: do not show HTML if there is no content and there is an attachment #1988
+
+- fix recovering offline/lost connections, fixes background receive bug #1983
+
+- fix ordering of accounts returned by `dc_accounts_get_all()` #1909
+
+- fix whitespace for summaries #1938
+
+- fix: improve sentbox name guessing #1941
+
+- fix: avoid manual poll impl for accounts events #1944
+
+- fix encoding newlines in param as a preparation for storing quotes #1945
+
+- fix: internal and ffi error handling #1967 #1966 #1959 #1911 #1916 #1917 #1915
+
+- fix ci #1928 #1931 #1932 #1933 #1934 #1943
+
+- update provider-database #1940 #2005 #2006
+
+- update dependencies #1919 #1908 #1950 #1963 #1996 #2010 #2013
+
+
+## 1.46.0
+
+- breaking change: `dc_configure()` report errors in
+  `DC_EVENT_CONFIGURE_PROGRESS`: capturing error events is no longer working
+  #1886 #1905
+
+- breaking change: removed `DC_LP_{IMAP|SMTP}_SOCKET*` from `server_flags`;
+  added `mail_security` and `send_security` using `DC_SOCKET` enum #1835
+
+- parse multiple servers in Mozilla autoconfig #1860
+
+- try multiple servers for each protocol #1871
+
+- do IMAP and SMTP configuration in parallel #1891
+
+- configuration cleanup and speedup #1858 #1875 #1889 #1904 #1906
+
+- secure-join cleanup, testing, fixing #1876 #1877 #1887 #1888 #1896 #1899 #1900
+
+- do not reset peerstate on encrypted messages,
+  ignore reordered autocrypt headers #1885 #1890
+
+- always sort message replies after parent message #1852
+
+- add an index to significantly speed up `get_fresh_msg_cnt()` #1881
+
+- improve mimetype guessing for PDF and many other formats #1857 #1861
+
+- improve accepting invalid html #1851
+
+- improve tests, cleanup and ci #1850 #1856 #1859 #1861 #1884 #1894 #1895
+
+- tweak HELO command #1908
+
+- make `dc_accounts_get_all()` return accounts sorted #1909
+
+- fix KML coordinates precision used for location streaming #1872
+
+- fix cancelling import/export #1855
+
+
+## 1.45.0
+
+- add `dc_accounts_t` account manager object and related api functions #1784
+
+- add capability to import backups as .tar files,
+  which will become the default in a subsequent release #1749
+
+- try various server domains on configuration #1780 #1838
+
+- recognize .tgs files as stickers #1826
+
+- remove X-Mailer debug header #1819
+
+- improve guessing message types from extension #1818
+
+- fix showing unprotected subjects in encrypted messages #1822
+
+- fix threading in interaction with non-delta-clients #1843
+
+- fix handling if encryption degrades #1829
+
+- fix webrtc-servers names set by the user #1831
+
+- update provider database #1828
+
+- update async-imap to fix Oauth2 #1837
+
+- optimize jpeg assets with trimage #1840
+
+- add tests and documentations #1809 #1820
+
+
+## 1.44.0
+
+- fix peerstate issues #1800 #1805
+
+- fix a crash related to muted chats #1803
+
+- fix incorrect dimensions sometimes reported for images #1806
+
+- fixed `dc_chat_get_remaining_mute_duration` function #1807
+
+- handle empty tags (e.g. `<br/>`) in HTML mails #1810
+
+- always translate the message about disappearing messages timer change #1813
+
+- improve footer detection in plain text email #1812
+
+- update device chat icon to fix warnings in iOS logs #1802
+
+- fix deletion of multiple messages #1795
+
+
+## 1.43.0
+
+- improve using own jitsi-servers #1785
+
+- fix smtp-timeout tweaks for larger mails #1797
+
+- more bug fixes and updates #1794 #1792 #1789 #1787
+
+
+## 1.42.0
+
+- new qr-code type `DC_QR_WEBRTC` #1779
+
+- new `dc_chatlist_get_summary2()` api #1771
+
+- tweak smtp-timeout for larger mails #1782
+
+- optimize read-receipts #1765
+
+- Allow http scheme for DCACCOUNT URLs #1770
+
+- improve tests #1769
+
+- bug fixes #1766 #1772 #1773 #1775 #1776 #1777
+
+
+## 1.41.0
+
+- new apis to initiate video chats #1718 #1735
+
+- new apis `dc_msg_get_ephemeral_timer()`
+  and `dc_msg_get_ephemeral_timestamp()`
+
+- new api `dc_chatlist_get_summary2()` #1771
+
+- improve IMAP handling #1703 #1704
+
+- improve ephemeral messages #1696 #1705
+
+- mark location-messages as auto-generated #1715
+
+- multi-device avatar-sync #1716 #1717
+
+- improve python bindings #1732 #1733 #1738 #1769
+
+- Allow http scheme for DCACCOUNT urls #1770
+
+- more fixes #1702 #1706 #1707 #1710 #1719 #1721
+  #1723 #1734 #1740 #1744 #1748 #1760 #1766 #1773 #1765
+
+- refactorings #1712 #1714 #1757
+
+- update toolchains and dependencies #1726 #1736 #1737 #1742 #1743 #1746
+
+
+## 1.40.0
+
+- introduce ephemeral messages #1540 #1680 #1683 #1684 #1691 #1692
+
+- `DC_MSG_ID_DAYMARKER` gets timestamp attached #1677 #1685
+
+- improve idle #1690 #1688
+
+- fix message processing issues by sequential processing #1694
+
+- refactorings #1670 #1673
+
+
 ## 1.39.0
 
 - fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
@@ -583,4 +865,3 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
-

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.39.0"
+version = "1.49.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,7 +12,7 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.6.0", default-features = false } 
+pgp = { version = "0.7.0", default-features = false } 
 hex = "0.4.0"
 sha2 = "0.9.0"
 rand = "0.7.0"
@@ -20,12 +20,12 @@ smallvec = "1.0.0"
 surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = { version = "0.3" }
+async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.3.1"
+async-imap = "0.4.0"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.1", features = ["unstable"] }
+async-std = { version = "1.6.4", features = ["unstable"] }
 base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -34,23 +34,22 @@ serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
 kamadak-exif = "0.5"
-lazy_static = "1.4.0"
+once_cell = "1.4.1"
 regex = "1.1.6"
-rusqlite = { version = "0.23", features = ["bundled"] }
-r2d2_sqlite = "0.16.0"
+rusqlite = { version = "0.24", features = ["bundled"] }
+r2d2_sqlite = "0.17.0"
 r2d2 = "0.8.5"
-strum = "0.18.0"
-strum_macros = "0.18.0"
+strum = "0.19.0"
+strum_macros = "0.19.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
-itertools = "0.8.0"
-image-meta = "0.1.0"
+itertools = "0.9.0"
 quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-sanitize-filename = "0.2.1"
+sanitize-filename = "0.3.0"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.1"
+mailparse = "0.13.0"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
@@ -60,11 +59,15 @@ anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
 async-std-resolver = "0.19.5"
+async-tar = "0.3.0"
+uuid = { version = "0.8", features = ["serde", "v4"] }
 
 pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
+dirs = { version = "3.0.1", optional=true }
+toml = "0.5.6"
 
 
 [dev-dependencies]
@@ -72,8 +75,9 @@ tempfile = "3.0"
 pretty_assertions = "0.6.1"
 pretty_env_logger = "0.4.0"
 proptest = "0.10"
-async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
-smol = "0.1.10"
+async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
+futures-lite = "1.7.0"
+criterion = "0.3"
 
 [workspace]
 members = [
@@ -92,10 +96,14 @@ path = "examples/repl/main.rs"
 required-features = ["repl"]
 
 
+[[bench]]
+name = "create_account"
+harness = false
+
 [features]
 default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 

README.md

@@ -95,7 +95,8 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-
+- `RUST_LOG=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+SMTP tracing in addition to info messages.
 
 ### Expensive tests
 

benches/create_account.rs

@@ -0,0 +1,26 @@
+use async_std::path::PathBuf;
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::accounts::Accounts;
+use tempfile::tempdir;
+
+async fn create_accounts(n: u32) {
+    let dir = tempdir().unwrap();
+    let p: PathBuf = dir.path().join("accounts").into();
+
+    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+    for expected_id in 2..n {
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, expected_id);
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("create 1 account", |b| {
+        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

ci_scripts/docker-coredeps/Dockerfile

@@ -16,6 +16,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust nightly
+# Install Rust
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share
+rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share

ci_scripts/remote_python_packaging.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
@@ -30,6 +28,7 @@ set +x
 
 ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
     set +x -e
+    shopt -s huponexit
     cd $BUILDDIR
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL

ci_scripts/remote_tests_python.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
@@ -24,10 +22,12 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
+
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+
+    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL

ci_scripts/remote_tests_rust.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 set -e
@@ -20,10 +18,10 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=x86_64-unknown-linux-gnu
     export RUSTC_WRAPPER=sccache
 

ci_scripts/run-python-test.sh

@@ -4,6 +4,7 @@
 # and tox/pytest. 
 
 set -e -x
+shopt -s huponexit
 
 # for core-building and python install step
 export DCC_RS_TARGET=debug

ci_scripts/run-rust-test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -ex
+shopt -s huponexit
 
 #export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1

ci_scripts/run_all.sh

@@ -15,6 +15,7 @@ cargo build --release -p deltachat_ffi
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV=$(pwd)
+export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.39.0"
+version = "1.49.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -23,6 +23,7 @@ serde_json = "1.0"
 async-std = "1.6.0"
 anyhow = "1.0.28"
 thiserror = "1.0.14"
+rand = "0.7.3"
 
 [features]
 default = ["vendored"]

deltachat-ffi/Doxyfile

@@ -236,12 +236,6 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
-# This tag can be used to specify a number of word-keyword mappings (TCL only).
-# A mapping has the form "name=value". For example adding "class=itcl::class"
-# will allow you to use the command class in the itcl::class meaning.
-
-TCL_SUBST              =
-
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all

deltachat-ffi/build.rs

@@ -19,10 +19,10 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
+        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();

deltachat-ffi/src/dc_array.rs

@@ -1,46 +1,56 @@
+use crate::chat::ChatItem;
+use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
+use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
+    MsgIds(Vec<MsgId>),
+    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub fn new(capacity: usize) -> Self {
-        dc_array_t::Uint(Vec::with_capacity(capacity))
-    }
-
-    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
-    pub fn new_locations(capacity: usize) -> Self {
-        dc_array_t::Locations(Vec::with_capacity(capacity))
-    }
-
-    pub fn add_id(&mut self, item: u32) {
-        if let Self::Uint(array) = self {
-            array.push(item);
-        } else {
-            panic!("Attempt to add id to array of other type");
-        }
-    }
-
-    pub fn add_location(&mut self, location: Location) {
-        if let Self::Locations(array) = self {
-            array.push(location)
-        } else {
-            panic!("Attempt to add a location to array of other type");
-        }
-    }
-
-    pub fn get_id(&self, index: usize) -> u32 {
+    pub(crate) fn get_id(&self, index: usize) -> u32 {
         match self {
+            Self::MsgIds(array) => array[index].to_u32(),
+            Self::Chat(array) => match array[index] {
+                ChatItem::Message { msg_id } => msg_id.to_u32(),
+                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
+                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
+            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index] as u32,
+            Self::Uint(array) => array[index],
         }
     }
 
-    pub fn get_location(&self, index: usize) -> &Location {
+    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(array) => array.get(index).and_then(|item| match item {
+                ChatItem::Message { .. } => None,
+                ChatItem::Marker1 { .. } => None,
+                ChatItem::DayMarker { timestamp } => Some(*timestamp),
+            }),
+            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(_) => None,
+            Self::Locations(array) => array
+                .get(index)
+                .and_then(|location| location.marker.as_deref()),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -48,55 +58,18 @@ impl dc_array_t {
         }
     }
 
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Self::Locations(array) => array.is_empty(),
-            Self::Uint(array) => array.is_empty(),
-        }
-    }
-
     /// Returns the number of elements in the array.
-    pub fn len(&self) -> usize {
+    pub(crate) fn len(&self) -> usize {
         match self {
+            Self::MsgIds(array) => array.len(),
+            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub fn clear(&mut self) {
-        match self {
-            Self::Locations(array) => array.clear(),
-            Self::Uint(array) => array.clear(),
-        }
-    }
-
-    pub fn search_id(&self, needle: u32) -> Option<usize> {
-        if let Self::Uint(array) = self {
-            for (i, &u) in array.iter().enumerate() {
-                if u == needle {
-                    return Some(i);
-                }
-            }
-            None
-        } else {
-            panic!("Attempt to search for id in array of other type");
-        }
-    }
-
-    pub fn sort_ids(&mut self) {
-        if let dc_array_t::Uint(v) = self {
-            v.sort();
-        } else {
-            panic!("Attempt to sort array of something other than uints");
-        }
-    }
-
-    pub fn as_ptr(&self) -> *const u32 {
-        if let dc_array_t::Uint(v) = self {
-            v.as_ptr()
-        } else {
-            panic!("Attempt to convert array of something other than uints to raw");
-        }
+    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
+        (0..self.len()).find(|i| self.get_id(*i) == needle)
     }
 }
 
@@ -106,6 +79,18 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
+impl From<Vec<MsgId>> for dc_array_t {
+    fn from(array: Vec<MsgId>) -> Self {
+        dc_array_t::MsgIds(array)
+    }
+}
+
+impl From<Vec<ChatItem>> for dc_array_t {
+    fn from(array: Vec<ChatItem>) -> Self {
+        dc_array_t::Chat(array)
+    }
+}
+
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -118,12 +103,11 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let mut arr = dc_array_t::new(7);
-        assert!(arr.is_empty());
+        let arr: dc_array_t = Vec::<u32>::new().into();
+        assert!(arr.len() == 0);
 
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
 
         assert_eq!(arr.len(), 1000);
 
@@ -131,31 +115,15 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        arr.clear();
-
-        assert!(arr.is_empty());
-
-        arr.add_id(13);
-        arr.add_id(7);
-        arr.add_id(666);
-        arr.add_id(0);
-        arr.add_id(5000);
-
-        arr.sort_ids();
-
-        assert_eq!(arr.get_id(0), 0);
-        assert_eq!(arr.get_id(1), 7);
-        assert_eq!(arr.get_id(2), 13);
-        assert_eq!(arr.get_id(3), 666);
+        assert_eq!(arr.search_id(10), Some(8));
+        assert_eq!(arr.search_id(1), None);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let mut arr = dc_array_t::new(7);
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/string.rs

@@ -105,8 +105,9 @@ impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
     #[cfg(not(target_os = "windows"))]
     fn to_c_string(&self) -> Result<CString, CStringError> {
         use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        CString::new(self.as_ref().as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         })
     }
 
@@ -122,8 +123,9 @@ fn os_str_to_c_string_unicode(
     os_str: &dyn AsRef<std::ffi::OsStr>,
 ) -> Result<CString, CStringError> {
     match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         }),
         None => Err(CStringError::NotUnicode),
     }

draft/msgwork_new_imap_jobs.rst

@@ -0,0 +1,66 @@
+
+simplify/streamline mark-seen/delete/move/send-mdn job handling
+---------------------------------------------------------------
+
+Idea: Introduce a new "msgwork" sql table that looks very
+much like the jobs table but has a primary key "msgid"
+and no job id and no foreign-id anymore. This opens up
+bulk-processing by looking at the whole table and combining
+flag-setting to reduce imap-roundtrips and select-folder calls.
+
+Concretely, these IMAP jobs:
+
+    DeleteMsgOnImap
+    MarkseenMsgOnImap
+    MoveMsg
+
+Would be replaced by a few per-message columns in the new msgwork table:
+
+- needs_mark_seen: (bool) message shall be marked as seen on imap
+- needs_to_move: (bool) message should be moved to mvbox_folder
+- deletion_time: (target_time or 0) message shall be deleted at specified time
+- needs_send_mdn: (bool) MDN shall be sent
+
+The various places that currently add the (replaced) jobs
+would now add/modify the respective message record in the message-work table.
+
+Looking at a single message-work entry conceptually looks like this::
+
+    if msg.server_uid==0:
+        return RetryLater  # nothing can be done without server_uid
+
+    if msg.deletion_time > current_time:
+        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
+        clear(needs_deletion)
+        clear(mark_seen)
+
+    if needs_mark_seen:
+        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
+        clear(needs_mark_seen)
+
+    if needs_send_mdn:
+        schedule_smtp_send_mdn(msg)
+        clear(needs_send_mdn)
+
+    if any_flag_set():
+        retrylater
+    # remove msgwork entry from table
+
+
+Notes/Questions:
+
+- it's unclear how much we need per-message retry-time tracking/backoff
+
+- drafting bulk processing algo is useful before
+  going for the implementation, i.e. including select_folder calls etc.
+
+- maybe it's better to not have bools for the flags but
+
+  0 (no change)
+  1 (set the imap flag)
+  2 (clear the imap flag)
+
+  and design such that we can cover all imap flags.
+
+- It might not be neccessary to keep needs_send_mdn state in this table
+  if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/repl/cmdline.rs

@@ -1,8 +1,10 @@
+extern crate dirs;
+
 use std::str::FromStr;
 
 use anyhow::{bail, ensure};
 use async_std::path::Path;
-use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
+use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility, ProtectionStatus};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
@@ -17,7 +19,7 @@ use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
-use deltachat::Event;
+use deltachat::EventType;
 use deltachat::{config, provider};
 
 /// Reset database tables.
@@ -86,7 +88,7 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
@@ -157,7 +159,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -191,7 +193,6 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.is_starred() { "★" } else { "" },
         if msg.get_from_id() == 1 as libc::c_uint {
             ""
         } else if msg.get_state() == MessageState::InSeen {
@@ -202,6 +203,15 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             "[FRESH]"
         },
         if msg.is_info() { "[INFO]" } else { "" },
+        if msg.get_viewtype() == Viewtype::VideochatInvitation {
+            format!(
+                "[VIDEOCHAT-INVITATION: {}, type={}]",
+                msg.get_videochat_url().unwrap_or_default(),
+                msg.get_videochat_type().unwrap_or_default()
+            )
+        } else {
+            "".to_string()
+        },
         if msg.is_forwarded() {
             "[FORWARDED]"
         } else {
@@ -215,7 +225,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
 async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
-        if msg_id.is_daymarker() {
+        if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
             println!(
                 "--------------------------------------------------------------------------------"
             );
@@ -275,7 +285,9 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
                     "addr unset"
                 }
             );
-            let peerstate = Peerstate::from_addr(context, &addr).await;
+            let peerstate = Peerstate::from_addr(context, &addr)
+                .await
+                .expect("peerstate error");
             if peerstate.is_some() && contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
@@ -345,7 +357,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  createchat <contact-id>\n\
                  createchatbymsg <msg-id>\n\
                  creategroup <name>\n\
-                 createverified <name>\n\
+                 createprotected <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
                  groupname <name>\n\
@@ -359,6 +371,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
+                 videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
                  listmedia\n\
@@ -366,6 +379,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  unarchive <chat-id>\n\
                  pin <chat-id>\n\
                  unpin <chat-id>\n\
+                 protect <chat-id>\n\
+                 unprotect <chat-id>\n\
                  delchat <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
@@ -373,8 +388,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
-                 star <msg-id>\n\
-                 unstar <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
                  listcontacts [<query>]\n\
@@ -391,7 +404,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
-                 emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
                  ============================================="
@@ -430,17 +442,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
-            imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportBackup, &dir).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
+            imex(&context, ImexMode::ImportBackup, arg1).await?;
         }
         "export-keys" => {
-            imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportSelfKeys, &dir).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -509,7 +525,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}",
+                        "{}#{}: {} [{} fresh] {}{}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
@@ -519,6 +535,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             ChatVisibility::Archived => "📦",
                             ChatVisibility::Pinned => "📌",
                         },
+                        if chat.is_protected() { "🛡️" } else { "" },
                     );
                     let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
@@ -574,6 +591,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let sel_chat = sel_chat.as_ref().unwrap();
 
             let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await;
+            let msglist: Vec<MsgId> = msglist
+                .into_iter()
+                .map(|x| match x {
+                    ChatItem::Message { msg_id } => msg_id,
+                    ChatItem::Marker1 => MsgId::new(DC_MSG_ID_MARKER1),
+                    ChatItem::DayMarker { .. } => MsgId::new(DC_MSG_ID_DAYMARKER),
+                })
+                .collect();
+
             let members = chat::get_chat_contacts(&context, sel_chat.id).await;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
@@ -584,7 +610,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 format!("{} member(s)", members.len())
             };
             println!(
-                "{}#{}: {} [{}]{}{}",
+                "{}#{}: {} [{}]{}{} {}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
                 sel_chat.get_name(),
@@ -601,6 +627,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     },
                     _ => "".to_string(),
                 },
+                if sel_chat.is_protected() {
+                    "🛡️"
+                } else {
+                    ""
+                },
             );
             log_msglist(&context, &msglist).await?;
             if let Some(draft) = sel_chat.get_id().get_draft(&context).await? {
@@ -631,15 +662,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
-                chat::create_group_chat(&context, VerifiedStatus::Unverified, arg1).await?;
+                chat::create_group_chat(&context, ProtectionStatus::Unprotected, arg1).await?;
 
             println!("Group#{} created successfully.", chat_id);
         }
-        "createverified" => {
+        "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id = chat::create_group_chat(&context, VerifiedStatus::Verified, arg1).await?;
+            let chat_id =
+                chat::create_group_chat(&context, ProtectionStatus::Protected, arg1).await?;
 
-            println!("VerifiedGroup#{} created successfully.", chat_id);
+            println!("Group#{} created and protected successfully.", chat_id);
         }
         "addmember" => {
             ensure!(sel_chat.is_some(), "No chat selected");
@@ -799,6 +831,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "videochat" => {
+            ensure!(sel_chat.is_some(), "No chat selected.");
+            chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
+        }
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
@@ -845,9 +881,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg.set_text(Some(arg1.to_string()));
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
-        "updatedevicechats" => {
-            context.update_device_chats().await?;
-        }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
@@ -879,7 +912,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                         "archive" => ChatVisibility::Archived,
                         "unarchive" | "unpin" => ChatVisibility::Normal,
                         "pin" => ChatVisibility::Pinned,
-                        _ => panic!("Unexpected command (This should never happen)"),
+                        _ => unreachable!("arg0={:?}", arg0),
+                    },
+                )
+                .await?;
+        }
+        "protect" | "unprotect" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id
+                .set_protection(
+                    &context,
+                    match arg0 {
+                        "protect" => ProtectionStatus::Protected,
+                        "unprotect" => ProtectionStatus::Unprotected,
+                        _ => unreachable!("arg0={:?}", arg0),
                     },
                 )
                 .await?;
@@ -918,12 +965,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             message::markseen_msgs(&context, msg_ids).await;
         }
-        "star" | "unstar" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = vec![MsgId::new(0); 1];
-            msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::star_msgs(&context, msg_ids, arg0 == "star").await;
-        }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
@@ -1034,7 +1075,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         // "event" => {
         //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
         //     let event = arg1.parse()?;
-        //     let event = Event::from_u32(event).ok_or(format_err!("Event::from_u32({})", event))?;
+        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
         //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
         //     println!(
         //         "Sending event {:?}({}), received value {}.",
@@ -1061,11 +1102,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 seconds, device_cnt, server_cnt
             );
         }
-        "emptyserver" => {
-            ensure!(!arg1.is_empty(), "Argument <flags> missing");
-
-            message::dc_empty_server(&context, arg1.parse()?).await;
-        }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),
     }

examples/repl/main.rs

@@ -19,7 +19,7 @@ use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::Event;
+use deltachat::EventType;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
@@ -34,35 +34,35 @@ mod cmdline;
 use self::cmdline::*;
 
 /// Event Handler
-fn receive_event(event: Event) {
+fn receive_event(event: EventType) {
     let yellow = Color::Yellow.normal();
     match event {
-        Event::Info(msg) => {
+        EventType::Info(msg) => {
             /* do not show the event as this would fill the screen */
             info!("{}", msg);
         }
-        Event::SmtpConnected(msg) => {
+        EventType::SmtpConnected(msg) => {
             info!("[SMTP_CONNECTED] {}", msg);
         }
-        Event::ImapConnected(msg) => {
+        EventType::ImapConnected(msg) => {
             info!("[IMAP_CONNECTED] {}", msg);
         }
-        Event::SmtpMessageSent(msg) => {
+        EventType::SmtpMessageSent(msg) => {
             info!("[SMTP_MESSAGE_SENT] {}", msg);
         }
-        Event::Warning(msg) => {
+        EventType::Warning(msg) => {
             warn!("{}", msg);
         }
-        Event::Error(msg) => {
+        EventType::Error(msg) => {
             error!("{}", msg);
         }
-        Event::ErrorNetwork(msg) => {
+        EventType::ErrorNetwork(msg) => {
             error!("[NETWORK] msg={}", msg);
         }
-        Event::ErrorSelfNotInGroup(msg) => {
+        EventType::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
-        Event::MsgsChanged { chat_id, msg_id } => {
+        EventType::MsgsChanged { chat_id, msg_id } => {
             info!(
                 "{}",
                 yellow.paint(format!(
@@ -71,34 +71,44 @@ fn receive_event(event: Event) {
                 ))
             );
         }
-        Event::ContactsChanged(_) => {
+        EventType::ContactsChanged(_) => {
             info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
-        Event::LocationChanged(contact) => {
+        EventType::LocationChanged(contact) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
-        Event::ConfigureProgress(progress) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
-            );
+        EventType::ConfigureProgress { progress, comment } => {
+            if let Some(comment) = comment {
+                info!(
+                    "{}",
+                    yellow.paint(format!(
+                        "Received CONFIGURE_PROGRESS({} ‰, {})",
+                        progress, comment
+                    ))
+                );
+            } else {
+                info!(
+                    "{}",
+                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+                );
+            }
         }
-        Event::ImexProgress(progress) => {
+        EventType::ImexProgress(progress) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
-        Event::ImexFileWritten(file) => {
+        EventType::ImexFileWritten(file) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
-        Event::ChatModified(chat) => {
+        EventType::ChatModified(chat) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
@@ -158,7 +168,7 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 26] = [
+const CHAT_COMMANDS: [&str; 27] = [
     "listchats",
     "listarchived",
     "chat",
@@ -178,6 +188,7 @@ const CHAT_COMMANDS: [&str; 26] = [
     "send",
     "sendimage",
     "sendfile",
+    "videochat",
     "draft",
     "listmedia",
     "archive",
@@ -186,14 +197,12 @@ const CHAT_COMMANDS: [&str; 26] = [
     "unpin",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&str; 6] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
-    "star",
-    "unstar",
     "delmsg",
 ];
 const CONTACT_COMMANDS: [&str; 6] = [
@@ -271,12 +280,12 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            receive_event(event);
+            receive_event(event.typ);
         }
     });
 
@@ -409,7 +418,7 @@ async fn handle_cmd(
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await;
+                dc_join_securejoin(&ctx, arg1).await?;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),

examples/simple.rs

@@ -6,20 +6,20 @@ use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::message::Message;
-use deltachat::Event;
+use deltachat::EventType;
 
-fn cb(event: Event) {
+fn cb(event: EventType) {
     match event {
-        Event::ConfigureProgress(progress) => {
+        EventType::ConfigureProgress { progress, .. } => {
             log::info!("progress: {}", progress);
         }
-        Event::Info(msg) => {
+        EventType::Info(msg) => {
             log::info!("{}", msg);
         }
-        Event::Warning(msg) => {
+        EventType::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        Event::Error(msg) | Event::ErrorNetwork(msg) => {
+        EventType::Error(msg) | EventType::ErrorNetwork(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new("FakeOs".into(), dbfile.into())
+    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
@@ -45,7 +45,7 @@ async fn main() {
     let events = ctx.get_event_emitter();
     let events_spawn = async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            cb(event);
+            cb(event.typ);
         }
     });
 

python/CHANGELOG

@@ -1,6 +1,22 @@
-0.900.0 (DRAFT)
+1.44.0
+------
+
+- fix Chat.get_mute_duration()
+
+1.40.1
 ---------------
 
+- emit "ac_member_removed" event (with 'actor' being the removed contact) 
+  for when a user leaves a group.
+
+- fix create_contact(addr) when addr is the self-contact. 
+
+
+1.40.0
+---------------
+
+- uses latest 1.40+ Delta Chat core
+
 - refactored internals to use plugin-approach 
 
 - introduced PerAccount and Global hooks that plugins can implement 
@@ -10,6 +26,7 @@
 - introduced two documented examples for an echo and a group-membership
   tracking plugin. 
 
+
 0.800.0
 -------
 

python/README.rst

@@ -7,76 +7,14 @@ which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
-Installing bindings from source (Updated: 20-Jan-2020)
-=========================================================
-
-Install Rust and Cargo first.  Deltachat needs a specific nightly
-version, the easiest is probably to first install Rust stable from
-rustup and then use this to install the correct nightly version.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then GIT clone the deltachat-core-rust repo and get the actual
-rust- and cargo-toolchain needed by deltachat::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-   rustup show
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   ./install_python_bindings.py
-
-The installation might take a while, depending on your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run "liveconfig" functional tests you can set
-``DCC_NEW_TMP_EMAIL`` to:
-
-- a particular https-url that you can ask for from the delta
-  chat devs. This is implemented on the server side via
-  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
-
-- or the path of a file that contains two lines, each describing
-  via "addr=... mail_pw=..." a test account login that will
-  be used for the live tests.
-
-With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
-e-mail accounts and run through all functional "liveconfig" tests.
-
-
 Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
+without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
+yourself <sourceinstall>`_.
 
-We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -103,6 +41,78 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
+Running tests
+=============
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+This will run all "offline" tests and skip all functional
+end-to-end tests that require accounts on real e-mail servers.
+
+.. _livetests:
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
+
+    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
+
+With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
+These accounts exists for one 1hour and then are removed completely.
+One hour is enough to invoke pytest and run all offline and online tests:
+
+    pytest
+
+    # or if you have installed pytest-xdist for parallel test execution
+    pytest -n6
+
+Each test run creates new accounts.
+
+
+.. _sourceinstall:
+
+Installing bindings from source (Updated: July 2020)
+=========================================================
+
+Install Rust and Cargo first.
+The easiest is probably to use `rustup <https://rustup.rs/>`_.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then clone the deltachat-core-rust repo::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   python install_python_bindings.py
+
+The core compilation and bindings building might take a while,
+depending on the speed of your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+
 Code examples
 =============
 

python/doc/_templates/globaltoc.html

@@ -9,8 +9,7 @@
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
-  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
+  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/examples/group_tracking.py

@@ -32,16 +32,16 @@ class GroupTrackingPlugin:
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, message):
+    def ac_member_added(self, chat, contact, actor, message):
         print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_removed(self, chat, contact, message):
+    def ac_member_removed(self, chat, contact, actor, message):
         print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -69,11 +69,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     lp.sec("now looking at what the bot received")
     botproc.fnmatch_lines("""
-        *ac_member_added {}*
-    """.format(contact3.addr))
+        *ac_member_added {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
     botproc.fnmatch_lines("""
-        *ac_member_removed {}*
-    """.format(contact3.addr))
+        *ac_member_removed {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))

python/install_python_bindings.py

@@ -17,8 +17,12 @@ if __name__ == "__main__":
         os.environ["DCC_RS_DEV"] = dn
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
+
     if target == 'release':
+        extra = " -C lto=on -C embed-bitcode=yes"
+        os.environ["RUSTFLAGS"] = os.environ.get("RUSTFLAGS", "") + extra
         cmd.append("--release")
+
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)

python/src/deltachat/__init__.py

@@ -77,6 +77,7 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
+        ac.set_config("bot", "1")
         configtracker = ac.configure()
         configtracker.wait_finish()
 

python/src/deltachat/account.py

@@ -179,17 +179,6 @@ class Account(object):
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def empty_server_folders(self, inbox=False, mvbox=False):
-        """ empty server folders. """
-        flags = 0
-        if inbox:
-            flags |= const.DC_EMPTY_INBOX
-        if mvbox:
-            flags |= const.DC_EMPTY_MVBOX
-        if not flags:
-            raise ValueError("no flags set")
-        lib.dc_empty_server(self._dc_context, flags)
-
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -216,7 +205,7 @@ class Account(object):
     def create_contact(self, obj, name=None):
         """ create a (new) Contact or return an existing one.
 
-        Calling this method will always resulut in the same
+        Calling this method will always result in the same
         underlying contact id.  If there already is a Contact
         with that e-mail address, it is unblocked and its display
         `name` is updated if specified.
@@ -225,6 +214,19 @@ class Account(object):
         :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
+        (name, addr) = self.get_contact_addr_and_name(obj, name)
+        name = as_dc_charpointer(name)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        return Contact(self, contact_id)
+
+    def get_contact(self, obj):
+        if isinstance(obj, Contact):
+            return obj
+        (_, addr) = self.get_contact_addr_and_name(obj)
+        return self.get_contact_by_addr(addr)
+
+    def get_contact_addr_and_name(self, obj, name=None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -240,14 +242,7 @@ class Account(object):
 
         if name is None and displayname:
             name = displayname
-        return self._create_contact(addr, name)
-
-    def _create_contact(self, addr, name):
-        addr = as_dc_charpointer(addr)
-        name = as_dc_charpointer(name)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
-        return Contact(self, contact_id)
+        return (name, addr)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -275,6 +270,17 @@ class Account(object):
         """
         return Contact(self, contact_id)
 
+    def get_blocked_contacts(self):
+        """ return a list of all blocked contacts.
+
+        :returns: list of :class:`deltachat.contact.Contact` objects.
+        """
+        dc_array = ffi.gc(
+            lib.dc_get_blocked_contacts(self._dc_context),
+            lib.dc_array_unref
+        )
+        return list(iter_array(dc_array, lambda x: Contact(self, x)))
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -348,6 +354,9 @@ class Account(object):
     def get_deaddrop_chat(self):
         return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
+    def get_device_chat(self):
+        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
+
     def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
@@ -558,6 +567,9 @@ class Account(object):
         You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
         account is started.
 
+        If you are using this from a test, you may want to call
+        wait_all_initial_fetches() afterwards.
+
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
@@ -607,12 +619,24 @@ class Account(object):
         self.stop_io()
 
         self.log("remove dc_context references")
-        # the dc_context_unref triggers get_next_event to return ffi.NULL
-        # which in turns makes the event thread finish execution
+
+        # if _dc_context is unref'ed the event thread should quickly
+        # receive the termination signal. However, some python code might
+        # still hold a reference and so we use a secondary signal
+        # to make sure the even thread terminates if it receives any new
+        # event, indepedently from waiting for the core to send NULL to
+        # get_next_event().
+        self._event_thread.mark_shutdown()
         self._dc_context = None
 
         self.log("wait for event thread to finish")
-        self._event_thread.wait()
+        try:
+            self._event_thread.wait(timeout=2)
+        except RuntimeError as e:
+            self.log("Waiting for event thread failed: {}".format(e))
+
+        if self._event_thread.is_alive():
+            self.log("WARN: event thread did not terminate yet, ignoring.")
 
         self._shutdown_event.set()
 

python/src/deltachat/chat.py

@@ -57,10 +57,7 @@ class Chat(object):
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
-        return lib.dc_chat_get_type(self._dc_chat) in (
-            const.DC_CHAT_TYPE_GROUP,
-            const.DC_CHAT_TYPE_VERIFIED_GROUP
-        )
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
@@ -85,12 +82,20 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def is_verified(self):
-        """ return True if this chat is a verified group.
+    def can_send(self):
+        """Check if messages can be sent to a give chat.
+        This is not true eg. for the deaddrop or for the device-talk
 
-        :returns: True if chat is verified, False otherwise.
+        :returns: True if the chat is writable, False otherwise
         """
-        return lib.dc_chat_is_verified(self._dc_chat)
+        return lib.dc_chat_can_send(self._dc_chat)
+
+    def is_protected(self):
+        """ return True if this chat is a protected chat.
+
+        :returns: True if chat is protected, False otherwise.
+        """
+        return lib.dc_chat_is_protected(self._dc_chat)
 
     def get_name(self):
         """ return name of this chat.
@@ -137,7 +142,23 @@ class Chat(object):
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
         """
-        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
+        return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
+
+    def get_ephemeral_timer(self):
+        """ get ephemeral timer.
+
+        :returns: ephemeral timer value in seconds
+        """
+        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
+
+    def set_ephemeral_timer(self, timer):
+        """ set ephemeral timer.
+
+        :param: timer value in seconds
+
+        :returns: None
+        """
+        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
 
     def get_type(self):
         """ (deprecated) return type of this chat.
@@ -350,7 +371,7 @@ class Chat(object):
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.create_contact(obj)
+        contact = self.account.get_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))

python/src/deltachat/contact.py

@@ -51,6 +51,18 @@ class Contact(object):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
 
+    def set_blocked(self, block=True):
+        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
+        return lib.dc_block_contact(self.account._dc_context, self.id, block)
+
+    def block(self):
+        """ Block this contact. Message will not be seen/retrieved from this contact. """
+        return lib.dc_block_contact(self.account._dc_context, self.id, True)
+
+    def unblock(self):
+        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
+        return lib.dc_block_contact(self.account._dc_context, self.id, False)
+
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)

python/src/deltachat/cutil.py

@@ -17,7 +17,8 @@ def iter_array(dc_array_t, constructor):
 
 
 def from_dc_charpointer(obj):
-    return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    if obj != ffi.NULL:
+        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
 
 
 class DCLot:

python/src/deltachat/direct_imap.py

@@ -24,12 +24,13 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    imap = DirectImap(account)
-    if imap.select_config_folder("mvbox"):
+    if not hasattr(account, "direct_imap"):
+        imap = DirectImap(account)
+        if imap.select_config_folder("mvbox"):
+            imap.delete(ALL, expunge=True)
+        assert imap.select_config_folder("inbox")
         imap.delete(ALL, expunge=True)
-    assert imap.select_config_folder("inbox")
-    imap.delete(ALL, expunge=True)
-    setattr(account, "direct_imap", imap)
+        setattr(account, "direct_imap", imap)
 
 
 @deltachat.global_hookimpl
@@ -108,6 +109,15 @@ class DirectImap:
 
     def get_all_messages(self):
         assert not self._idling
+
+        # Flush unsolicited responses. IMAPClient has problems
+        # dealing with them: https://github.com/mjs/imapclient/issues/334
+        # When this NOOP was introduced, next FETCH returned empty
+        # result instead of a single message, even though IMAP server
+        # can only return more untagged responses than required, not
+        # less.
+        self.conn.noop()
+
         return self.conn.fetch(ALL, [FLAGS])
 
     def get_unread_messages(self):
@@ -159,9 +169,13 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[HEADER]', FLAGS]
+            requested = [b'BODY.PEEK[]', FLAGS]
             for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[HEADER]']
+                body_bytes = data[b'BODY[]']
+                if not body_bytes:
+                    log("Message", uid, "has empty body")
+                    continue
+
                 flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
@@ -192,6 +206,7 @@ class DirectImap:
             raise TimeoutError
         if terminate:
             self.idle_done()
+        self.account.log("imap-direct: idle_check returned {!r}".format(res))
         return res
 
     def idle_wait_for_seen(self):

python/src/deltachat/events.py

@@ -1,6 +1,7 @@
 import threading
 import time
 import re
+import os
 from queue import Queue, Empty
 
 import deltachat
@@ -48,6 +49,15 @@ class FFIEventLogger:
         if self.logid:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+
+        if os.name == "posix":
+            WARN = '\033[93m'
+            ERROR = '\033[91m'
+            ENDC = '\033[0m'
+            if message.startswith("DC_EVENT_WARNING"):
+                s = WARN + s + ENDC
+            if message.startswith("DC_EVENT_ERROR"):
+                s = ERROR + s + ENDC
         with self._loglock:
             print(s, flush=True)
 
@@ -86,13 +96,21 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
+    def get_info_contains(self, regex):
+        rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
+            if rex.search(ev.data2):
                 return ev
 
+    def get_info_regex_groups(self, regex, check_error=True):
+        rex = re.compile(regex)
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
+            m = rex.match(ev.data2)
+            if m is not None:
+                return m.groups()
+
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))
@@ -111,6 +129,15 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
+    def wait_all_initial_fetches(self):
+        """Has to be called after start_io() to wait for fetch_existing_msgs to run
+        so that new messages are not mistaken for old ones:
+        - ac1 and ac2 are created
+        - ac1 sends a message to ac2
+        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
+        - therefore no DC_EVENT_INCOMING_MSG is sent"""
+        self.get_info_contains("Done fetching existing messages")
+
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")
@@ -139,6 +166,7 @@ class EventThread(threading.Thread):
         self.account = account
         super(EventThread, self).__init__(name="events")
         self.setDaemon(True)
+        self._marked_for_shutdown = False
         self.start()
 
     @contextmanager
@@ -147,12 +175,15 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def wait(self):
+    def mark_shutdown(self):
+        self._marked_for_shutdown = True
+
+    def wait(self, timeout=None):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
-        self.join()
+        self.join(timeout=timeout)
 
     def run(self):
         """ get and run events until shutdown. """
@@ -164,10 +195,12 @@ class EventThread(threading.Thread):
             lib.dc_get_event_emitter(self.account._dc_context),
             lib.dc_event_emitter_unref,
         )
-        while 1:
+        while not self._marked_for_shutdown:
             event = lib.dc_get_next_event(event_emitter)
             if event == ffi.NULL:
                 break
+            if self._marked_for_shutdown:
+                break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
             # the following code relates to the deltachat/_build.py's helper

python/src/deltachat/hookspec.py

@@ -16,7 +16,7 @@ class PerAccount:
     """ per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
-    Hooks are not allowed to block/last long as this
+    Hooks are generally not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
     @classmethod
@@ -31,10 +31,6 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
-
-        DANGER: this hook is executed from the callback invoked by core.
-        Hook implementations need to be short running and can typically
-        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec
@@ -55,19 +51,37 @@ class PerAccount:
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP. """
+        """ Called when an outgoing message has been delivered to SMTP.
+
+        :param message: Message that was just delivered.
+        """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title. """
+        """ Chat was created or modified regarding membership, avatar, title.
+
+        :param chat: Chat which was modified.
+        """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, message):
-        """ Called for each contact added to an accepted chat. """
+    def ac_member_added(self, chat, contact, actor, message):
+        """ Called for each contact added to an accepted chat.
+
+        :param chat: Chat where contact was added.
+        :param contact: Contact that was added.
+        :param actor: Who added the contact (None if it was our self-addr)
+        :param message: The original system message that reports the addition.
+        """
 
     @account_hookspec
-    def ac_member_removed(self, chat, contact, message):
-        """ Called for each contact removed from a chat.  """
+    def ac_member_removed(self, chat, contact, actor, message):
+        """ Called for each contact removed from a chat.
+
+        :param chat: Chat where contact was removed.
+        :param contact: Contact that was removed.
+        :param actor: Who removed the contact (None if it was our self-addr)
+        :param message: The original system message that reports the removal.
+        """
 
 
 class Global:

python/src/deltachat/message.py

@@ -1,6 +1,7 @@
 """ The Message object. """
 
 import os
+import re
 from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
@@ -154,6 +155,47 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
+    @props.with_doc
+    def ephemeral_timer(self):
+        """Ephemeral timer in seconds
+
+        :returns: timer in seconds or None if there is no timer
+        """
+        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
+        if timer:
+            return timer
+
+    @props.with_doc
+    def ephemeral_timestamp(self):
+        """UTC time when the message will be deleted.
+
+        :returns: naive datetime.datetime() object or None if the timer is not started.
+        """
+        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
+        if ts:
+            return datetime.utcfromtimestamp(ts)
+
+    @property
+    def quoted_text(self):
+        """Text inside the quote
+
+        :returns: Quoted text"""
+        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
+
+    @property
+    def quote(self):
+        """Quote getter
+
+        :returns: Quoted message, if found in the database"""
+        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
+        if msg:
+            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
+
+    @quote.setter
+    def quote(self, quoted_message):
+        """Quote setter"""
+        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -336,20 +378,43 @@ def get_viewtype_code_from_name(view_type_name):
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
-        if res:
-            contact = msg.account.get_contact_by_addr(res[1])
-            if contact:
-                d = dict(chat=msg.chat, contact=contact, message=msg)
-                return "ac_member_" + res[0], d
+        if not res:
+            return
+        action, affected, actor = res
+        affected = msg.account.get_contact_by_addr(affected)
+        if actor == "me":
+            actor = None
+        else:
+            actor = msg.account.get_contact_by_addr(actor)
+        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
+        return "ac_member_" + res[0], d
+
+
+def extract_addr(text):
+    m = re.match(r'.*\((.+@.+)\)', text)
+    if m:
+        text = m.group(1)
+    text = text.rstrip(".")
+    return text.strip()
 
 
 def parse_system_add_remove(text):
+    """ return add/remove info from parsing the given system message text.
+
+    returns a (action, affected, actor) triple """
+
     # Member Me (x@y) removed by a@b.
-    # Member x@y removed by a@b
+    # Member x@y added by a@b
+    # Member With space (tmp1@x.org) removed by tmp2@x.org.
+    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+    # Group left by some one (tmp1@x.org).
+    # Group left by tmp1@x.org.
     text = text.lower()
-    parts = text.split()
-    if parts[0] == "member":
-        if parts[2] in ("removed", "added"):
-            return parts[2], parts[1]
-        if parts[3] in ("removed", "added"):
-            return parts[3], parts[2].strip("()")
+    m = re.match(r'member (.+) (removed|added) by (.+)', text)
+    if m:
+        affected, action, actor = m.groups()
+        return action, extract_addr(affected), extract_addr(actor)
+    if text.startswith("group left by "):
+        addr = extract_addr(text[13:])
+        if addr:
+            return "removed", addr, addr

python/src/deltachat/testplugin.py

@@ -32,6 +32,10 @@ def pytest_addoption(parser):
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
+    parser.addoption(
+        "--strict-tls", action="store_true",
+        help="Never accept invalid TLS certificates for test accounts",
+    )
 
 
 def pytest_configure(config):
@@ -152,7 +156,7 @@ class SessionLiveConfigFromURL:
             assert index == len(self.configlist), index
             res = requests.post(self.url)
             if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
+                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
             d = res.json()
             config = dict(addr=d["email"], mail_pw=d["password"])
             self.configlist.append(config)
@@ -231,10 +235,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._evtracker.set_timeout(30)
             ac.addr = ac.get_self_contact().addr
             ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac))
+                logger = FFIEventLogger(ac)
+                logger.init_time = self.init_time
+                ac.add_account_plugin(logger)
             self._accounts.append(ac)
             return ac
 
@@ -244,10 +251,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(2)
-            return ac
+            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
 
         def _preconfigure_key(self, account, addr):
             # Only set a key if we haven't used it yet for another account.
@@ -282,16 +286,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
 
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            if pytestconfig.getoption("--strict-tls"):
+                # Enable strict certificate checks for online accounts
+                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
             if pre_generated_key:
                 self._preconfigure_key(ac, configdict['addr'])
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             return ac, dict(configdict)
 
         def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
@@ -327,13 +330,18 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
+            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
+            direct_imap object of an online account. This simulates the user setting
+            up a new device without importing a backup.
+
+            `pre_generated_key` only means that a key from python/tests/data/key is
+            used in order to speed things up.
+            """
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             if pre_generated_key:
                 self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
@@ -341,19 +349,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
+            if hasattr(account, "direct_imap"):
+                # Attach the existing direct_imap. If we did not do this, a new one would be created and
+                # delete existing messages (see dc_account_extra_configure(configure))
+                ac.direct_imap = account.direct_imap
             ac._configtracker = ac.configure()
             return ac
 
         def wait_configure_and_start_io(self):
+            started_accounts = []
             for acc in self._accounts:
                 if hasattr(acc, "_configtracker"):
                     acc._configtracker.wait_finish()
+                    acc._evtracker.consume_events()
+                    acc.get_device_chat().mark_noticed()
                     del acc._configtracker
                 acc.set_config("bcc_self", "0")
                 if acc.is_configured() and not acc.is_started():
                     acc.start_io()
+                    started_accounts.append(acc)
                 print("{}: {} account was successfully setup".format(
                     acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in started_accounts:
+                acc._evtracker.wait_all_initial_fetches()
 
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
@@ -492,6 +510,9 @@ def lp():
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
 
+        def indent(self, msg):
+            print("  " + msg)
+
     return Printer()
 
 

python/tests/test_account.py

@@ -11,8 +11,21 @@ from datetime import datetime, timedelta
 
 
 @pytest.mark.parametrize("msgtext,res", [
-    ("Member Me (tmp1@x.org) removed by tmp2@x.org.", ("removed", "tmp1@x.org")),
-    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org")),
+    ("Member Me (tmp1@x.org) removed by tmp2@x.org.",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by tmp2@x.org.",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by me",
+        ("removed", "tmp1@x.org", "me")),
+    ("Group left by some one (tmp1@x.org).",
+        ("removed", "tmp1@x.org", "tmp1@x.org")),
+    ("Group left by tmp1@x.org.",
+        ("removed", "tmp1@x.org", "tmp1@x.org")),
+    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org", "tmp2@x.org")),
+    ("Member nothing bla bla", None),
+    ("Another unknown system message", None),
 ])
 def test_parse_system_add_remove(msgtext, res):
     from deltachat.message import parse_system_add_remove
@@ -116,6 +129,25 @@ class TestOfflineContact:
         assert not contact1.is_blocked()
         assert not contact1.is_verified()
 
+    def test_get_blocked(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact1 = ac1.create_contact("some1@example.org", name="some1")
+        contact2 = ac1.create_contact("some2@example.org", name="some2")
+        ac1.create_contact("some3@example.org", name="some3")
+        assert ac1.get_blocked_contacts() == []
+        contact1.block()
+        assert ac1.get_blocked_contacts() == [contact1]
+        contact2.block()
+        blocked = ac1.get_blocked_contacts()
+        assert len(blocked) == 2 and contact1 in blocked and contact2 in blocked
+        contact2.unblock()
+        assert ac1.get_blocked_contacts() == [contact1]
+
+    def test_create_self_contact(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact1 = ac1.create_contact(ac1.get_config("addr"))
+        assert contact1.id == 1
+
     def test_get_contacts_and_delete(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact("some1@example.org", name="some1")
@@ -148,6 +180,16 @@ class TestOfflineContact:
         with pytest.raises(ValueError):
             ac1.create_chat(ac3)
 
+    def test_contact_rename(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact = ac1.create_contact("some1@example.com", name="some1")
+        chat = ac1.create_chat(contact)
+        assert chat.get_name() == "some1"
+        ac1.create_contact("some1@example.com", name="renamed")
+        ev = ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED")
+        assert ev.data1 == chat.id
+        assert chat.get_name() == "renamed"
+
 
 class TestOfflineChat:
     @pytest.fixture
@@ -246,6 +288,28 @@ class TestOfflineChat:
         qr = chat.get_join_qr()
         assert ac2.check_qr(qr).is_ask_verifygroup
 
+    def test_removing_blocked_user_from_group(self, ac1, lp):
+        """
+        Test that blocked contact is not unblocked when removed from a group.
+        See https://github.com/deltachat/deltachat-core-rust/issues/2030
+        """
+        lp.sec("Create a group chat with a contact")
+        contact = ac1.create_contact("some1@example.org")
+        group = ac1.create_group_chat("title", contacts=[contact])
+        group.send_text("First group message")
+
+        lp.sec("ac1 blocks contact")
+        contact.block()
+        assert contact.is_blocked()
+
+        lp.sec("ac1 removes contact from their group")
+        group.remove_contact(contact)
+        assert contact.is_blocked()
+
+        lp.sec("ac1 adding blocked contact unblocks it")
+        group.add_contact(contact)
+        assert not contact.is_blocked()
+
     def test_get_set_profile_image_simple(self, ac1, data):
         chat = ac1.create_group_chat(name="title1")
         p = data.get_path("d.png")
@@ -258,15 +322,23 @@ class TestOfflineChat:
     def test_mute(self, ac1):
         chat = ac1.create_group_chat(name="title1")
         assert not chat.is_muted()
+        assert chat.get_mute_duration() == 0
         chat.mute()
         assert chat.is_muted()
+        assert chat.get_mute_duration() == -1
         chat.unmute()
         assert not chat.is_muted()
         chat.mute(50)
         assert chat.is_muted()
+        assert chat.get_mute_duration() <= 50
         with pytest.raises(ValueError):
             chat.mute(-51)
 
+        # Regression test, this caused Rust panic previously
+        chat.mute(2**63 - 1)
+        assert chat.is_muted()
+        assert chat.get_mute_duration() == -1
+
     def test_delete_and_send_fails(self, ac1, chat1):
         chat1.delete()
         ac1._evtracker.wait_next_messages_changed()
@@ -440,6 +512,21 @@ class TestOfflineChat:
         assert not res.is_ask_verifygroup()
         assert res.contact_id == 10
 
+    def test_quote(self, chat1):
+        """Offline quoting test"""
+        msg = Message.new_empty(chat1.account, "text")
+        msg.set_text("Multi\nline\nmessage")
+        assert msg.quoted_text is None
+
+        # Prepare message to assign it a Message-Id.
+        # Messages without Message-Id cannot be quoted.
+        msg = chat1.prepare_message(msg)
+
+        reply_msg = Message.new_empty(chat1.account, "text")
+        reply_msg.set_text("reply")
+        reply_msg.quote = msg
+        assert reply_msg.quoted_text == "Multi\nline\nmessage"
+
     def test_group_chat_many_members_add_remove(self, ac1, lp):
         lp.sec("ac1: creating group chat with 10 other members")
         chat = ac1.create_group_chat(name="title1")
@@ -452,12 +539,12 @@ class TestOfflineChat:
 
         class InPlugin:
             @account_hookimpl
-            def ac_member_added(self, chat, contact):
-                in_list.append(("added", chat, contact))
+            def ac_member_added(self, chat, contact, actor):
+                in_list.append(("added", chat, contact, actor))
 
             @account_hookimpl
-            def ac_member_removed(self, chat, contact):
-                in_list.append(("removed", chat, contact))
+            def ac_member_removed(self, chat, contact, actor):
+                in_list.append(("removed", chat, contact, actor))
 
         ac1.add_account_plugin(InPlugin())
 
@@ -486,10 +573,11 @@ class TestOfflineChat:
 
         assert len(in_list) == 10
         chat_contacts = chat.get_contacts()
-        for in_cmd, in_chat, in_contact in in_list:
+        for in_cmd, in_chat, in_contact, in_actor in in_list:
             assert in_cmd == "added"
             assert in_chat == chat
             assert in_contact in chat_contacts
+            assert in_actor is None
             chat_contacts.remove(in_contact)
 
         assert chat_contacts[0].id == 1  # self contact
@@ -517,7 +605,7 @@ def test_basic_imap_api(acfactory, tmpdir):
 
     imap2 = ac2.direct_imap
 
-    ac2.direct_imap.idle_start()
+    imap2.idle_start()
     chat12.send_text("hello")
     ac2._evtracker.wait_next_incoming_message()
 
@@ -580,7 +668,7 @@ class TestOnlineAccount:
         except Exception:
             pass
 
-    def test_export_import_self_keys(self, acfactory, tmpdir):
+    def test_export_import_self_keys(self, acfactory, tmpdir, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         dir = tmpdir.mkdir("exportdir")
@@ -588,8 +676,17 @@ class TestOnlineAccount:
         assert len(export_files) == 2
         for x in export_files:
             assert x.startswith(dir.strpath)
+        key_id, = ac1._evtracker.get_info_regex_groups(r".*xporting.*KeyId\((.*)\).*")
         ac1._evtracker.consume_events()
+
+        lp.sec("exported keys (private and public)")
+        for name in os.listdir(dir.strpath):
+            lp.indent(dir.strpath + os.sep + name)
+        lp.sec("importing into existing account")
         ac2.import_self_keys(dir.strpath)
+        key_id2, = ac2._evtracker.get_info_regex_groups(
+            r".*stored.*KeyId\((.*)\).*", check_error=False)
+        assert key_id2 == key_id
 
     def test_one_account_send_bcc_setting(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
@@ -792,18 +889,12 @@ class TestOnlineAccount:
         assert msg_in.text == "message2"
         assert msg_in.is_forwarded()
 
-    def test_send_self_message_and_empty_folder(self, acfactory, lp):
+    def test_send_self_message(self, acfactory, lp):
         ac1 = acfactory.get_one_online_account(mvbox=True, move=True)
         lp.sec("ac1: create self chat")
         chat = ac1.get_self_contact().create_chat()
         chat.send_text("hello")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
-        ac1.empty_server_folders(inbox=True, mvbox=True)
-        ev1 = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
-        ev2 = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
-        boxes = [ev1.data2, ev2.data2]
-        boxes.remove("INBOX")
-        assert len(boxes) == 1 and boxes[0].endswith("DeltaChat")
 
     def test_send_and_receive_message_markseen(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -851,7 +942,13 @@ class TestOnlineAccount:
         lp.sec("mark messages as seen on ac2, wait for changes on ac1")
         ac2.direct_imap.idle_start()
         ac1.direct_imap.idle_start()
+
         ac2.mark_seen_messages([msg2, msg4])
+        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+        assert msg2.chat.id == msg4.chat.id
+        assert ev.data1 == msg2.chat.id
+        assert ev.data2 == 0
+
         ac2.direct_imap.idle_check(terminate=True)
         lp.step("1")
         for i in range(2):
@@ -872,6 +969,30 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
+    def test_reply_privately(self, acfactory):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        group1 = ac1.create_group_chat("group")
+        group1.add_contact(ac2)
+        group1.send_text("hello")
+
+        msg2 = ac2._evtracker.wait_next_messages_changed()
+        group2 = msg2.create_chat()
+        assert group2.get_name() == group1.get_name()
+
+        msg_reply = Message.new_empty(ac2, "text")
+        msg_reply.set_text("message reply")
+        msg_reply.quote = msg2
+
+        private_chat1 = ac1.create_chat(ac2)
+        private_chat2 = ac2.create_chat(ac1)
+        private_chat2.send_msg(msg_reply)
+
+        msg_reply1 = ac1._evtracker.wait_next_incoming_message()
+        assert msg_reply1.quoted_text == "hello"
+        assert not msg_reply1.chat.is_group()
+        assert msg_reply1.chat.id == private_chat1.id
+
     def test_mdn_asymetric(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts(move=True)
 
@@ -959,7 +1080,10 @@ class TestOnlineAccount:
         chat = acfactory.get_accepted_chat(ac1, ac2)
 
         lp.sec("sending multi-line non-unicode message from ac1 to ac2")
-        text1 = "hello\nworld"
+        text1 = (
+            "hello\nworld\nthis is a very long message that should be"
+            + " wrapped using format=flowed and unwrapped on the receiver"
+        )
         msg_out = chat.send_text(text1)
         assert not msg_out.is_encrypted()
 
@@ -977,7 +1101,68 @@ class TestOnlineAccount:
         assert msg_in.text == text2
         assert ac1.get_config("addr") in [x.addr for x in msg_in.chat.get_contacts()]
 
-    def test_reply_encrypted(self, acfactory, lp):
+    def test_no_draft_if_cant_send(self, acfactory):
+        """Tests that no quote can be set if the user can't send to this chat"""
+        ac1 = acfactory.get_one_online_account()
+        device_chat = ac1.get_device_chat()
+        msg = Message.new_empty(ac1, "text")
+        device_chat.set_draft(msg)
+
+        assert not device_chat.can_send()
+        assert device_chat.get_draft() is None
+
+    def test_prefer_encrypt(self, acfactory, lp):
+        """Test quorum rule for encryption preference in 1:1 and group chat."""
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+        ac1.set_config("e2ee_enabled", "0")
+        ac2.set_config("e2ee_enabled", "1")
+        ac3.set_config("e2ee_enabled", "0")
+
+        # Make sure we do not send a copy to ourselves. This is to
+        # test that we count own preference even when we are not in
+        # the recipient list.
+        ac1.set_config("bcc_self", "0")
+        ac2.set_config("bcc_self", "0")
+        ac3.set_config("bcc_self", "0")
+
+        acfactory.introduce_each_other([ac1, ac2, ac3])
+
+        lp.sec("ac1: sending message to ac2")
+        chat1 = ac1.create_chat(ac2)
+        msg1 = chat1.send_text("message1")
+        assert not msg1.is_encrypted()
+        ac2._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac2: sending message to ac1")
+        chat2 = ac2.create_chat(ac1)
+        msg2 = chat2.send_text("message2")
+        assert not msg2.is_encrypted()
+        ac1._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac1: sending message to group chat with ac2 and ac3")
+        group = ac1.create_group_chat("hello")
+        group.add_contact(ac2)
+        group.add_contact(ac3)
+        msg3 = group.send_text("message3")
+        assert not msg3.is_encrypted()
+        ac2._evtracker.wait_next_incoming_message()
+        ac3._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac3: start preferring encryption and inform ac1")
+        ac3.set_config("e2ee_enabled", "1")
+        chat3 = ac3.create_chat(ac1)
+        msg4 = chat3.send_text("message4")
+        # ac1 still does not prefer encryption
+        assert not msg4.is_encrypted()
+        ac1._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac1: sending another message to group chat with ac2 and ac3")
+        msg5 = group.send_text("message5")
+        # Majority prefers encryption now
+        assert msg5.is_encrypted()
+
+    def test_quote_encrypted(self, acfactory, lp):
+        """Test that replies to encrypted messages with quotes are encrypted."""
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         lp.sec("ac1: create chat with ac2")
@@ -1005,26 +1190,59 @@ class TestOnlineAccount:
         print("ac2: e2ee_enabled={}".format(ac2.get_config("e2ee_enabled")))
         ac1.set_config("e2ee_enabled", "0")
 
-        # Set unprepared and unencrypted draft to test that it is not
-        # taken into account when determining whether last message is
-        # encrypted.
-        msg_draft = Message.new_empty(ac1, "text")
-        msg_draft.set_text("message2 -- should be encrypted")
-        chat.set_draft(msg_draft)
+        for quoted_msg in msg1, msg3:
+            # Save the draft with a quote.
+            # It should be encrypted if quoted message is encrypted.
+            msg_draft = Message.new_empty(ac1, "text")
+            msg_draft.set_text("message reply")
+            msg_draft.quote = quoted_msg
+            chat.set_draft(msg_draft)
 
-        # Get the draft, prepare and send it.
-        msg_draft = chat.get_draft()
-        msg_out = chat.prepare_message(msg_draft)
-        chat.send_prepared(msg_out)
+            # Get the draft, prepare and send it.
+            msg_draft = chat.get_draft()
+            msg_out = chat.prepare_message(msg_draft)
+            chat.send_prepared(msg_out)
 
-        chat.set_draft(None)
-        assert chat.get_draft() is None
+            chat.set_draft(None)
+            assert chat.get_draft() is None
 
-        lp.sec("wait for ac2 to receive message")
-        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
-        msg_in = ac2.get_message_by_id(ev.data2)
-        assert msg_in.text == "message2 -- should be encrypted"
-        assert msg_in.is_encrypted()
+            msg_in = ac2._evtracker.wait_next_incoming_message()
+            assert msg_in.text == "message reply"
+            assert msg_in.quoted_text == quoted_msg.text
+            assert msg_in.is_encrypted() == quoted_msg.is_encrypted()
+
+    def test_quote_attachment(self, tmpdir, acfactory, lp):
+        """Test that replies with an attachment and a quote are received correctly."""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1 creates chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+
+        lp.sec("ac1 sends text message to ac2")
+        chat1.send_text("hi")
+
+        lp.sec("ac2 receives contact request from ac1")
+        received_message = ac2._evtracker.wait_next_messages_changed()
+        assert received_message.text == "hi"
+
+        basename = "attachment.txt"
+        p = os.path.join(tmpdir.strpath, basename)
+        with open(p, "w") as f:
+            f.write("data to send")
+
+        lp.sec("ac2 sends a reply to ac1")
+        chat2 = received_message.create_chat()
+        reply = Message.new_empty(ac2, "file")
+        reply.set_text("message reply")
+        reply.set_file(p)
+        reply.quote = received_message
+        chat2.send_msg(reply)
+
+        lp.sec("ac1 receives a reply from ac2")
+        received_reply = ac1._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "message reply"
+        assert received_reply.quoted_text == received_message.text
+        assert open(received_reply.filename).read() == "data to send"
 
     def test_saved_mime_on_received_message(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1237,7 +1455,7 @@ class TestOnlineAccount:
         ac1, ac2 = acfactory.get_two_online_accounts()
         lp.sec("ac1: create verified-group QR, ac2 scans and joins")
         chat1 = ac1.create_group_chat("hello", verified=True)
-        assert chat1.is_verified()
+        assert chat1.is_protected()
         qr = chat1.get_join_qr()
         lp.sec("ac2: start QR-code based join-group protocol")
         chat2 = ac2.qr_join_chat(qr)
@@ -1256,7 +1474,7 @@ class TestOnlineAccount:
         lp.sec("ac2: read message and check it's verified chat")
         msg = ac2._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
-        assert msg.chat.is_verified()
+        assert msg.chat.is_protected()
         assert msg.is_encrypted()
 
         lp.sec("ac2: send message and let ac1 read it")
@@ -1375,6 +1593,48 @@ class TestOnlineAccount:
         assert ev.action == "removed"
         assert ev.message.get_sender_contact().addr == ac1_addr
 
+    def test_system_group_msg_from_blocked_user(self, acfactory, lp):
+        """
+        Tests that a blocked user removes you from a group.
+        The message has to be fetched even though the user is blocked
+        to avoid inconsistent group state.
+        Also tests blocking in general.
+        """
+        lp.sec("Create a group chat with ac1 and ac2")
+        (ac1, ac2) = acfactory.get_two_online_accounts()
+        acfactory.introduce_each_other((ac1, ac2))
+        chat_on_ac1 = ac1.create_group_chat("title", contacts=[ac2])
+        chat_on_ac1.send_text("First group message")
+        chat_on_ac2 = ac2._evtracker.wait_next_incoming_message().chat
+
+        lp.sec("ac1 blocks ac2")
+        contact = ac1.create_contact(ac2)
+        contact.block()
+        assert contact.is_blocked()
+        ev = ac1._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        assert ev.data1 == contact.id
+
+        lp.sec("ac2 sends a message to ac1 that does not arrive because it is blocked")
+        ac2.create_chat(ac1).send_text("This will not arrive!")
+
+        lp.sec("ac2 sends a group message to ac1 that arrives")
+        # Groups would be hardly usable otherwise: If you have blocked some
+        # users, they write messages and you only see replies to them without context
+        chat_on_ac2.send_text("This will arrive")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "This will arrive"
+        message_texts = [m.text for m in chat_on_ac1.get_messages()]
+        assert len(message_texts) == 2
+        assert "First group message" in message_texts
+        assert "This will arrive" in message_texts
+
+        lp.sec("ac2 removes ac1 from their group")
+        assert ac1.get_self_contact() in chat_on_ac1.get_contacts()
+        assert contact.is_blocked()
+        chat_on_ac2.remove_contact(ac1)
+        ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED")
+        assert not ac1.get_self_contact() in chat_on_ac1.get_contacts()
+
     def test_set_get_group_image(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -1496,6 +1756,7 @@ class TestOnlineAccount:
         chat41 = ac4.create_chat(ac1)
         chat42 = ac4.create_chat(ac2)
         ac4.start_io()
+        ac4._evtracker.wait_all_initial_fetches()
 
         lp.sec("ac1: creating group chat with 2 other members")
         chat = ac1.create_group_chat("title", contacts=[ac2, ac3])
@@ -1542,6 +1803,277 @@ class TestOnlineAccount:
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac2.create_chat(ac4)
 
+    def test_immediate_autodelete(self, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account(mvbox=False, move=False, sentbox=False)
+
+        # "1" means delete immediately, while "0" means do not delete
+        ac2.set_config("delete_server_after", "1")
+
+        acfactory.wait_configure_and_start_io()
+
+        imap2 = ac2.direct_imap
+        imap2.idle_start()
+
+        lp.sec("ac1: create chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+        ac2.create_chat(ac1)
+
+        sent_msg = chat1.send_text("hello")
+        imap2.idle_check(terminate=False)
+
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        imap2.idle_check(terminate=True)
+        ac2._evtracker.get_info_contains("close/expunge succeeded")
+
+        assert len(imap2.get_all_messages()) == 0
+
+        # Mark deleted message as seen and check that read receipt arrives
+        msg.mark_seen()
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
+        assert ev.data1 == chat1.id
+        assert ev.data2 == sent_msg.id
+
+    def test_ephemeral_timer(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1: create chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+        chat2 = ac2.create_chat(ac1)
+
+        lp.sec("ac1: set ephemeral timer to 60")
+        chat1.set_ephemeral_timer(60)
+
+        lp.sec("ac1: check that ephemeral timer is set for chat")
+        assert chat1.get_ephemeral_timer() == 60
+        chat1_summary = chat1.get_summary()
+        assert chat1_summary["ephemeral_timer"] == {'Enabled': {'duration': 60}}
+
+        lp.sec("ac2: receive system message about ephemeral timer modification")
+        ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        system_message1 = ac2._evtracker.wait_next_incoming_message()
+        assert chat2.get_ephemeral_timer() == 60
+        assert system_message1.is_system_message()
+
+        # Disabled until markers are implemented
+        # assert "Ephemeral timer: 60\n" in system_message1.get_message_info()
+
+        lp.sec("ac2: send message to ac1")
+        sent_message = chat2.send_text("message")
+        assert sent_message.ephemeral_timer == 60
+        assert "Ephemeral timer: 60\n" in sent_message.get_message_info()
+
+        # Timer is started immediately for sent messages
+        assert sent_message.ephemeral_timestamp is not None
+        assert "Expires: " in sent_message.get_message_info()
+
+        lp.sec("ac1: waiting for message from ac2")
+        text_message = ac1._evtracker.wait_next_incoming_message()
+        assert text_message.text == "message"
+        assert text_message.ephemeral_timer == 60
+        assert "Ephemeral timer: 60\n" in text_message.get_message_info()
+
+        # Timer should not start until message is displayed
+        assert text_message.ephemeral_timestamp is None
+        assert "Expires: " not in text_message.get_message_info()
+        text_message.mark_seen()
+        text_message = ac1.get_message_by_id(text_message.id)
+        assert text_message.ephemeral_timestamp is not None
+        assert "Expires: " in text_message.get_message_info()
+
+        lp.sec("ac2: set ephemeral timer to 0")
+        chat2.set_ephemeral_timer(0)
+        ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+
+        lp.sec("ac1: receive system message about ephemeral timer modification")
+        ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        system_message2 = ac1._evtracker.wait_next_incoming_message()
+        assert system_message2.ephemeral_timer is None
+        assert "Ephemeral timer: " not in system_message2.get_message_info()
+        assert chat1.get_ephemeral_timer() == 0
+
+    def test_delete_multiple_messages(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("ac1: sending seven messages")
+        texts = ["first", "second", "third", "fourth", "fifth", "sixth", "seventh"]
+        for text in texts:
+            chat12.send_text(text)
+
+        lp.sec("ac2: waiting for all messages on the other side")
+        to_delete = []
+        for text in texts:
+            msg = ac2._evtracker.wait_next_incoming_message()
+            assert msg.text in texts
+            if text != "third":
+                to_delete.append(msg)
+
+        lp.sec("ac2: deleting all messages except third")
+        assert len(to_delete) == len(texts) - 1
+        ac2.delete_messages(to_delete)
+        for msg in to_delete:
+            ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+
+        ac2._evtracker.get_info_contains("close/expunge succeeded")
+
+        lp.sec("imap2: test that only one message is left")
+        imap2 = ac2.direct_imap
+
+        assert len(imap2.get_all_messages()) == 1
+
+    def test_configure_error_msgs(self, acfactory):
+        ac1, configdict = acfactory.get_online_config()
+        ac1.update_config(configdict)
+        ac1.set_config("mail_pw", "abc")  # Wrong mail pw
+        ac1.configure()
+        while True:
+            ev = ac1._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+            if ev.data1 == 0:
+                break
+        # Password is wrong so it definitely has to say something about "password"
+        assert "password" in ev.data2
+
+        ac2, configdict = acfactory.get_online_config()
+        ac2.update_config(configdict)
+        ac2.set_config("addr", "abc@def.invalid")  # mail server can't be reached
+        ac2.configure()
+        while True:
+            ev = ac2._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+            if ev.data1 == 0:
+                break
+        # Can't connect so it probably should say something about "internet"
+        # again, should not repeat itself
+        # If this fails then probably `e.msg.to_lowercase().contains("could not resolve")`
+        # in configure/mod.rs returned false because the error message was changed
+        # (i.e. did not contain "could not resolve" anymore)
+        assert (ev.data2.count("internet") + ev.data2.count("network")) == 1
+        # Should mention that it can't connect:
+        assert ev.data2.count("connect") == 1
+        # The users do not know what "configuration" is
+        assert "configuration" not in ev.data2.lower()
+
+    def test_name_changes(self, acfactory):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1.set_config("displayname", "Account 1")
+
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        contact = None
+
+        def update_name():
+            """Send a message from ac1 to ac2 to update the name"""
+            nonlocal contact
+            chat12.send_text("Hello")
+            msg = ac2._evtracker.wait_next_incoming_message()
+            contact = msg.get_sender_contact()
+            return contact.name
+
+        assert update_name() == "Account 1"
+
+        ac1.set_config("displayname", "Account 1 revision 2")
+        assert update_name() == "Account 1 revision 2"
+
+        # Explicitly rename contact on ac2 to "Renamed"
+        ac2.create_contact(contact, name="Renamed")
+        assert contact.name == "Renamed"
+        ev = ac2._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        assert ev.data1 == contact.id
+
+        # ac1 also renames itself into "Renamed"
+        assert update_name() == "Renamed"
+        ac1.set_config("displayname", "Renamed")
+        assert update_name() == "Renamed"
+
+        # Contact name was set to "Renamed" explicitly before,
+        # so it should not be changed.
+        ac1.set_config("displayname", "Renamed again")
+        updated_name = update_name()
+        if updated_name == "Renamed again":
+            # Known bug, mark as XFAIL
+            pytest.xfail("Contact was renamed after explicit rename")
+        else:
+            # No renames should happen after explicit rename
+            assert updated_name == "Renamed"
+
+    def test_group_quote(self, acfactory, lp):
+        """Test quoting in a group with a new member who have not seen the quoted message."""
+        ac1, ac2, ac3 = accounts = acfactory.get_many_online_accounts(3)
+        acfactory.introduce_each_other(accounts)
+        chat = ac1.create_group_chat(name="quote group")
+        chat.add_contact(ac2)
+
+        lp.sec("ac1: sending message")
+        out_msg = chat.send_text("hello")
+
+        lp.sec("ac2: receiving message")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        chat.add_contact(ac3)
+        ac2._evtracker.wait_next_incoming_message()
+        ac3._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac2: sending reply with a quote")
+        reply_msg = Message.new_empty(msg.chat.account, "text")
+        reply_msg.set_text("reply")
+        reply_msg.quote = msg
+        reply_msg = msg.chat.prepare_message(reply_msg)
+        assert reply_msg.quoted_text == "hello"
+        msg.chat.send_prepared(reply_msg)
+
+        lp.sec("ac3: receiving reply")
+        received_reply = ac3._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "reply"
+        assert received_reply.quoted_text == "hello"
+        # ac3 was not in the group and has not received quoted message
+        assert received_reply.quote is None
+
+        lp.sec("ac1: receiving reply")
+        received_reply = ac1._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "reply"
+        assert received_reply.quoted_text == "hello"
+        assert received_reply.quote.id == out_msg.id
+
+    @pytest.mark.parametrize("mvbox_move", [False, True])
+    def test_add_all_recipients_as_contacts(self, acfactory, lp, mvbox_move):
+        """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
+        This way, we can already offer them some email addresses they can write to.
+
+        Also test that existing emails are fetched during onboarding.
+
+        Lastly, tests that bcc_self messages moved to the mvbox are marked as read."""
+        ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure_and_start_io()
+
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("send out message with bcc to ourselves")
+        if mvbox_move:
+            ac1.direct_imap.select_config_folder("mvbox")
+        ac1.direct_imap.idle_start()
+        ac1.set_config("bcc_self", "1")
+        chat.send_text("message text")
+
+        # now wait until the bcc_self message arrives
+        # Also test that bcc_self messages moved to the mvbox are marked as read.
+        assert ac1.direct_imap.idle_wait_for_seen()
+
+        ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone.set_config("fetch_existing_msgs", "1")
+        ac1_clone._configtracker.wait_finish()
+        ac1_clone.start_io()
+
+        ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        ac2_addr = ac2.get_config("addr")
+        assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
+
+        msg = ac1_clone._evtracker.wait_next_messages_changed()
+        assert msg.text == "message text"
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):
@@ -1667,8 +2199,7 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "cannot login" in ev.data2.lower()
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_user(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -1676,8 +2207,7 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "cannot login" in ev.data2.lower()
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_domain(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -1685,5 +2215,4 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "could not connect" in ev.data2.lower()
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")

python/tox.ini

@@ -7,7 +7,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 

rust-toolchain

@@ -1 +1 @@
-1.43.1
+1.45.0

set_core_version.py

@@ -5,18 +5,27 @@ import sys
 import re
 import pathlib
 import subprocess
+from argparse import ArgumentParser
 
 rex = re.compile(r'version = "(\S+)"')
 
-def read_toml_version(relpath):
+
+def regex_matches(relpath, regex=rex):
     p = pathlib.Path(relpath)
     assert p.exists()
     for line in open(str(p)):
-        m = rex.match(line)
+        m = regex.match(line)
         if m is not None:
-            return m.group(1)
+            return m
+
+
+def read_toml_version(relpath):
+    res = regex_matches(relpath, rex)
+    if res is not None:
+        return res.group(1)
     raise ValueError("no version found in {}".format(relpath))
 
+
 def replace_toml_version(relpath, newversion):
     p = pathlib.Path(relpath)
     assert p.exists()
@@ -25,18 +34,28 @@ def replace_toml_version(relpath, newversion):
         for line in open(str(p)):
             m = rex.match(line)
             if m is not None:
+                print("{}: set version={}".format(relpath, newversion))
                 f.write('version = "{}"\n'.format(newversion))
             else:
                 f.write(line)
     os.rename(tmp_path, str(p))
 
-if __name__ == "__main__":
 
-    if len(sys.argv) < 2:
-        for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
+def main():
+    parser = ArgumentParser(prog="set_core_version")
+    parser.add_argument("newversion")
+
+    toml_list = ["Cargo.toml", "deltachat-ffi/Cargo.toml"]
+    try:
+        opts = parser.parse_args()
+    except SystemExit:
+        print()
+        for x in toml_list:
             print("{}: {}".format(x, read_toml_version(x)))
+        print()
         raise SystemExit("need argument: new version, example: 1.25.0")
-    newversion = sys.argv[1]
+
+    newversion = opts.newversion
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
 
@@ -55,7 +74,10 @@ if __name__ == "__main__":
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
 
+    print("running cargo check")
     subprocess.call(["cargo", "check"])
+
+    print("adding changes to git index")
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
@@ -63,3 +85,8 @@ if __name__ == "__main__":
     print("")
     print("   git tag {}".format(newversion))
     print("")
+
+
+if __name__ == "__main__":
+    main()
+

spec.md

@@ -32,7 +32,7 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
+by the [Protected Headers](https://tools.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages

src/accounts.rs

@@ -0,0 +1,531 @@
+use std::collections::BTreeMap;
+
+use async_std::fs;
+use async_std::path::PathBuf;
+use async_std::prelude::*;
+use async_std::sync::{Arc, RwLock};
+use uuid::Uuid;
+
+use anyhow::{ensure, Context as _};
+use serde::{Deserialize, Serialize};
+
+use crate::context::Context;
+use crate::error::Result;
+use crate::events::Event;
+
+/// Account manager, that can handle multiple accounts in a single place.
+#[derive(Debug, Clone)]
+pub struct Accounts {
+    dir: PathBuf,
+    config: Config,
+    accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
+}
+
+impl Accounts {
+    /// Loads or creates an accounts folder at the given `dir`.
+    pub async fn new(os_name: String, dir: PathBuf) -> Result<Self> {
+        if !dir.exists().await {
+            Accounts::create(os_name, &dir).await?;
+        }
+
+        Accounts::open(dir).await
+    }
+
+    /// Creates a new default structure, including a default account.
+    pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
+        fs::create_dir_all(dir)
+            .await
+            .context("failed to create folder")?;
+
+        // create default account
+        let config = Config::new(os_name.clone(), dir).await?;
+        let account_config = config.new_account(dir).await?;
+
+        Context::new(os_name, account_config.dbfile().into(), account_config.id)
+            .await
+            .context("failed to create default account")?;
+
+        Ok(())
+    }
+
+    /// Opens an existing accounts structure. Will error if the folder doesn't exist,
+    /// no account exists and no config exists.
+    pub async fn open(dir: PathBuf) -> Result<Self> {
+        ensure!(dir.exists().await, "directory does not exist");
+
+        let config_file = dir.join(CONFIG_NAME);
+        ensure!(config_file.exists().await, "accounts.toml does not exist");
+
+        let config = Config::from_file(config_file).await?;
+        let accounts = config.load_accounts().await?;
+
+        Ok(Self {
+            dir,
+            config,
+            accounts: Arc::new(RwLock::new(accounts)),
+        })
+    }
+
+    /// Get an account by its `id`:
+    pub async fn get_account(&self, id: u32) -> Option<Context> {
+        self.accounts.read().await.get(&id).cloned()
+    }
+
+    /// Get the currently selected account.
+    pub async fn get_selected_account(&self) -> Context {
+        let id = self.config.get_selected_account().await;
+        self.accounts
+            .read()
+            .await
+            .get(&id)
+            .cloned()
+            .expect("inconsistent state")
+    }
+
+    /// Select the given account.
+    pub async fn select_account(&self, id: u32) -> Result<()> {
+        self.config.select_account(id).await?;
+
+        Ok(())
+    }
+
+    /// Add a new account.
+    pub async fn add_account(&self) -> Result<u32> {
+        let os_name = self.config.os_name().await;
+        let account_config = self.config.new_account(&self.dir).await?;
+
+        let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
+        self.accounts.write().await.insert(account_config.id, ctx);
+
+        Ok(account_config.id)
+    }
+
+    /// Remove an account.
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        let ctx = self.accounts.write().await.remove(&id);
+        ensure!(ctx.is_some(), "no account with this id: {}", id);
+        let ctx = ctx.unwrap();
+        ctx.stop_io().await;
+        drop(ctx);
+
+        if let Some(cfg) = self.config.get_account(id).await {
+            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                .await
+                .context("failed to remove account data")?;
+        }
+        self.config.remove_account(id).await?;
+
+        Ok(())
+    }
+
+    /// Migrate an existing account into this structure.
+    pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
+        let blobdir = Context::derive_blobdir(&dbfile);
+
+        ensure!(
+            dbfile.exists().await,
+            "no database found: {}",
+            dbfile.display()
+        );
+        ensure!(
+            blobdir.exists().await,
+            "no blobdir found: {}",
+            blobdir.display()
+        );
+
+        let old_id = self.config.get_selected_account().await;
+
+        // create new account
+        let account_config = self.config.new_account(&self.dir).await?;
+
+        let new_dbfile = account_config.dbfile().into();
+        let new_blobdir = Context::derive_blobdir(&new_dbfile);
+
+        let res = {
+            fs::create_dir_all(&account_config.dir).await?;
+            fs::rename(&dbfile, &new_dbfile).await?;
+            fs::rename(&blobdir, &new_blobdir).await?;
+            Ok(())
+        };
+
+        match res {
+            Ok(_) => {
+                let ctx = Context::with_blobdir(
+                    self.config.os_name().await,
+                    new_dbfile,
+                    new_blobdir,
+                    account_config.id,
+                )
+                .await?;
+                self.accounts.write().await.insert(account_config.id, ctx);
+                Ok(account_config.id)
+            }
+            Err(err) => {
+                // remove temp account
+                fs::remove_dir_all(async_std::path::PathBuf::from(&account_config.dir))
+                    .await
+                    .context("failed to remove account data")?;
+
+                self.config.remove_account(account_config.id).await?;
+
+                // set selection back
+                self.select_account(old_id).await?;
+
+                Err(err)
+            }
+        }
+    }
+
+    /// Get a list of all account ids.
+    pub async fn get_all(&self) -> Vec<u32> {
+        self.accounts.read().await.keys().copied().collect()
+    }
+
+    /// Import a backup using a new account and selects it.
+    pub async fn import_account(&self, file: PathBuf) -> Result<u32> {
+        let old_id = self.config.get_selected_account().await;
+
+        let id = self.add_account().await?;
+        let ctx = self.get_account(id).await.expect("just added");
+
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, &file).await {
+            Ok(_) => Ok(id),
+            Err(err) => {
+                // remove temp account
+                self.remove_account(id).await?;
+                // set selection back
+                self.select_account(old_id).await?;
+                Err(err)
+            }
+        }
+    }
+
+    pub async fn start_io(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.start_io().await;
+        }
+    }
+
+    pub async fn stop_io(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.stop_io().await;
+        }
+    }
+
+    pub async fn maybe_network(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.maybe_network().await;
+        }
+    }
+
+    /// Unified event emitter.
+    pub async fn get_event_emitter(&self) -> EventEmitter {
+        let emitters: Vec<_> = self
+            .accounts
+            .read()
+            .await
+            .iter()
+            .map(|(_id, a)| a.get_event_emitter())
+            .collect();
+
+        EventEmitter(futures::stream::select_all(emitters))
+    }
+}
+
+#[derive(Debug)]
+pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
+
+impl EventEmitter {
+    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
+    pub fn recv_sync(&mut self) -> Option<Event> {
+        async_std::task::block_on(self.recv())
+    }
+
+    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
+    pub async fn recv(&mut self) -> Option<Event> {
+        self.0.next().await
+    }
+}
+
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
+    }
+}
+
+pub const CONFIG_NAME: &str = "accounts.toml";
+pub const DB_NAME: &str = "dc.db";
+
+#[derive(Debug, Clone)]
+pub struct Config {
+    file: PathBuf,
+    inner: Arc<RwLock<InnerConfig>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+struct InnerConfig {
+    pub os_name: String,
+    /// The currently selected account.
+    pub selected_account: u32,
+    pub next_id: u32,
+    pub accounts: Vec<AccountConfig>,
+}
+
+impl Config {
+    pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
+        let cfg = Config {
+            file: dir.join(CONFIG_NAME),
+            inner: Arc::new(RwLock::new(InnerConfig {
+                os_name,
+                accounts: Vec::new(),
+                selected_account: 0,
+                next_id: 1,
+            })),
+        };
+
+        cfg.sync().await?;
+
+        Ok(cfg)
+    }
+
+    pub async fn os_name(&self) -> String {
+        self.inner.read().await.os_name.clone()
+    }
+
+    /// Sync the inmemory representation to disk.
+    async fn sync(&self) -> Result<()> {
+        fs::write(
+            &self.file,
+            toml::to_string_pretty(&*self.inner.read().await)?,
+        )
+        .await
+        .context("failed to write config")
+    }
+
+    /// Read a configuration from the given file into memory.
+    pub async fn from_file(file: PathBuf) -> Result<Self> {
+        let bytes = fs::read(&file).await.context("failed to read file")?;
+        let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
+
+        Ok(Config {
+            file,
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
+
+    pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
+        let cfg = &*self.inner.read().await;
+        let mut accounts = BTreeMap::new();
+        for account_config in &cfg.accounts {
+            let ctx = Context::new(
+                cfg.os_name.clone(),
+                account_config.dbfile().into(),
+                account_config.id,
+            )
+            .await?;
+            accounts.insert(account_config.id, ctx);
+        }
+
+        Ok(accounts)
+    }
+
+    /// Create a new account in the given root directory.
+    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
+        let id = {
+            let inner = &mut self.inner.write().await;
+            let id = inner.next_id;
+            let uuid = Uuid::new_v4();
+            let target_dir = dir.join(uuid.to_simple_ref().to_string());
+
+            inner.accounts.push(AccountConfig {
+                id,
+                name: String::new(),
+                dir: target_dir.into(),
+                uuid,
+            });
+            inner.next_id += 1;
+            id
+        };
+
+        self.sync().await?;
+
+        self.select_account(id).await.expect("just added");
+        let cfg = self.get_account(id).await.expect("just added");
+        Ok(cfg)
+    }
+
+    /// Removes an existing acccount entirely.
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        {
+            let inner = &mut *self.inner.write().await;
+            if let Some(idx) = inner.accounts.iter().position(|e| e.id == id) {
+                // remove account from the configs
+                inner.accounts.remove(idx);
+            }
+            if inner.selected_account == id {
+                // reset selected account
+                inner.selected_account = inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+            }
+        }
+
+        self.sync().await
+    }
+
+    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
+        self.inner
+            .read()
+            .await
+            .accounts
+            .iter()
+            .find(|e| e.id == id)
+            .cloned()
+    }
+
+    pub async fn get_selected_account(&self) -> u32 {
+        self.inner.read().await.selected_account
+    }
+
+    pub async fn select_account(&self, id: u32) -> Result<()> {
+        {
+            let inner = &mut *self.inner.write().await;
+            ensure!(
+                inner.accounts.iter().any(|e| e.id == id),
+                "invalid account id: {}",
+                id
+            );
+
+            inner.selected_account = id;
+        }
+
+        self.sync().await?;
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct AccountConfig {
+    /// Unique id.
+    pub id: u32,
+    /// Display name
+    pub name: String,
+    /// Root directory for all data for this account.
+    pub dir: std::path::PathBuf,
+    pub uuid: Uuid,
+}
+
+impl AccountConfig {
+    /// Get the canoncial dbfile name for this configuration.
+    pub fn dbfile(&self) -> std::path::PathBuf {
+        self.dir.join(DB_NAME)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[async_std::test]
+    async fn test_account_new_open() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts1").into();
+
+        let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let accounts2 = Accounts::open(p).await.unwrap();
+
+        assert_eq!(accounts1.accounts.read().await.len(), 1);
+        assert_eq!(accounts1.config.get_selected_account().await, 1);
+
+        assert_eq!(accounts1.dir, accounts2.dir);
+        assert_eq!(
+            &*accounts1.config.inner.read().await,
+            &*accounts2.config.inner.read().await,
+        );
+        assert_eq!(
+            accounts1.accounts.read().await.len(),
+            accounts2.accounts.read().await.len()
+        );
+    }
+
+    #[async_std::test]
+    async fn test_account_new_add_remove() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, 2);
+        assert_eq!(accounts.config.get_selected_account().await, id);
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+
+        accounts.select_account(1).await.unwrap();
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        accounts.remove_account(1).await.unwrap();
+        assert_eq!(accounts.config.get_selected_account().await, 2);
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+    }
+
+    #[async_std::test]
+    async fn test_migrate_account() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        let extern_dbfile: PathBuf = dir.path().join("other").into();
+        let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
+            .await
+            .unwrap();
+        ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
+            .await
+            .unwrap();
+
+        drop(ctx);
+
+        accounts
+            .migrate_account(extern_dbfile.clone())
+            .await
+            .unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+        assert_eq!(accounts.config.get_selected_account().await, 2);
+
+        let ctx = accounts.get_selected_account().await;
+        assert_eq!(
+            "me@mail.com",
+            ctx.get_config(crate::config::Config::Addr).await.unwrap()
+        );
+    }
+
+    /// Tests that accounts are sorted by ID.
+    #[async_std::test]
+    async fn test_accounts_sorted() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+        for expected_id in 2..10 {
+            let id = accounts.add_account().await.unwrap();
+            assert_eq!(id, expected_id);
+        }
+
+        let ids = accounts.get_all().await;
+        for (i, expected_id) in (1..10).enumerate() {
+            assert_eq!(ids.get(i), Some(&expected_id));
+        }
+    }
+}

src/blob.rs

@@ -15,7 +15,7 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::error::Error;
-use crate::events::Event;
+use crate::events::EventType;
 use crate::message;
 
 /// Represents a file in the blob directory.
@@ -63,11 +63,17 @@ impl<'a> BlobObject<'a> {
                 blobname: name.clone(),
                 cause: err.into(),
             })?;
+
+        // workaround a bug in async-std
+        // (the executor does not handle blocking operation in Drop correctly,
+        // see https://github.com/async-rs/async-std/issues/900 )
+        let _ = file.flush().await;
+
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -151,11 +157,15 @@ impl<'a> BlobObject<'a> {
                 cause: err,
             });
         }
+
+        // workaround, see create() for details
+        let _ = dst_file.flush().await;
+
         let blob = BlobObject {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -168,6 +178,9 @@ impl<'a> BlobObject<'a> {
     /// subdirectory is used and [BlobObject::sanitise_name] does not
     /// modify the filename.
     ///
+    /// Paths into the blob directory may be either defined by an absolute path
+    /// or by the relative prefix `$BLOBDIR`.
+    ///
     /// # Errors
     ///
     /// This merely delegates to the [BlobObject::create_and_copy] and
@@ -179,6 +192,11 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'_>, BlobError> {
         if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
+        } else if src.as_ref().starts_with("$BLOBDIR/") {
+            BlobObject::from_name(
+                context,
+                src.as_ref().to_str().unwrap_or_default().to_string(),
+            )
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -677,7 +695,7 @@ mod tests {
 
         let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
         assert!(stem.contains("foo"));
-        assert!(!stem.contains("?"));
+        assert!(!stem.contains('?'));
         assert_eq!(ext, ".bar");
 
         let (stem, ext) = BlobObject::sanitise_name("no-extension");
@@ -690,10 +708,10 @@ mod tests {
         assert!(!stem.contains("ignored"));
         assert!(stem.contains("this"));
         assert!(stem.contains("forbidden"));
-        assert!(!stem.contains("/"));
-        assert!(!stem.contains("\\"));
-        assert!(!stem.contains(":"));
-        assert!(!stem.contains("*"));
-        assert!(!stem.contains("?"));
+        assert!(!stem.contains('/'));
+        assert!(!stem.contains('\\'));
+        assert!(!stem.contains(':'));
+        assert!(!stem.contains('*'));
+        assert!(!stem.contains('?'));
     }
 }

src/chatlist.rs

@@ -5,6 +5,7 @@ use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
+use crate::ephemeral::delete_expired_messages;
 use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
@@ -76,7 +77,7 @@ impl Chatlist {
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
     ///   and hides the device-chat,
-    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
+    ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -99,7 +100,7 @@ impl Chatlist {
 
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
         // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context).await {
+        if let Err(err) = delete_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
@@ -328,20 +329,30 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let mut ret = Lot::new();
-
         let (chat_id, lastmsg_id) = match self.ids.get(index) {
             Some(ids) => ids,
             None => {
+                let mut ret = Lot::new();
                 ret.text2 = Some("ErrBadChatlistIndex".to_string());
-                return ret;
+                return Lot::new();
             }
         };
 
+        Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
+    }
+
+    pub async fn get_summary2(
+        context: &Context,
+        chat_id: ChatId,
+        lastmsg_id: MsgId,
+        chat: Option<&Chat>,
+    ) -> Lot {
+        let mut ret = Lot::new();
+
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id).await {
+        } else if let Ok(chat) = Chat::load_from_db(context, chat_id).await {
             chat_loaded = chat;
             &chat_loaded
         } else {
@@ -350,10 +361,8 @@ impl Chatlist {
 
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id).await {
-            if lastmsg.from_id != DC_CONTACT_ID_SELF
-                && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
-            {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
+            if lastmsg.from_id != DC_CONTACT_ID_SELF && chat.typ == Chattype::Group {
                 lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
             }
 
@@ -429,13 +438,13 @@ mod tests {
     #[async_std::test]
     async fn test_try_load() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat")
+        let chat_id2 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "b chat")
             .await
             .unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat")
+        let chat_id3 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "c chat")
             .await
             .unwrap();
 
@@ -478,7 +487,7 @@ mod tests {
     async fn test_sort_self_talk_up_on_forward() {
         let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
-        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
@@ -535,7 +544,7 @@ mod tests {
     #[async_std::test]
     async fn test_get_summary_unwrap() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 

src/config.rs

@@ -8,7 +8,8 @@ use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::events::Event;
+use crate::events::EventType;
+use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
@@ -24,11 +25,13 @@ pub enum Config {
     MailUser,
     MailPw,
     MailPort,
+    MailSecurity,
     ImapCertificateChecks,
     SendServer,
     SendUser,
     SendPw,
     SendPort,
+    SendSecurity,
     SmtpCertificateChecks,
     ServerFlags,
 
@@ -66,6 +69,13 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
     MediaQuality,
 
+    /// If set to "1", on the first time `start_io()` is called after configuring,
+    /// the newest existing messages are fetched.
+    /// Existing recipients are added to the contact database regardless of this setting.
+    #[strum(props(default = "0"))]
+    // disabled for now, we'll set this back to "1" at some point
+    FetchExistingMsgs,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -117,6 +127,16 @@ pub enum Config {
 
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
+
+    Bot,
+
+    /// Whether we send a warning if the password is wrong (set to false when we send a warning
+    /// because we do not want to send a second warning)
+    #[strum(props(default = "0"))]
+    NotifyAboutWrongPw,
+
+    /// address to webrtc instance to use for videochats
+    WebrtcInstance,
 }
 
 impl Context {
@@ -218,12 +238,21 @@ impl Context {
             Config::DeleteDeviceAfter => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
                 // Force chatlist reload to delete old messages immediately.
-                self.emit_event(Event::MsgsChanged {
+                self.emit_event(EventType::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
                 });
                 ret
             }
+            Config::Displayname => {
+                let value = value.map(improve_single_line_input);
+                self.sql.set_raw_config(self, key, value.as_deref()).await
+            }
+            Config::DeleteServerAfter => {
+                let ret = self.sql.set_raw_config(self, key, value).await;
+                job::schedule_resync(self).await;
+                ret
+            }
             _ => self.sql.set_raw_config(self, key, value).await,
         }
     }

src/configure/auto_mozilla.rs

@@ -1,32 +1,31 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration
+use quick_xml::events::{BytesStart, Event};
+
+use std::io::BufRead;
+use std::str::FromStr;
 
-use crate::constants::*;
 use crate::context::Context;
 use crate::login_param::LoginParam;
+use crate::provider::{Protocol, Socket};
 
 use super::read_url::read_url;
-use super::Error;
+use super::{Error, ServerParams};
 
 #[derive(Debug)]
-struct MozAutoconfigure<'a> {
-    pub in_emailaddr: &'a str,
-    pub in_emaildomain: &'a str,
-    pub in_emaillocalpart: &'a str,
-    pub out: LoginParam,
-    pub out_imap_set: bool,
-    pub out_smtp_set: bool,
-    pub tag_server: MozServer,
-    pub tag_config: MozConfigTag,
+struct Server {
+    pub typ: String,
+    pub hostname: String,
+    pub port: u16,
+    pub sockettype: Socket,
+    pub username: String,
 }
 
-#[derive(Debug, PartialEq)]
-enum MozServer {
-    Undefined,
-    Imap,
-    Smtp,
+#[derive(Debug)]
+struct MozAutoconfigure {
+    pub incoming_servers: Vec<Server>,
+    pub outgoing_servers: Vec<Server>,
 }
 
 #[derive(Debug)]
@@ -38,10 +37,147 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
-    let mut reader = quick_xml::Reader::from_str(xml_raw);
-    reader.trim_text(true);
+impl Default for MozConfigTag {
+    fn default() -> Self {
+        Self::Undefined
+    }
+}
 
+impl FromStr for MozConfigTag {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.trim().to_lowercase().as_ref() {
+            "hostname" => Ok(MozConfigTag::Hostname),
+            "port" => Ok(MozConfigTag::Port),
+            "sockettype" => Ok(MozConfigTag::Sockettype),
+            "username" => Ok(MozConfigTag::Username),
+            _ => Err(()),
+        }
+    }
+}
+
+/// Parses a single IncomingServer or OutgoingServer section.
+fn parse_server<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+    server_event: &BytesStart,
+) -> Result<Option<Server>, quick_xml::Error> {
+    let end_tag = String::from_utf8_lossy(server_event.name())
+        .trim()
+        .to_lowercase();
+
+    let typ = server_event
+        .attributes()
+        .find(|attr| {
+            attr.as_ref()
+                .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "type")
+                .unwrap_or_default()
+        })
+        .map(|typ| {
+            typ.unwrap()
+                .unescape_and_decode_value(reader)
+                .unwrap_or_default()
+                .to_lowercase()
+        })
+        .unwrap_or_default();
+
+    let mut hostname = None;
+    let mut port = None;
+    let mut sockettype = Socket::Automatic;
+    let mut username = None;
+
+    let mut tag_config = MozConfigTag::Undefined;
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                tag_config = String::from_utf8_lossy(event.name())
+                    .parse()
+                    .unwrap_or_default();
+            }
+            Event::End(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+
+                if tag == end_tag {
+                    break;
+                }
+            }
+            Event::Text(ref event) => {
+                let val = event
+                    .unescape_and_decode(reader)
+                    .unwrap_or_default()
+                    .trim()
+                    .to_owned();
+
+                match tag_config {
+                    MozConfigTag::Hostname => hostname = Some(val),
+                    MozConfigTag::Port => port = Some(val.parse().unwrap_or_default()),
+                    MozConfigTag::Username => username = Some(val),
+                    MozConfigTag::Sockettype => {
+                        sockettype = match val.to_lowercase().as_ref() {
+                            "ssl" => Socket::SSL,
+                            "starttls" => Socket::STARTTLS,
+                            "plain" => Socket::Plain,
+                            _ => Socket::Automatic,
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            Event::Eof => break,
+            _ => (),
+        }
+    }
+
+    if let (Some(hostname), Some(port), Some(username)) = (hostname, port, username) {
+        Ok(Some(Server {
+            typ,
+            hostname,
+            port,
+            sockettype,
+            username,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+fn parse_xml_reader<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<MozAutoconfigure, quick_xml::Error> {
+    let mut incoming_servers = Vec::new();
+    let mut outgoing_servers = Vec::new();
+
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+
+                if tag == "incomingserver" {
+                    if let Some(incoming_server) = parse_server(reader, event)? {
+                        incoming_servers.push(incoming_server);
+                    }
+                } else if tag == "outgoingserver" {
+                    if let Some(outgoing_server) = parse_server(reader, event)? {
+                        outgoing_servers.push(outgoing_server);
+                    }
+                }
+            }
+            Event::Eof => break,
+            _ => (),
+        }
+        buf.clear();
+    }
+
+    Ok(MozAutoconfigure {
+        incoming_servers,
+        outgoing_servers,
+    })
+}
+
+/// Parses XML and fills in address and domain placeholders.
+fn parse_xml_with_address(in_emailaddr: &str, xml_raw: &str) -> Result<MozAutoconfigure, Error> {
     // Split address into local part and domain part.
     let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
     let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
@@ -49,59 +185,78 @@ fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
         _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
     };
 
-    let mut moz_ac = MozAutoconfigure {
-        in_emailaddr,
-        in_emaildomain,
-        in_emaillocalpart,
-        out: LoginParam::new(),
-        out_imap_set: false,
-        out_smtp_set: false,
-        tag_server: MozServer::Undefined,
-        tag_config: MozConfigTag::Undefined,
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
+    reader.trim_text(true);
+
+    let moz_ac = parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
+        position: reader.buffer_position(),
+        error,
+    })?;
+
+    let fill_placeholders = |val: &str| -> String {
+        val.replace("%EMAILADDRESS%", in_emailaddr)
+            .replace("%EMAILLOCALPART%", in_emaillocalpart)
+            .replace("%EMAILDOMAIN%", in_emaildomain)
     };
 
-    let mut buf = Vec::new();
-    loop {
-        let event = reader
-            .read_event(&mut buf)
-            .map_err(|error| Error::InvalidXml {
-                position: reader.buffer_position(),
-                error,
-            })?;
-
-        match event {
-            quick_xml::events::Event::Start(ref e) => {
-                moz_autoconfigure_starttag_cb(e, &mut moz_ac, &reader)
-            }
-            quick_xml::events::Event::End(ref e) => moz_autoconfigure_endtag_cb(e, &mut moz_ac),
-            quick_xml::events::Event::Text(ref e) => {
-                moz_autoconfigure_text_cb(e, &mut moz_ac, &reader)
-            }
-            quick_xml::events::Event::Eof => break,
-            _ => (),
+    let fill_server_placeholders = |server: Server| -> Server {
+        Server {
+            typ: server.typ,
+            hostname: fill_placeholders(&server.hostname),
+            port: server.port,
+            sockettype: server.sockettype,
+            username: fill_placeholders(&server.username),
         }
-        buf.clear();
-    }
+    };
 
-    if moz_ac.out.mail_server.is_empty()
-        || moz_ac.out.mail_port == 0
-        || moz_ac.out.send_server.is_empty()
-        || moz_ac.out.send_port == 0
-    {
-        Err(Error::IncompleteAutoconfig(moz_ac.out))
-    } else {
-        Ok(moz_ac.out)
-    }
+    Ok(MozAutoconfigure {
+        incoming_servers: moz_ac
+            .incoming_servers
+            .into_iter()
+            .map(fill_server_placeholders)
+            .collect(),
+        outgoing_servers: moz_ac
+            .outgoing_servers
+            .into_iter()
+            .map(fill_server_placeholders)
+            .collect(),
+    })
 }
 
-pub async fn moz_autoconfigure(
+/// Parses XML into `ServerParams` vector.
+fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerParams>, Error> {
+    let moz_ac = parse_xml_with_address(in_emailaddr, xml_raw)?;
+
+    let res = moz_ac
+        .incoming_servers
+        .into_iter()
+        .chain(moz_ac.outgoing_servers.into_iter())
+        .filter_map(|server| {
+            let protocol = match server.typ.as_ref() {
+                "imap" => Some(Protocol::IMAP),
+                "smtp" => Some(Protocol::SMTP),
+                _ => None,
+            };
+            Some(ServerParams {
+                protocol: protocol?,
+                socket: server.sockettype,
+                hostname: server.hostname,
+                port: server.port,
+                username: server.username,
+            })
+        })
+        .collect();
+    Ok(res)
+}
+
+pub(crate) async fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<Vec<ServerParams>, Error> {
     let xml_raw = read_url(context, url).await?;
 
-    let res = parse_xml(&param_in.addr, &xml_raw);
+    let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {
         warn!(
             context,
@@ -111,212 +266,62 @@ pub async fn moz_autoconfigure(
     res
 }
 
-fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
-    event: &BytesText,
-    moz_ac: &mut MozAutoconfigure,
-    reader: &quick_xml::Reader<B>,
-) {
-    let val = event.unescape_and_decode(reader).unwrap_or_default();
-
-    let addr = moz_ac.in_emailaddr;
-    let email_local = moz_ac.in_emaillocalpart;
-    let email_domain = moz_ac.in_emaildomain;
-
-    let val = val
-        .trim()
-        .replace("%EMAILADDRESS%", addr)
-        .replace("%EMAILLOCALPART%", email_local)
-        .replace("%EMAILDOMAIN%", email_domain);
-
-    match moz_ac.tag_server {
-        MozServer::Imap => match moz_ac.tag_config {
-            MozConfigTag::Hostname => moz_ac.out.mail_server = val,
-            MozConfigTag::Port => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
-            MozConfigTag::Username => moz_ac.out.mail_user = val,
-            MozConfigTag::Sockettype => {
-                let val_lower = val.to_lowercase();
-                if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                }
-                if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32
-                }
-                if val_lower == "plain" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
-                }
-            }
-            _ => {}
-        },
-        MozServer::Smtp => match moz_ac.tag_config {
-            MozConfigTag::Hostname => moz_ac.out.send_server = val,
-            MozConfigTag::Port => moz_ac.out.send_port = val.parse().unwrap_or_default(),
-            MozConfigTag::Username => moz_ac.out.send_user = val,
-            MozConfigTag::Sockettype => {
-                let val_lower = val.to_lowercase();
-                if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                }
-                if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32
-                }
-                if val_lower == "plain" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
-                }
-            }
-            _ => {}
-        },
-        MozServer::Undefined => {}
-    }
-}
-
-fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut MozAutoconfigure) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "incomingserver" {
-        if moz_ac.tag_server == MozServer::Imap {
-            moz_ac.out_imap_set = true;
-        }
-        moz_ac.tag_server = MozServer::Undefined;
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "outgoingserver" {
-        if moz_ac.tag_server == MozServer::Smtp {
-            moz_ac.out_smtp_set = true;
-        }
-        moz_ac.tag_server = MozServer::Undefined;
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else {
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    }
-}
-
-fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
-    event: &BytesStart,
-    moz_ac: &mut MozAutoconfigure,
-    reader: &quick_xml::Reader<B>,
-) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "incomingserver" {
-        moz_ac.tag_server = if let Some(typ) = event.attributes().find(|attr| {
-            attr.as_ref()
-                .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "type")
-                .unwrap_or_default()
-        }) {
-            let typ = typ
-                .unwrap()
-                .unescape_and_decode_value(reader)
-                .unwrap_or_default()
-                .to_lowercase();
-
-            if typ == "imap" && !moz_ac.out_imap_set {
-                MozServer::Imap
-            } else {
-                MozServer::Undefined
-            }
-        } else {
-            MozServer::Undefined
-        };
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "outgoingserver" {
-        moz_ac.tag_server = if !moz_ac.out_smtp_set {
-            MozServer::Smtp
-        } else {
-            MozServer::Undefined
-        };
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "hostname" {
-        moz_ac.tag_config = MozConfigTag::Hostname;
-    } else if tag == "port" {
-        moz_ac.tag_config = MozConfigTag::Port;
-    } else if tag == "sockettype" {
-        moz_ac.tag_config = MozConfigTag::Sockettype;
-    } else if tag == "username" {
-        moz_ac.tag_config = MozConfigTag::Username;
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]
     fn test_parse_outlook_autoconfig() {
-        // Copied from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11
-        let xml_raw =
-"<clientConfig version=\"1.1\">
-  <emailProvider id=\"outlook.com\">
-    <domain>hotmail.com</domain>
-    <domain>hotmail.co.uk</domain>
-    <domain>hotmail.co.jp</domain>
-    <domain>hotmail.com.br</domain>
-    <domain>hotmail.de</domain>
-    <domain>hotmail.fr</domain>
-    <domain>hotmail.it</domain>
-    <domain>hotmail.es</domain>
-    <domain>live.com</domain>
-    <domain>live.co.uk</domain>
-    <domain>live.co.jp</domain>
-    <domain>live.de</domain>
-    <domain>live.fr</domain>
-    <domain>live.it</domain>
-    <domain>live.jp</domain>
-    <domain>msn.com</domain>
-    <domain>outlook.com</domain>
-    <displayName>Outlook.com (Microsoft)</displayName>
-    <displayShortName>Outlook</displayShortName>
-    <incomingServer type=\"exchange\">
-      <hostname>outlook.office365.com</hostname>
-      <port>443</port>
-      <username>%EMAILADDRESS%</username>
-      <socketType>SSL</socketType>
-      <authentication>OAuth2</authentication>
-      <owaURL>https://outlook.office365.com/owa/</owaURL>
-      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
-      <useGlobalPreferredServer>true</useGlobalPreferredServer>
-    </incomingServer>
-    <incomingServer type=\"imap\">
-      <hostname>outlook.office365.com</hostname>
-      <port>993</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </incomingServer>
-    <incomingServer type=\"pop3\">
-      <hostname>outlook.office365.com</hostname>
-      <port>995</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-      <pop3>
-        <leaveMessagesOnServer>true</leaveMessagesOnServer>
-        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
-      </pop3>
-    </incomingServer>
-    <outgoingServer type=\"smtp\">
-      <hostname>smtp.office365.com</hostname>
-      <port>587</port>
-      <socketType>STARTTLS</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </outgoingServer>
-    <documentation url=\"http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app\">
-      <descr lang=\"en\">Set up an email app with Outlook.com</descr>
-    </documentation>
-  </emailProvider>
-  <webMail>
-    <loginPage url=\"https://www.outlook.com/\"/>
-    <loginPageInfo url=\"https://www.outlook.com/\">
-      <username>%EMAILADDRESS%</username>
-      <usernameField id=\"i0116\" name=\"login\"/>
-      <passwordField id=\"i0118\" name=\"passwd\"/>
-      <loginButton id=\"idSIButton9\" name=\"SI\"/>
-    </loginPageInfo>
-  </webMail>
-</clientConfig>";
-        let res = parse_xml("example@outlook.com", xml_raw).expect("XML parsing failed");
-        assert_eq!(res.mail_server, "outlook.office365.com");
-        assert_eq!(res.mail_port, 993);
-        assert_eq!(res.send_server, "smtp.office365.com");
-        assert_eq!(res.send_port, 587);
+        let xml_raw = include_str!("../../test-data/autoconfig/outlook.com.xml");
+        let res = parse_serverparams("example@outlook.com", xml_raw).expect("XML parsing failed");
+        assert_eq!(res[0].protocol, Protocol::IMAP);
+        assert_eq!(res[0].hostname, "outlook.office365.com");
+        assert_eq!(res[0].port, 993);
+        assert_eq!(res[1].protocol, Protocol::SMTP);
+        assert_eq!(res[1].hostname, "smtp.office365.com");
+        assert_eq!(res[1].port, 587);
+    }
+
+    #[test]
+    fn test_parse_lakenet_autoconfig() {
+        let xml_raw = include_str!("../../test-data/autoconfig/lakenet.ch.xml");
+        let res =
+            parse_xml_with_address("example@lakenet.ch", xml_raw).expect("XML parsing failed");
+
+        assert_eq!(res.incoming_servers.len(), 4);
+
+        assert_eq!(res.incoming_servers[0].typ, "imap");
+        assert_eq!(res.incoming_servers[0].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[0].port, 993);
+        assert_eq!(res.incoming_servers[0].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[0].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[1].typ, "imap");
+        assert_eq!(res.incoming_servers[1].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[1].port, 143);
+        assert_eq!(res.incoming_servers[1].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[1].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[2].typ, "pop3");
+        assert_eq!(res.incoming_servers[2].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[2].port, 995);
+        assert_eq!(res.incoming_servers[2].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[2].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[3].typ, "pop3");
+        assert_eq!(res.incoming_servers[3].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[3].port, 110);
+        assert_eq!(res.incoming_servers[3].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[3].username, "example@lakenet.ch");
+
+        assert_eq!(res.outgoing_servers.len(), 1);
+
+        assert_eq!(res.outgoing_servers[0].typ, "smtp");
+        assert_eq!(res.outgoing_servers[0].hostname, "mail.lakenet.ch");
+        assert_eq!(res.outgoing_servers[0].port, 587);
+        assert_eq!(res.outgoing_servers[0].sockettype, Socket::STARTTLS);
+        assert_eq!(res.outgoing_servers[0].username, "example@lakenet.ch");
     }
 }

src/configure/auto_outlook.rs

@@ -1,122 +1,194 @@
-//! Outlook's Autodiscover
+//! # Outlook's Autodiscover
+//!
+//! This module implements autoconfiguration via POX (Plain Old XML) interface to Autodiscover
+//! Service. Newer SOAP interface, introduced in Exchange 2010, is not used.
 
-use quick_xml::events::BytesEnd;
+use quick_xml::events::Event;
+
+use std::io::BufRead;
 
-use crate::constants::*;
 use crate::context::Context;
-use crate::login_param::LoginParam;
+use crate::provider::{Protocol, Socket};
 
 use super::read_url::read_url;
-use super::Error;
+use super::{Error, ServerParams};
 
-struct OutlookAutodiscover {
-    pub out: LoginParam,
-    pub out_imap_set: bool,
-    pub out_smtp_set: bool,
-    pub config_type: Option<String>,
-    pub config_server: String,
-    pub config_port: i32,
-    pub config_ssl: String,
-    pub config_redirecturl: Option<String>,
+/// Result of parsing a single `Protocol` tag.
+///
+/// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox
+#[derive(Debug)]
+struct ProtocolTag {
+    /// Server type, such as "IMAP", "SMTP" or "POP3".
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox
+    pub typ: String,
+
+    /// Server identifier, hostname or IP address for IMAP and SMTP.
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox
+    pub server: String,
+
+    /// Network port.
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox
+    pub port: u16,
+
+    /// Whether connection should be secure, "on" or "off", default is "on".
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox
+    pub ssl: bool,
 }
 
 enum ParsingResult {
-    LoginParam(LoginParam),
+    Protocols(Vec<ProtocolTag>),
+
+    /// XML redirect via `RedirectUrl` tag.
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
-    let mut outlk_ad = OutlookAutodiscover {
-        out: LoginParam::new(),
-        out_imap_set: false,
-        out_smtp_set: false,
-        config_type: None,
-        config_server: String::new(),
-        config_port: 0,
-        config_ssl: String::new(),
-        config_redirecturl: None,
-    };
-
-    let mut reader = quick_xml::Reader::from_str(&xml_raw);
-    reader.trim_text(true);
+/// Parses a single Protocol section.
+fn parse_protocol<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<Option<ProtocolTag>, quick_xml::Error> {
+    let mut protocol_type = None;
+    let mut protocol_server = None;
+    let mut protocol_port = None;
+    let mut protocol_ssl = true;
 
     let mut buf = Vec::new();
 
     let mut current_tag: Option<String> = None;
-
     loop {
-        let event = reader
-            .read_event(&mut buf)
-            .map_err(|error| Error::InvalidXml {
-                position: reader.buffer_position(),
-                error,
-            })?;
-
-        match event {
-            quick_xml::events::Event::Start(ref e) => {
-                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
-
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                current_tag = Some(String::from_utf8_lossy(event.name()).trim().to_lowercase());
+            }
+            Event::End(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
                 if tag == "protocol" {
-                    outlk_ad.config_type = None;
-                    outlk_ad.config_server = String::new();
-                    outlk_ad.config_port = 0;
-                    outlk_ad.config_ssl = String::new();
-                    outlk_ad.config_redirecturl = None;
-
+                    break;
+                }
+                if Some(tag) == current_tag {
                     current_tag = None;
-                } else {
-                    current_tag = Some(tag);
                 }
             }
-            quick_xml::events::Event::End(ref e) => {
-                outlk_autodiscover_endtag_cb(e, &mut outlk_ad);
-                current_tag = None;
-            }
-            quick_xml::events::Event::Text(ref e) => {
+            Event::Text(ref e) => {
                 let val = e.unescape_and_decode(&reader).unwrap_or_default();
 
                 if let Some(ref tag) = current_tag {
                     match tag.as_str() {
-                        "type" => {
-                            outlk_ad.config_type = Some(val.trim().to_lowercase().to_string())
+                        "type" => protocol_type = Some(val.trim().to_string()),
+                        "server" => protocol_server = Some(val.trim().to_string()),
+                        "port" => protocol_port = Some(val.trim().parse().unwrap_or_default()),
+                        "ssl" => {
+                            protocol_ssl = match val.trim() {
+                                "on" => true,
+                                "off" => false,
+                                _ => true,
+                            }
                         }
-                        "server" => outlk_ad.config_server = val.trim().to_string(),
-                        "port" => outlk_ad.config_port = val.trim().parse().unwrap_or_default(),
-                        "ssl" => outlk_ad.config_ssl = val.trim().to_string(),
-                        "redirecturl" => outlk_ad.config_redirecturl = Some(val.trim().to_string()),
                         _ => {}
                     };
                 }
             }
-            quick_xml::events::Event::Eof => break,
+            Event::Eof => break,
+            _ => {}
+        }
+    }
+
+    if let (Some(protocol_type), Some(protocol_server), Some(protocol_port)) =
+        (protocol_type, protocol_server, protocol_port)
+    {
+        Ok(Some(ProtocolTag {
+            typ: protocol_type,
+            server: protocol_server,
+            port: protocol_port,
+            ssl: protocol_ssl,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Parses `RedirectUrl` tag.
+fn parse_redirecturl<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<String, quick_xml::Error> {
+    let mut buf = Vec::new();
+    match reader.read_event(&mut buf)? {
+        Event::Text(ref e) => {
+            let val = e.unescape_and_decode(&reader).unwrap_or_default();
+            Ok(val.trim().to_string())
+        }
+        _ => Ok("".to_string()),
+    }
+}
+
+fn parse_xml_reader<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<ParsingResult, quick_xml::Error> {
+    let mut protocols = Vec::new();
+
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref e) => {
+                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
+
+                if tag == "protocol" {
+                    if let Some(protocol) = parse_protocol(reader)? {
+                        protocols.push(protocol);
+                    }
+                } else if tag == "redirecturl" {
+                    let redirecturl = parse_redirecturl(reader)?;
+                    return Ok(ParsingResult::RedirectUrl(redirecturl));
+                }
+            }
+            Event::Eof => break,
             _ => (),
         }
         buf.clear();
     }
 
-    // XML redirect via redirecturl
-    let res = if outlk_ad.config_redirecturl.is_none()
-        || outlk_ad.config_redirecturl.as_ref().unwrap().is_empty()
-    {
-        if outlk_ad.out.mail_server.is_empty()
-            || outlk_ad.out.mail_port == 0
-            || outlk_ad.out.send_server.is_empty()
-            || outlk_ad.out.send_port == 0
-        {
-            return Err(Error::IncompleteAutoconfig(outlk_ad.out));
-        }
-        ParsingResult::LoginParam(outlk_ad.out)
-    } else {
-        ParsingResult::RedirectUrl(outlk_ad.config_redirecturl.unwrap())
-    };
-    Ok(res)
+    Ok(ParsingResult::Protocols(protocols))
 }
 
-pub async fn outlk_autodiscover(
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    reader.trim_text(true);
+
+    parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
+        position: reader.buffer_position(),
+        error,
+    })
+}
+
+fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
+    protocols
+        .into_iter()
+        .filter_map(|protocol| {
+            Some(ServerParams {
+                protocol: match protocol.typ.to_lowercase().as_ref() {
+                    "imap" => Some(Protocol::IMAP),
+                    "smtp" => Some(Protocol::SMTP),
+                    _ => None,
+                }?,
+                socket: match protocol.ssl {
+                    true => Socket::Automatic,
+                    false => Socket::Plain,
+                },
+                hostname: protocol.server,
+                port: protocol.port,
+                username: String::new(),
+            })
+        })
+        .collect()
+}
+
+pub(crate) async fn outlk_autodiscover(
     context: &Context,
     url: &str,
-    _param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<Vec<ServerParams>, Error> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
@@ -127,47 +199,18 @@ pub async fn outlk_autodiscover(
         }
         match res? {
             ParsingResult::RedirectUrl(redirect_url) => url = redirect_url,
-            ParsingResult::LoginParam(login_param) => return Ok(login_param),
+            ParsingResult::Protocols(protocols) => {
+                return Ok(protocols_to_serverparams(protocols));
+            }
         }
     }
     Err(Error::RedirectionError)
 }
 
-fn outlk_autodiscover_endtag_cb(event: &BytesEnd, outlk_ad: &mut OutlookAutodiscover) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "protocol" {
-        if let Some(type_) = &outlk_ad.config_type {
-            let port = outlk_ad.config_port;
-            let ssl_on = outlk_ad.config_ssl == "on";
-            let ssl_off = outlk_ad.config_ssl == "off";
-            if type_ == "imap" && !outlk_ad.out_imap_set {
-                outlk_ad.out.mail_server =
-                    std::mem::replace(&mut outlk_ad.config_server, String::new());
-                outlk_ad.out.mail_port = port;
-                if ssl_on {
-                    outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                } else if ssl_off {
-                    outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
-                }
-                outlk_ad.out_imap_set = true
-            } else if type_ == "smtp" && !outlk_ad.out_smtp_set {
-                outlk_ad.out.send_server =
-                    std::mem::replace(&mut outlk_ad.config_server, String::new());
-                outlk_ad.out.send_port = outlk_ad.config_port;
-                if ssl_on {
-                    outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                } else if ssl_off {
-                    outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
-                }
-                outlk_ad.out_smtp_set = true
-            }
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]
@@ -184,16 +227,13 @@ mod tests {
     </Response>
   </Autodiscover>
  ").expect("XML is not parsed successfully");
-        match res {
-            ParsingResult::LoginParam(_lp) => {
-                panic!("redirecturl is not found");
-            }
-            ParsingResult::RedirectUrl(url) => {
-                assert_eq!(
-                    url,
-                    "https://mail.example.com/autodiscover/autodiscover.xml"
-                );
-            }
+        if let ParsingResult::RedirectUrl(url) = res {
+            assert_eq!(
+                url,
+                "https://mail.example.com/autodiscover/autodiscover.xml"
+            );
+        } else {
+            panic!("redirecturl is not found");
         }
     }
 
@@ -228,11 +268,16 @@ mod tests {
         .expect("XML is not parsed successfully");
 
         match res {
-            ParsingResult::LoginParam(lp) => {
-                assert_eq!(lp.mail_server, "example.com");
-                assert_eq!(lp.mail_port, 993);
-                assert_eq!(lp.send_server, "smtp.example.com");
-                assert_eq!(lp.send_port, 25);
+            ParsingResult::Protocols(protocols) => {
+                assert_eq!(protocols[0].typ, "IMAP");
+                assert_eq!(protocols[0].server, "example.com");
+                assert_eq!(protocols[0].port, 993);
+                assert_eq!(protocols[0].ssl, true);
+
+                assert_eq!(protocols[1].typ, "SMTP");
+                assert_eq!(protocols[1].server, "smtp.example.com");
+                assert_eq!(protocols[1].port, 25);
+                assert_eq!(protocols[1].ssl, false);
             }
             ParsingResult::RedirectUrl(_) => {
                 panic!("RedirectUrl is not expected");

src/configure/mod.rs

@@ -1,36 +1,47 @@
 //! Email accounts autoconfiguration process module
 
-#![forbid(clippy::indexing_slicing)]
-
 mod auto_mozilla;
 mod auto_outlook;
 mod read_url;
+mod server_params;
 
-use anyhow::{bail, ensure, format_err, Context as _, Result};
+use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
+use async_std::task;
+use itertools::Itertools;
+use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::config::Config;
-use crate::constants::*;
-use crate::context::Context;
 use crate::dc_tools::*;
 use crate::imap::Imap;
-use crate::login_param::{CertificateChecks, LoginParam};
+use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
 use crate::oauth2::*;
+use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
+use crate::stock::StockMessage;
 use crate::{chat, e2ee, provider};
+use crate::{constants::*, job};
+use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
+use server_params::{expand_param_vector, ServerParams};
 
 macro_rules! progress {
-    ($context:tt, $progress:expr) => {
+    ($context:tt, $progress:expr, $comment:expr) => {
         assert!(
             $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::Event::ConfigureProgress($progress));
+        $context.emit_event($crate::events::EventType::ConfigureProgress {
+            progress: $progress,
+            comment: $comment,
+        });
+    };
+    ($context:tt, $progress:expr) => {
+        progress!($context, $progress, None);
     };
 }
 
@@ -72,6 +83,7 @@ impl Context {
 
         let mut param = LoginParam::from_database(self, "").await;
         let success = configure(self, &mut param).await;
+        self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
         if let Some(provider) = provider::get_provider_info(&param.addr) {
             if let Some(config_defaults) = &provider.config_defaults {
@@ -102,12 +114,24 @@ impl Context {
 
         match success {
             Ok(_) => {
+                self.set_config(Config::NotifyAboutWrongPw, Some("1"))
+                    .await?;
                 progress!(self, 1000);
                 Ok(())
             }
             Err(err) => {
-                error!(self, "Configure Failed: {}", err);
-                progress!(self, 0);
+                progress!(
+                    self,
+                    0,
+                    Some(
+                        self.stock_string_repl_str(
+                            StockMessage::ConfigurationFailed,
+                            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                            format!("{:#}", err),
+                        )
+                        .await
+                    )
+                );
                 Err(err)
             }
         }
@@ -115,20 +139,40 @@ impl Context {
 }
 
 async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
-    let mut param_autoconfig: Option<LoginParam> = None;
-    let mut keep_flags = 0;
-
-    // Read login parameters from the database
     progress!(ctx, 1);
+
+    // Check basic settings.
     ensure!(!param.addr.is_empty(), "Please enter an email address.");
 
+    // Only check for IMAP password, SMTP password is an "advanced" setting.
+    ensure!(!param.imap.password.is_empty(), "Please enter a password.");
+    if param.smtp.password.is_empty() {
+        param.smtp.password = param.imap.password.clone()
+    }
+
+    // Normalize authentication flags.
+    let oauth2 = match param.server_flags & DC_LP_AUTH_FLAGS as i32 {
+        DC_LP_AUTH_OAUTH2 => true,
+        DC_LP_AUTH_NORMAL => false,
+        _ => false,
+    };
+    param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
+    param.server_flags |= if oauth2 {
+        DC_LP_AUTH_OAUTH2 as i32
+    } else {
+        DC_LP_AUTH_NORMAL as i32
+    };
+
+    let ctx2 = ctx.clone();
+    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
+
     // Step 1: Load the parameters and check email-address and password
 
-    if 0 != param.server_flags & DC_LP_AUTH_OAUTH2 {
+    if oauth2 {
         // the used oauth2 addr may differ, check this.
         // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
-        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.mail_pw)
+        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.imap.password)
             .await
             .and_then(|e| e.parse().ok())
         {
@@ -149,130 +193,145 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     // Step 2: Autoconfig
     progress!(ctx, 200);
 
-    // param.mail_user.is_empty() -- the user can enter a loginname which is used by autoconfig then
-    // param.send_pw.is_empty()   -- the password cannot be auto-configured and is no criterion for
-    //                               autoconfig or not
-    if param.mail_server.is_empty()
-        && param.mail_port == 0
-        && param.send_server.is_empty()
-        && param.send_port == 0
-        && param.send_user.is_empty()
-        && (param.server_flags & !DC_LP_AUTH_OAUTH2) == 0
+    let param_autoconfig;
+    if param.imap.server.is_empty()
+        && param.imap.port == 0
+        && param.imap.security == Socket::Automatic
+        && param.imap.user.is_empty()
+        && param.smtp.server.is_empty()
+        && param.smtp.port == 0
+        && param.smtp.security == Socket::Automatic
+        && param.smtp.user.is_empty()
     {
         // no advanced parameters entered by the user: query provider-database or do Autoconfig
-        keep_flags = param.server_flags & DC_LP_AUTH_OAUTH2;
-        if let Some(new_param) = get_offline_autoconfig(ctx, &param) {
-            // got parameters from our provider-database, skip Autoconfig, preserve the OAuth2 setting
-            param_autoconfig = Some(new_param);
-        }
 
-        if param_autoconfig.is_none() {
+        if let Some(servers) = get_offline_autoconfig(ctx, &param.addr) {
+            param_autoconfig = Some(servers);
+        } else {
             param_autoconfig =
                 get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
         }
+    } else {
+        param_autoconfig = None;
     }
 
-    // C.  Do we have any autoconfig result?
     progress!(ctx, 500);
-    if let Some(ref cfg) = param_autoconfig {
-        info!(ctx, "Got autoconfig: {}", &cfg);
-        if !cfg.mail_user.is_empty() {
-            param.mail_user = cfg.mail_user.clone();
-        }
-        // all other values are always NULL when entering autoconfig
-        param.mail_server = cfg.mail_server.clone();
-        param.mail_port = cfg.mail_port;
-        param.send_server = cfg.send_server.clone();
-        param.send_port = cfg.send_port;
-        param.send_user = cfg.send_user.clone();
-        param.server_flags = cfg.server_flags;
-        // although param_autoconfig's data are no longer needed from,
-        // it is used to later to prevent trying variations of port/server/logins
-    }
-    param.server_flags |= keep_flags;
 
-    // Step 3: Fill missing fields with defaults
-    if param.mail_server.is_empty() {
-        param.mail_server = format!("imap.{}", param_domain,)
+    let mut servers = param_autoconfig.unwrap_or_default();
+    if !servers
+        .iter()
+        .any(|server| server.protocol == Protocol::IMAP)
+    {
+        servers.push(ServerParams {
+            protocol: Protocol::IMAP,
+            hostname: param.imap.server.clone(),
+            port: param.imap.port,
+            socket: param.imap.security,
+            username: param.imap.user.clone(),
+        })
     }
-    if param.mail_port == 0 {
-        param.mail_port = if 0 != param.server_flags & (0x100 | 0x400) {
-            143
-        } else {
-            993
-        }
-    }
-    if param.mail_user.is_empty() {
-        param.mail_user = param.addr.clone();
-    }
-    if param.send_server.is_empty() && !param.mail_server.is_empty() {
-        param.send_server = param.mail_server.clone();
-        if param.send_server.starts_with("imap.") {
-            param.send_server = param.send_server.replacen("imap", "smtp", 1);
-        }
-    }
-    if param.send_port == 0 {
-        param.send_port = if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
-            587
-        } else if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
-            25
-        } else {
-            465
-        }
-    }
-    if param.send_user.is_empty() && !param.mail_user.is_empty() {
-        param.send_user = param.mail_user.clone();
-    }
-    if param.send_pw.is_empty() && !param.mail_pw.is_empty() {
-        param.send_pw = param.mail_pw.clone()
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_AUTH_FLAGS as i32) {
-        param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
-        param.server_flags |= DC_LP_AUTH_NORMAL as i32
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= if param.send_port == 143 {
-            DC_LP_IMAP_SOCKET_STARTTLS as i32
-        } else {
-            DC_LP_IMAP_SOCKET_SSL as i32
-        }
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= if param.send_port == 587 {
-            DC_LP_SMTP_SOCKET_STARTTLS as i32
-        } else if param.send_port == 25 {
-            DC_LP_SMTP_SOCKET_PLAIN as i32
-        } else {
-            DC_LP_SMTP_SOCKET_SSL as i32
-        }
+    if !servers
+        .iter()
+        .any(|server| server.protocol == Protocol::SMTP)
+    {
+        servers.push(ServerParams {
+            protocol: Protocol::SMTP,
+            hostname: param.smtp.server.clone(),
+            port: param.smtp.port,
+            socket: param.smtp.security,
+            username: param.smtp.user.clone(),
+        })
     }
+    let servers = expand_param_vector(servers, &param.addr, &param_domain);
 
-    // do we have a complete configuration?
-    ensure!(
-        !param.mail_server.is_empty()
-            && param.mail_port != 0
-            && !param.mail_user.is_empty()
-            && !param.mail_pw.is_empty()
-            && !param.send_server.is_empty()
-            && param.send_port != 0
-            && !param.send_user.is_empty()
-            && !param.send_pw.is_empty()
-            && param.server_flags != 0,
-        "Account settings incomplete."
-    );
+    progress!(ctx, 550);
+
+    // Spawn SMTP configuration task
+    let mut smtp = Smtp::new();
+
+    let context_smtp = ctx.clone();
+    let mut smtp_param = param.smtp.clone();
+    let smtp_addr = param.addr.clone();
+    let smtp_servers: Vec<ServerParams> = servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::SMTP)
+        .cloned()
+        .collect();
+
+    let smtp_config_task = task::spawn(async move {
+        let mut smtp_configured = false;
+        let mut errors = Vec::new();
+        for smtp_server in smtp_servers {
+            smtp_param.user = smtp_server.username.clone();
+            smtp_param.server = smtp_server.hostname.clone();
+            smtp_param.port = smtp_server.port;
+            smtp_param.security = smtp_server.socket;
+
+            match try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp)
+                .await
+            {
+                Ok(_) => {
+                    smtp_configured = true;
+                    break;
+                }
+                Err(e) => errors.push(e),
+            }
+        }
+
+        if smtp_configured {
+            Ok(smtp_param)
+        } else {
+            Err(errors)
+        }
+    });
 
     progress!(ctx, 600);
-    // try to connect to IMAP - if we did not got an autoconfig,
-    // do some further tries with different settings and username variations
+
+    // Configure IMAP
     let (_s, r) = async_std::sync::channel(1);
     let mut imap = Imap::new(r);
 
-    try_imap_connections(ctx, param, param_autoconfig.is_some(), &mut imap).await?;
-    progress!(ctx, 800);
+    let mut imap_configured = false;
+    let imap_servers: Vec<&ServerParams> = servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::IMAP)
+        .collect();
+    let imap_servers_count = imap_servers.len();
+    let mut errors = Vec::new();
+    for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
+        param.imap.user = imap_server.username.clone();
+        param.imap.server = imap_server.hostname.clone();
+        param.imap.port = imap_server.port;
+        param.imap.security = imap_server.socket;
+
+        match try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
+            Ok(_) => {
+                imap_configured = true;
+                break;
+            }
+            Err(e) => errors.push(e),
+        }
+        progress!(
+            ctx,
+            600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
+        );
+    }
+    if !imap_configured {
+        bail!(nicer_configuration_error(ctx, errors).await);
+    }
+
+    progress!(ctx, 850);
+
+    // Wait for SMTP configuration
+    match smtp_config_task.await {
+        Ok(smtp_param) => {
+            param.smtp = smtp_param;
+        }
+        Err(errors) => {
+            bail!(nicer_configuration_error(ctx, errors).await);
+        }
+    }
 
-    try_smtp_connections(ctx, param, param_autoconfig.is_some()).await?;
     progress!(ctx, 900);
 
     let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
@@ -298,7 +357,14 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     e2ee::ensure_secret_key_exists(ctx).await?;
     info!(ctx, "key generation completed");
 
+    job::add(
+        ctx,
+        job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
+    )
+    .await;
+
     progress!(ctx, 940);
+    update_device_chats_handle.await?;
 
     Ok(())
 }
@@ -341,8 +407,8 @@ impl AutoconfigSource {
             AutoconfigSource {
                 provider: AutoconfigProvider::Outlook,
                 url: format!(
-                    "https://{}{}/autodiscover/autodiscover.xml",
-                    "autodiscover.", domain
+                    "https://autodiscover.{}/autodiscover/autodiscover.xml",
+                    domain
                 ),
             },
             // always SSL for Thunderbird's database
@@ -353,10 +419,10 @@ impl AutoconfigSource {
         ]
     }
 
-    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<LoginParam> {
+    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<Vec<ServerParams>> {
         let params = match self.provider {
             AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
-            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url, &param).await?,
+            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url).await?,
         };
 
         Ok(params)
@@ -372,7 +438,7 @@ async fn get_autoconfig(
     param: &LoginParam,
     param_domain: &str,
     param_addr_urlencoded: &str,
-) -> Option<LoginParam> {
+) -> Option<Vec<ServerParams>> {
     let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
 
     let mut progress = 300;
@@ -388,212 +454,136 @@ async fn get_autoconfig(
     None
 }
 
-fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<LoginParam> {
+fn get_offline_autoconfig(context: &Context, addr: &str) -> Option<Vec<ServerParams>> {
     info!(
         context,
         "checking internal provider-info for offline autoconfig"
     );
 
-    if let Some(provider) = provider::get_provider_info(&param.addr) {
+    if let Some(provider) = provider::get_provider_info(&addr) {
         match provider.status {
             provider::Status::OK | provider::Status::PREPARATION => {
-                let imap = provider.get_imap_server();
-                let smtp = provider.get_smtp_server();
-                // clippy complains about these is_some()/unwrap() settings,
-                // however, rewriting the code to "if let" would make things less obvious,
-                // esp. if we allow more combinations of servers (pop, jmap).
-                // therefore, #[allow(clippy::unnecessary_unwrap)] is added above.
-                if let Some(imap) = imap {
-                    if let Some(smtp) = smtp {
-                        let mut p = LoginParam::new();
-                        p.addr = param.addr.clone();
-
-                        p.mail_server = imap.hostname.to_string();
-                        p.mail_user = imap.apply_username_pattern(param.addr.clone());
-                        p.mail_port = imap.port as i32;
-                        p.imap_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
-                        p.server_flags |= match imap.socket {
-                            provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
-                            provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
-                        };
-
-                        p.send_server = smtp.hostname.to_string();
-                        p.send_user = smtp.apply_username_pattern(param.addr.clone());
-                        p.send_port = smtp.port as i32;
-                        p.smtp_certificate_checks = CertificateChecks::AcceptInvalidCertificates;
-                        p.server_flags |= match smtp.socket {
-                            provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
-                            provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
-                        };
-
-                        info!(context, "offline autoconfig found: {}", p);
-                        return Some(p);
-                    }
+                if provider.server.is_empty() {
+                    info!(context, "offline autoconfig found, but no servers defined");
+                    None
+                } else {
+                    info!(context, "offline autoconfig found");
+                    let servers = provider
+                        .server
+                        .iter()
+                        .map(|s| ServerParams {
+                            protocol: s.protocol,
+                            socket: s.socket,
+                            hostname: s.hostname.to_string(),
+                            port: s.port,
+                            username: match s.username_pattern {
+                                UsernamePattern::EMAIL => addr.to_string(),
+                                UsernamePattern::EMAILLOCALPART => {
+                                    if let Some(at) = addr.find('@') {
+                                        addr.split_at(at).0.to_string()
+                                    } else {
+                                        addr.to_string()
+                                    }
+                                }
+                            },
+                        })
+                        .collect();
+                    Some(servers)
                 }
-                info!(context, "offline autoconfig found, but no servers defined");
-                return None;
             }
             provider::Status::BROKEN => {
                 info!(context, "offline autoconfig found, provider is broken");
-                return None;
+                None
             }
         }
-    }
-    info!(context, "no offline autoconfig found");
-    None
-}
-
-async fn try_imap_connections(
-    context: &Context,
-    param: &mut LoginParam,
-    was_autoconfig: bool,
-    imap: &mut Imap,
-) -> Result<()> {
-    // manually_set_param is used to check whether a particular setting was set manually by the user.
-    // If yes, we do not want to change it to avoid confusing error messages
-    // (you set port 443, but the app tells you it couldn't connect on port 993).
-    let manually_set_param = LoginParam::from_database(context, "").await;
-
-    // progress 650 and 660
-    if try_imap_connection(context, param, &manually_set_param, was_autoconfig, 0, imap)
-        .await
-        .is_ok()
-    {
-        return Ok(()); // we directly return here if it was autoconfig or the connection succeeded
-    }
-
-    progress!(context, 670);
-    // try_imap_connection() changed the flags and port. Change them back:
-    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-        param.server_flags |= DC_LP_IMAP_SOCKET_SSL;
-    }
-    if manually_set_param.mail_port == 0 {
-        param.mail_port = 993;
-    }
-    if let Some(at) = param.mail_user.find('@') {
-        param.mail_user = param.mail_user.split_at(at).0.to_string();
-    }
-    if let Some(at) = param.send_user.find('@') {
-        param.send_user = param.send_user.split_at(at).0.to_string();
-    }
-    // progress 680 and 690
-    try_imap_connection(context, param, &manually_set_param, was_autoconfig, 1, imap).await
-}
-
-async fn try_imap_connection(
-    context: &Context,
-    param: &mut LoginParam,
-    manually_set_param: &LoginParam,
-    was_autoconfig: bool,
-    variation: usize,
-    imap: &mut Imap,
-) -> Result<()> {
-    if try_imap_one_param(context, param, imap).await.is_ok() {
-        return Ok(());
-    }
-    if was_autoconfig {
-        return Ok(());
-    }
-
-    progress!(context, 650 + variation * 30);
-
-    if manually_set_param.server_flags & DC_LP_IMAP_SOCKET_FLAGS == 0 {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS);
-        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS;
-
-        if try_imap_one_param(context, &param, imap).await.is_ok() {
-            return Ok(());
-        }
-    }
-
-    progress!(context, 660 + variation * 30);
-
-    if manually_set_param.mail_port == 0 {
-        param.mail_port = 143;
-        try_imap_one_param(context, param, imap).await
     } else {
-        Err(format_err!("no more possible configs"))
+        info!(context, "no offline autoconfig found");
+        None
     }
 }
 
-async fn try_imap_one_param(context: &Context, param: &LoginParam, imap: &mut Imap) -> Result<()> {
-    let inf = format!(
-        "imap: {}@{}:{} flags=0x{:x} certificate_checks={}",
-        param.mail_user,
-        param.mail_server,
-        param.mail_port,
-        param.server_flags,
-        param.imap_certificate_checks
-    );
-    info!(context, "Trying: {}", inf);
-
-    if imap.connect(context, &param).await {
-        info!(context, "success: {}", inf);
-        return Ok(());
-    }
-
-    bail!("Could not connect: {}", inf);
-}
-
-async fn try_smtp_connections(
+async fn try_imap_one_param(
     context: &Context,
-    param: &mut LoginParam,
-    was_autoconfig: bool,
-) -> Result<()> {
-    // manually_set_param is used to check whether a particular setting was set manually by the user.
-    // If yes, we do not want to change it to avoid confusing error messages
-    // (you set port 443, but the app tells you it couldn't connect on port 993).
-    let manually_set_param = LoginParam::from_database(context, "").await;
-
-    let mut smtp = Smtp::new();
-    // try to connect to SMTP - if we did not got an autoconfig, the first try was SSL-465 and we do
-    // a second try with STARTTLS-587
-    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
-        return Ok(());
-    }
-    if was_autoconfig {
-        return Ok(());
-    }
-    progress!(context, 850);
-
-    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-    }
-    if manually_set_param.send_port == 0 {
-        param.send_port = 587;
-    }
-
-    if try_smtp_one_param(context, param, &mut smtp).await.is_ok() {
-        return Ok(());
-    }
-    progress!(context, 860);
-
-    if manually_set_param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32) == 0 {
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-    }
-    if manually_set_param.send_port == 0 {
-        param.send_port = 25;
-    }
-    try_smtp_one_param(context, param, &mut smtp).await
-}
-
-async fn try_smtp_one_param(context: &Context, param: &LoginParam, smtp: &mut Smtp) -> Result<()> {
+    param: &ServerLoginParam,
+    addr: &str,
+    oauth2: bool,
+    imap: &mut Imap,
+) -> Result<(), ConfigurationError> {
     let inf = format!(
-        "smtp: {}@{}:{} flags: 0x{:x}",
-        param.send_user, param.send_server, param.send_port, param.server_flags
+        "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
+        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = smtp.connect(context, &param).await {
-        bail!("could not connect: {}", err);
+    if let Err(err) = imap.connect(context, param, addr, oauth2).await {
+        info!(context, "failure: {}", err);
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
+    } else {
+        info!(context, "success: {}", inf);
+        Ok(())
+    }
+}
+
+async fn try_smtp_one_param(
+    context: &Context,
+    param: &ServerLoginParam,
+    addr: &str,
+    oauth2: bool,
+    smtp: &mut Smtp,
+) -> Result<(), ConfigurationError> {
+    let inf = format!(
+        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
+        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
+    );
+    info!(context, "Trying: {}", inf);
+
+    if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
+        info!(context, "failure: {}", err);
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
+    } else {
+        info!(context, "success: {}", inf);
+        smtp.disconnect().await;
+        Ok(())
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("Trying {config}…\nError: {msg}")]
+pub struct ConfigurationError {
+    config: String,
+    msg: String,
+}
+
+async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationError>) -> String {
+    let first_err = if let Some(f) = errors.first() {
+        f
+    } else {
+        // This means configuration failed but no errors have been captured. This should never
+        // happen, but if it does, the user will see classic "Error: no error".
+        return "no error".to_string();
+    };
+
+    if errors
+        .iter()
+        .all(|e| e.msg.to_lowercase().contains("could not resolve"))
+    {
+        return context
+            .stock_str(StockMessage::ErrorNoNetwork)
+            .await
+            .to_string();
     }
 
-    info!(context, "success: {}", inf);
-    smtp.disconnect().await;
-    Ok(())
+    if errors.iter().all(|e| e.msg == first_err.msg) {
+        return first_err.msg.to_string();
+    }
+
+    errors.iter().map(|e| e.to_string()).join("\n\n")
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -601,17 +591,14 @@ pub enum Error {
     #[error("Invalid email address: {0:?}")]
     InvalidEmailAddress(String),
 
-    #[error("XML error at position {position}")]
+    #[error("XML error at position {position}: {error}")]
     InvalidXml {
         position: usize,
         #[source]
         error: quick_xml::Error,
     },
 
-    #[error("Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
-    #[error("Failed to get URL")]
+    #[error("Failed to get URL: {0}")]
     ReadUrlError(#[from] self::read_url::Error),
 
     #[error("Number of redirection is exceeded")]
@@ -620,6 +607,7 @@ pub enum Error {
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
 
     use super::*;
     use crate::config::*;
@@ -643,14 +631,15 @@ mod tests {
     async fn test_get_offline_autoconfig() {
         let context = TestContext::new().await.ctx;
 
-        let mut params = LoginParam::new();
-        params.addr = "someone123@example.org".to_string();
-        assert!(get_offline_autoconfig(&context, &params).is_none());
+        let addr = "someone123@example.org";
+        assert!(get_offline_autoconfig(&context, addr).is_none());
 
-        let mut params = LoginParam::new();
-        params.addr = "someone123@nauta.cu".to_string();
-        let found_params = get_offline_autoconfig(&context, &params).unwrap();
-        assert_eq!(found_params.mail_server, "imap.nauta.cu".to_string());
-        assert_eq!(found_params.send_server, "smtp.nauta.cu".to_string());
+        let addr = "someone123@nauta.cu";
+        let found_params = get_offline_autoconfig(&context, addr).unwrap();
+        assert_eq!(found_params.len(), 2);
+        assert_eq!(found_params[0].protocol, Protocol::IMAP);
+        assert_eq!(found_params[0].hostname, "imap.nauta.cu".to_string());
+        assert_eq!(found_params[1].protocol, Protocol::SMTP);
+        assert_eq!(found_params[1].hostname, "smtp.nauta.cu".to_string());
     }
 }

src/configure/read_url.rs

@@ -12,7 +12,7 @@ pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
     match surf::get(url).recv_string().await {
         Ok(res) => Ok(res),
         Err(err) => {
-            info!(context, "Can\'t read URL {}", url);
+            info!(context, "Can\'t read URL {}: {}", url, err);
 
             Err(Error::GetError(err))
         }

src/configure/server_params.rs

@@ -0,0 +1,164 @@
+//! Variable server parameters lists
+
+use crate::provider::{Protocol, Socket};
+
+/// Set of variable parameters to try during configuration.
+///
+/// Can be loaded from offline provider database, online configuraiton
+/// or derived from user entered parameters.
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct ServerParams {
+    /// Protocol, such as IMAP or SMTP.
+    pub protocol: Protocol,
+
+    /// Server hostname, empty if unknown.
+    pub hostname: String,
+
+    /// Server port, zero if unknown.
+    pub port: u16,
+
+    /// Socket security, such as TLS or STARTTLS, Socket::Automatic if unknown.
+    pub socket: Socket,
+
+    /// Username, empty if unknown.
+    pub username: String,
+}
+
+impl ServerParams {
+    pub(crate) fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
+
+        if self.username.is_empty() {
+            self.username = addr.to_string();
+            res.push(self.clone());
+
+            if let Some(at) = addr.find('@') {
+                self.username = addr.split_at(at).0.to_string();
+                res.push(self);
+            }
+        } else {
+            res.push(self)
+        }
+        res
+    }
+
+    pub(crate) fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
+        if self.hostname.is_empty() {
+            self.hostname = param_domain.to_string();
+            res.push(self.clone());
+
+            self.hostname = match self.protocol {
+                Protocol::IMAP => "imap.".to_string() + param_domain,
+                Protocol::SMTP => "smtp.".to_string() + param_domain,
+            };
+            res.push(self.clone());
+
+            self.hostname = "mail.".to_string() + param_domain;
+            res.push(self);
+        } else {
+            res.push(self);
+        }
+        res
+    }
+
+    pub(crate) fn expand_ports(mut self) -> Vec<ServerParams> {
+        // Try to infer port from socket security.
+        if self.port == 0 {
+            self.port = match self.socket {
+                Socket::SSL => match self.protocol {
+                    Protocol::IMAP => 993,
+                    Protocol::SMTP => 465,
+                },
+                Socket::STARTTLS | Socket::Plain => match self.protocol {
+                    Protocol::IMAP => 143,
+                    Protocol::SMTP => 587,
+                },
+                Socket::Automatic => 0,
+            }
+        }
+
+        let mut res = Vec::new();
+        if self.port == 0 {
+            // Neither port nor security is set.
+            //
+            // Try common secure combinations.
+
+            // Try STARTTLS
+            self.socket = Socket::STARTTLS;
+            self.port = match self.protocol {
+                Protocol::IMAP => 143,
+                Protocol::SMTP => 587,
+            };
+            res.push(self.clone());
+
+            // Try TLS
+            self.socket = Socket::SSL;
+            self.port = match self.protocol {
+                Protocol::IMAP => 993,
+                Protocol::SMTP => 465,
+            };
+            res.push(self);
+        } else if self.socket == Socket::Automatic {
+            // Try TLS over user-provided port.
+            self.socket = Socket::SSL;
+            res.push(self.clone());
+
+            // Try STARTTLS over user-provided port.
+            self.socket = Socket::STARTTLS;
+            res.push(self);
+        } else {
+            res.push(self);
+        }
+        res
+    }
+}
+
+/// Expands vector of `ServerParams`, replacing placeholders with
+/// variants to try.
+pub(crate) fn expand_param_vector(
+    v: Vec<ServerParams>,
+    addr: &str,
+    domain: &str,
+) -> Vec<ServerParams> {
+    v.into_iter()
+        // The order of expansion is important: ports are expanded the
+        // last, so they are changed the first. Username is only
+        // changed if default value (address with domain) didn't work
+        // for all available hosts and ports.
+        .flat_map(|params| params.expand_usernames(addr).into_iter())
+        .flat_map(|params| params.expand_hostnames(domain).into_iter())
+        .flat_map(|params| params.expand_ports().into_iter())
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_expand_param_vector() {
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::IMAP,
+                hostname: "example.net".to_string(),
+                port: 0,
+                socket: Socket::SSL,
+                username: "foobar".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+
+        assert_eq!(
+            v,
+            vec![ServerParams {
+                protocol: Protocol::IMAP,
+                hostname: "example.net".to_string(),
+                port: 993,
+                socket: Socket::SSL,
+                username: "foobar".to_string(),
+            }],
+        );
+    }
+}

src/constants.rs

@@ -1,20 +1,9 @@
 //! # Constants
-#![allow(dead_code)]
-
 use deltachat_derive::*;
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
-lazy_static! {
-    pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
-}
-
-// some defaults
-const DC_E2EE_DEFAULT_ENABLED: i32 = 1;
-const DC_INBOX_WATCH_DEFAULT: i32 = 1;
-const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
-const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
-const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
+pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION").to_string());
 
 #[derive(
     Debug,
@@ -84,6 +73,20 @@ impl Default for KeyGenType {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(i8)]
+pub enum VideochatType {
+    Unknown = 0,
+    BasicWebrtc = 1,
+    Jitsi = 2,
+}
+
+impl Default for VideochatType {
+    fn default() -> Self {
+        VideochatType::Unknown
+    }
+}
+
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;
@@ -103,14 +106,16 @@ pub const DC_GCL_ADD_SELF: usize = 0x02;
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 
+// warn about an outdated app after a given number of days.
+// as we use the "provider-db generation date" as reference (that might not be updated very often)
+// and as not all system get speedy updates,
+// do not use too small value that will annoy users checking for nonexistant updates.
+pub const DC_OUTDATED_WARNING_DAYS: i64 = 365;
+
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
-pub(crate) const DC_CHAT_ID_DEADDROP: u32 = 1;
+pub const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
 pub const DC_CHAT_ID_TRASH: u32 = 3;
-/// a message is just in creation but not yet assigned to a chat (eg. we may need the message ID to set up blobs; this avoids unready message to be sent and shown)
-const DC_CHAT_ID_MSGS_IN_CREATION: u32 = 4;
-/// virtual chat showing all messages flagged with msgs.starred=2
-pub const DC_CHAT_ID_STARRED: u32 = 5;
 /// only an indicator in a chatlist
 pub const DC_CHAT_ID_ARCHIVED_LINK: u32 = 6;
 /// only an indicator in a chatlist
@@ -138,7 +143,6 @@ pub enum Chattype {
     Undefined = 0,
     Single = 100,
     Group = 120,
-    VerifiedGroup = 130,
 }
 
 impl Default for Chattype {
@@ -152,9 +156,9 @@ pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
 /// approx. max. length returned by dc_msg_get_text()
-const DC_MAX_GET_TEXT_LEN: usize = 30000;
+pub const DC_MAX_GET_TEXT_LEN: usize = 30000;
 /// approx. max. length returned by dc_get_msg_info()
-const DC_MAX_GET_INFO_LEN: usize = 100_000;
+pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -185,44 +189,11 @@ pub const DC_LP_AUTH_OAUTH2: i32 = 0x2;
 /// If this flag is set, automatic configuration is skipped.
 pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
 
-/// Connect to IMAP via STARTTLS.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_STARTTLS: i32 = 0x100;
-
-/// Connect to IMAP via SSL.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_SSL: i32 = 0x200;
-
-/// Connect to IMAP unencrypted, this should not be used.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_PLAIN: i32 = 0x400;
-
-/// Connect to SMTP via STARTTLS.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_STARTTLS: usize = 0x10000;
-
-/// Connect to SMTP via SSL.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
-
-/// Connect to SMTP unencrypted, this should not be used.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
-
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
-/// if none of these flags are set, the default is chosen
-pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
-    DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN;
-/// if none of these flags are set, the default is chosen
-pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
-    DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN;
 
-// QR code scanning (view from Bob, the joiner)
-pub const DC_VC_AUTH_REQUIRED: i32 = 2;
-pub const DC_VC_CONTACT_CONFIRM: i32 = 6;
-pub const DC_BOB_ERROR: i32 = 0;
-pub const DC_BOB_SUCCESS: i32 = 1;
+/// How many existing messages shall be fetched after configuration.
+pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
 
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
@@ -234,6 +205,11 @@ pub const WORSE_IMAGE_SIZE: u32 = 640;
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
+// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is splitted to chunks.
+// this does not affect MIME'e `To:` header.
+// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
+pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
+
 #[derive(
     Debug,
     Display,
@@ -296,6 +272,9 @@ pub enum Viewtype {
     /// The file is set via dc_msg_set_file()
     /// and retrieved via dc_msg_get_file().
     File = 60,
+
+    /// Message is an invitation to a videochat.
+    VideochatInvitation = 70,
 }
 
 impl Default for Viewtype {
@@ -314,64 +293,6 @@ mod tests {
     }
 }
 
-// These constants are used as events
-// reported to the callback given to dc_context_new().
-// If you do not want to handle an event, it is always safe to return 0,
-// so there is no need to add a "case" for every event.
-
-const DC_EVENT_FILE_COPIED: usize = 2055; // deprecated;
-const DC_EVENT_IS_OFFLINE: usize = 2081; // deprecated;
-const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
-const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
-const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
-
-// TODO: Strings need some doumentation about used placeholders.
-// These constants are used to set stock translation strings
-
-const DC_STR_NOMESSAGES: usize = 1;
-const DC_STR_SELF: usize = 2;
-const DC_STR_DRAFT: usize = 3;
-const DC_STR_VOICEMESSAGE: usize = 7;
-const DC_STR_DEADDROP: usize = 8;
-const DC_STR_IMAGE: usize = 9;
-const DC_STR_VIDEO: usize = 10;
-const DC_STR_AUDIO: usize = 11;
-const DC_STR_FILE: usize = 12;
-const DC_STR_STATUSLINE: usize = 13;
-const DC_STR_NEWGROUPDRAFT: usize = 14;
-const DC_STR_MSGGRPNAME: usize = 15;
-const DC_STR_MSGGRPIMGCHANGED: usize = 16;
-const DC_STR_MSGADDMEMBER: usize = 17;
-const DC_STR_MSGDELMEMBER: usize = 18;
-const DC_STR_MSGGROUPLEFT: usize = 19;
-const DC_STR_GIF: usize = 23;
-const DC_STR_ENCRYPTEDMSG: usize = 24;
-const DC_STR_E2E_AVAILABLE: usize = 25;
-const DC_STR_ENCR_TRANSP: usize = 27;
-const DC_STR_ENCR_NONE: usize = 28;
-const DC_STR_CANTDECRYPT_MSG_BODY: usize = 29;
-const DC_STR_FINGERPRINTS: usize = 30;
-const DC_STR_READRCPT: usize = 31;
-const DC_STR_READRCPT_MAILBODY: usize = 32;
-const DC_STR_MSGGRPIMGDELETED: usize = 33;
-const DC_STR_E2E_PREFERRED: usize = 34;
-const DC_STR_CONTACT_VERIFIED: usize = 35;
-const DC_STR_CONTACT_NOT_VERIFIED: usize = 36;
-const DC_STR_CONTACT_SETUP_CHANGED: usize = 37;
-const DC_STR_ARCHIVEDCHATS: usize = 40;
-const DC_STR_STARREDMSGS: usize = 41;
-const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
-const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
-const DC_STR_CANNOT_LOGIN: usize = 60;
-const DC_STR_SERVER_RESPONSE: usize = 61;
-const DC_STR_MSGACTIONBYUSER: usize = 62;
-const DC_STR_MSGACTIONBYME: usize = 63;
-const DC_STR_MSGLOCATIONENABLED: usize = 64;
-const DC_STR_MSGLOCATIONDISABLED: usize = 65;
-const DC_STR_LOCATION: usize = 66;
-const DC_STR_STICKER: usize = 67;
-const DC_STR_COUNT: usize = 67;
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]

src/contact.rs

@@ -1,11 +1,9 @@
 //! Contacts module
 
-#![forbid(clippy::indexing_slicing)]
-
 use async_std::path::PathBuf;
 use deltachat_derive::*;
 use itertools::Itertools;
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use regex::Regex;
 
 use crate::aheader::EncryptPreference;
@@ -15,13 +13,14 @@ use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::message::{MessageState, MsgId};
+use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::*;
 use crate::peerstate::*;
+use crate::provider::Socket;
 use crate::stock::StockMessage;
 
 /// An object representing a single contact in memory.
@@ -237,6 +236,7 @@ impl Contact {
         name: impl AsRef<str>,
         addr: impl AsRef<str>,
     ) -> Result<u32> {
+        let name = improve_single_line_input(name);
         ensure!(
             !addr.as_ref().is_empty(),
             "Cannot create contact with empty address"
@@ -247,13 +247,12 @@ impl Contact {
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, name, addr, Origin::ManuallyCreated).await?;
         let blocked = Contact::is_blocked_load(context, contact_id).await;
-        context.emit_event(Event::ContactsChanged(
-            if sth_modified == Modifier::Created {
-                Some(contact_id)
-            } else {
-                None
-            },
-        ));
+        match sth_modified {
+            Modifier::None => {}
+            Modifier::Modified | Modifier::Created => {
+                context.emit_event(EventType::ContactsChanged(Some(contact_id)))
+            }
+        }
         if blocked {
             Contact::unblock(context, contact_id).await;
         }
@@ -261,10 +260,9 @@ impl Contact {
         Ok(contact_id)
     }
 
-    /// Mark all messages sent by the given contact
-    /// as *noticed*.  See also dc_marknoticed_chat() and dc_markseen_msgs()
-    ///
-    /// Calling this function usually results in the event `#DC_EVENT_MSGS_CHANGED`.
+    /// Mark messages from a contact as noticed.
+    /// The contact is expected to belong to the deaddrop,
+    /// therefore, DC_EVENT_MSGS_NOTICED(DC_CHAT_ID_DEADDROP) is emitted.
     pub async fn mark_noticed(context: &Context, id: u32) {
         if context
             .sql
@@ -275,10 +273,7 @@ impl Contact {
             .await
             .is_ok()
         {
-            context.emit_event(Event::MsgsChanged {
-                chat_id: ChatId::new(0),
-                msg_id: MsgId::new(0),
-            });
+            context.emit_event(EventType::MsgsNoticed(ChatId::new(DC_CHAT_ID_DEADDROP)));
         }
     }
 
@@ -457,10 +452,20 @@ impl Contact {
                 if update_name {
                     // Update the contact name also if it is used as a group name.
                     // This is one of the few duplicated data, however, getting the chat list is easier this way.
-                    context.sql.execute(
-                    "UPDATE chats SET name=? WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    paramsv![new_name, Chattype::Single, row_id]
-                ).await.ok();
+                    let chat_id = context.sql.query_get_value::<i32>(
+                        context,
+                        "SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
+                        paramsv![Chattype::Single, row_id]
+                    ).await;
+                    if let Some(chat_id) = chat_id {
+                        match context.sql.execute("UPDATE chats SET name=? WHERE id=? AND name!=?1", paramsv![new_name, chat_id]).await {
+                            Err(err) => warn!(context, "Can't update chat name: {}", err),
+                            Ok(count) => if count > 0 {
+                                // Chat name updated
+                                context.emit_event(EventType::ChatModified(ChatId::new(chat_id as u32)));
+                            }
+                        }
+                    }
                 }
                 sth_modified = Modifier::Modified;
             }
@@ -535,7 +540,7 @@ impl Contact {
             }
         }
         if modify_cnt > 0 {
-            context.emit_event(Event::ContactsChanged(None));
+            context.emit_event(EventType::ContactsChanged(None));
         }
 
         Ok(modify_cnt)
@@ -680,7 +685,7 @@ impl Contact {
         let mut ret = String::new();
 
         if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            let peerstate = Peerstate::from_addr(context, &contact.addr).await;
+            let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
             let loginparam = LoginParam::from_database(context, "configured_").await;
 
             if peerstate.is_some()
@@ -730,8 +735,8 @@ impl Contact {
                     );
                     cat_fingerprint(&mut ret, &loginparam.addr, &fingerprint_self, "");
                 }
-            } else if 0 == loginparam.server_flags & DC_LP_IMAP_SOCKET_PLAIN as i32
-                && 0 == loginparam.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32
+            } else if loginparam.imap.security != Socket::Plain
+                && loginparam.smtp.security != Socket::Plain
             {
                 ret += &context.stock_str(StockMessage::EncrTransp).await;
             } else {
@@ -786,7 +791,7 @@ impl Contact {
                 .await
             {
                 Ok(_) => {
-                    context.emit_event(Event::ContactsChanged(None));
+                    context.emit_event(EventType::ContactsChanged(None));
                     return Ok(());
                 }
                 Err(err) => {
@@ -941,7 +946,17 @@ impl Contact {
             }
         }
 
-        let peerstate = Peerstate::from_addr(context, &self.addr).await;
+        let peerstate = match Peerstate::from_addr(context, &self.addr).await {
+            Ok(peerstate) => peerstate,
+            Err(err) => {
+                warn!(
+                    context,
+                    "Failed to load peerstate for address {}: {}", self.addr, err
+                );
+                return VerifiedStatus::Unverified;
+            }
+        };
+
         if let Some(ps) = peerstate {
             if ps.verified_key.is_some() {
                 return VerifiedStatus::BidirectVerified;
@@ -1038,9 +1053,7 @@ pub fn addr_normalize(addr: &str) -> &str {
 }
 
 fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (String, String) {
-    lazy_static! {
-        static ref ADDR_WITH_NAME_REGEX: Regex = Regex::new("(.*)<(.*)>").unwrap();
-    }
+    static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
     if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
             if name.as_ref().is_empty() {
@@ -1081,37 +1094,61 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
             // However, I'm not sure about this point; it may be confusing if the user wants to add other people;
             // this would result in recreating the same group...)
             if context.sql.execute(
-                    "UPDATE chats SET blocked=? WHERE type=? AND id IN (SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    paramsv![new_blocking, 100, contact_id as i32],
-                ).await.is_ok() {
-                    Contact::mark_noticed(context, contact_id).await;
-                    context.emit_event(Event::ContactsChanged(None));
-                }
+                "UPDATE chats SET blocked=? WHERE type=? AND id IN (SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
+                paramsv![new_blocking, 100, contact_id as i32]).await.is_ok()
+            {
+                Contact::mark_noticed(context, contact_id).await;
+                context.emit_event(EventType::ContactsChanged(Some(contact_id)));
+            }
         }
     }
 }
 
+/// Set profile image for a contact.
+///
+/// The given profile image is expected to be already in the blob directory
+/// as profile images can be set only by receiving messages, this should be always the case, however.
+///
+/// For contact SELF, the image is not saved in the contact-database but as Config::Selfavatar;
+/// this typically happens if we see message with our own profile image, sent from another device.
 pub(crate) async fn set_profile_image(
     context: &Context,
     contact_id: u32,
     profile_image: &AvatarAction,
+    was_encrypted: bool,
 ) -> Result<()> {
-    // the given profile image is expected to be already in the blob directory
-    // as profile images can be set only by receiving messages, this should be always the case, however.
     let mut contact = Contact::load_from_db(context, contact_id).await?;
     let changed = match profile_image {
         AvatarAction::Change(profile_image) => {
-            contact.param.set(Param::ProfileImage, profile_image);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context
+                        .set_config(Config::Selfavatar, Some(profile_image))
+                        .await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar.");
+                }
+            } else {
+                contact.param.set(Param::ProfileImage, profile_image);
+            }
             true
         }
         AvatarAction::Delete => {
-            contact.param.remove(Param::ProfileImage);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context.set_config(Config::Selfavatar, None).await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar deletion.");
+                }
+            } else {
+                contact.param.remove(Param::ProfileImage);
+            }
             true
         }
     };
     if changed {
         contact.update_param(context).await?;
-        context.emit_event(Event::ContactsChanged(Some(contact_id)));
+        context.emit_event(EventType::ContactsChanged(Some(contact_id)));
     }
     Ok(())
 }

src/context.rs

@@ -6,6 +6,7 @@ use std::ops::Deref;
 
 use async_std::path::{Path, PathBuf};
 use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
+use async_std::task;
 
 use crate::chat::*;
 use crate::config::Config;
@@ -13,14 +14,12 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::dc_tools::duration_to_str;
 use crate::error::*;
-use crate::events::{Event, EventEmitter, Events};
-use crate::job::{self, Action};
+use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::lot::Lot;
-use crate::message::{self, Message, MessengerMessage, MsgId};
-use crate::param::Params;
+use crate::message::{self, MsgId};
 use crate::scheduler::Scheduler;
+use crate::securejoin::Bob;
 use crate::sql::Sql;
 use std::time::SystemTime;
 
@@ -45,17 +44,23 @@ pub struct InnerContext {
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
     pub(crate) os_name: Option<String>,
-    pub(crate) bob: RwLock<BobStatus>,
+    pub(crate) bob: RwLock<Bob>,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
     pub(crate) generating_key_mutex: Mutex<()>,
     /// Mutex to enforce only a single running oauth2 is running.
     pub(crate) oauth2_mutex: Mutex<()>,
+    /// Mutex to prevent a race condition when a "your pw is wrong" warning is sent, resulting in multiple messeges being sent.
+    pub(crate) wrong_pw_warning_mutex: Mutex<()>,
     pub(crate) translated_stockstrings: RwLock<HashMap<usize, String>>,
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
+    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
+
+    /// Id for this context on the current device.
+    pub(crate) id: u32,
 
     creation_time: SystemTime,
 }
@@ -84,7 +89,7 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 
 impl Context {
     /// Creates new context.
-    pub async fn new(os_name: String, dbfile: PathBuf) -> Result<Context> {
+    pub async fn new(os_name: String, dbfile: PathBuf, id: u32) -> Result<Context> {
         // pretty_env_logger::try_init_timed().ok();
 
         let mut blob_fname = OsString::new();
@@ -94,13 +99,14 @@ impl Context {
         if !blobdir.exists().await {
             async_std::fs::create_dir_all(&blobdir).await?;
         }
-        Context::with_blobdir(os_name, dbfile, blobdir).await
+        Context::with_blobdir(os_name, dbfile, blobdir, id).await
     }
 
-    pub async fn with_blobdir(
+    pub(crate) async fn with_blobdir(
         os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
+        id: u32,
     ) -> Result<Context> {
         ensure!(
             blobdir.is_dir().await,
@@ -109,6 +115,7 @@ impl Context {
         );
 
         let inner = InnerContext {
+            id,
             blobdir,
             dbfile,
             os_name: Some(os_name),
@@ -118,19 +125,18 @@ impl Context {
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
+            wrong_pw_warning_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
+            ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
         };
 
         let ctx = Context {
             inner: Arc::new(inner),
         };
-        ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, false).await,
-            "Failed opening sqlite database"
-        );
+        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }
@@ -157,10 +163,6 @@ impl Context {
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
-        if !self.is_io_running().await {
-            info!(self, "IO is not running");
-            return;
-        }
 
         self.inner.stop_io().await;
     }
@@ -184,8 +186,11 @@ impl Context {
     }
 
     /// Emits a single event.
-    pub fn emit_event(&self, event: Event) {
-        self.events.emit(event);
+    pub fn emit_event(&self, event: EventType) {
+        self.events.emit(Event {
+            id: self.id,
+            typ: event,
+        });
     }
 
     /// Get the next queued event.
@@ -193,6 +198,11 @@ impl Context {
         self.events.get_emitter()
     }
 
+    /// Get the ID of this context.
+    pub fn get_id(&self) -> u32 {
+        self.id
+    }
+
     // Ongoing process allocation/free/check
 
     pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
@@ -455,32 +465,11 @@ impl Context {
             == Some(folder_name.as_ref().to_string())
     }
 
-    pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
-        if !self.get_config_bool(Config::MvboxMove).await {
-            return;
-        }
-
-        if self.is_mvbox(folder).await {
-            return;
-        }
-        if let Ok(msg) = Message::load_from_db(self, msg_id).await {
-            if msg.is_setupmessage() {
-                // do not move setup messages;
-                // there may be a non-delta device that wants to handle it
-                return;
-            }
-
-            match msg.is_dc_message {
-                MessengerMessage::No => {}
-                MessengerMessage::Yes | MessengerMessage::Reply => {
-                    job::add(
-                        self,
-                        job::Job::new(Action::MoveMsg, msg.id.to_u32(), Params::new(), 0),
-                    )
-                    .await;
-                }
-            }
-        }
+    pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
+        let mut blob_fname = OsString::new();
+        blob_fname.push(dbfile.file_name().unwrap_or_default());
+        blob_fname.push("-blobs");
+        dbfile.with_file_name(blob_fname)
     }
 }
 
@@ -490,14 +479,19 @@ impl InnerContext {
     }
 
     async fn stop_io(&self) {
-        assert!(self.is_io_running().await, "context is already stopped");
-        let token = {
-            let lock = &*self.scheduler.read().await;
-            lock.pre_stop().await
-        };
-        {
-            let lock = &mut *self.scheduler.write().await;
-            lock.stop(token).await;
+        if self.is_io_running().await {
+            let token = {
+                let lock = &*self.scheduler.read().await;
+                lock.pre_stop().await
+            };
+            {
+                let lock = &mut *self.scheduler.write().await;
+                lock.stop(token).await;
+            }
+        }
+
+        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
+            ephemeral_task.cancel().await;
         }
     }
 }
@@ -512,13 +506,6 @@ impl Default for RunningState {
     }
 }
 
-#[derive(Debug, Default)]
-pub(crate) struct BobStatus {
-    pub expects: i32,
-    pub status: i32,
-    pub qr_scan: Option<Lot>,
-}
-
 pub fn get_version_str() -> &'static str {
     &DC_VERSION_STR
 }
@@ -534,7 +521,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new("FakeOs".into(), dbfile.into()).await;
+        let res = Context::new("FakeOs".into(), dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -549,7 +536,9 @@ mod tests {
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -560,7 +549,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new("FakeOS".into(), dbfile.into()).await;
+        let res = Context::new("FakeOS".into(), dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -570,7 +559,9 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -580,7 +571,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir, 1).await;
         assert!(res.is_err());
     }
 
@@ -589,7 +580,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
         assert!(res.is_err());
     }
 

src/dc_receive_imf.rs

@@ -4,14 +4,15 @@ use sha2::{Digest, Sha256};
 
 use mailparse::SingleInfo;
 
-use crate::chat::{self, Chat, ChatId};
+use crate::chat::{self, Chat, ChatId, ProtectionStatus};
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::{stock_ephemeral_timer_changed, Timer as EphemeralTimer};
 use crate::error::{bail, ensure, format_err, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::job::{self, Action};
 use crate::message::{self, MessageState, MessengerMessage, MsgId};
@@ -42,6 +43,17 @@ pub async fn dc_receive_imf(
     server_folder: impl AsRef<str>,
     server_uid: u32,
     seen: bool,
+) -> Result<()> {
+    dc_receive_imf_inner(context, imf_raw, server_folder, server_uid, seen, false).await
+}
+
+pub(crate) async fn dc_receive_imf_inner(
+    context: &Context,
+    imf_raw: &[u8],
+    server_folder: impl AsRef<str>,
+    server_uid: u32,
+    seen: bool,
+    fetching_existing_messages: bool,
 ) -> Result<()> {
     info!(
         context,
@@ -92,8 +104,8 @@ pub async fn dc_receive_imf(
         if let Some(create_event_to_send) = create_event_to_send {
             for (chat_id, msg_id) in created_db_entries {
                 let event = match create_event_to_send {
-                    CreateEvent::MsgsChanged => Event::MsgsChanged { msg_id, chat_id },
-                    CreateEvent::IncomingMsg => Event::IncomingMsg { msg_id, chat_id },
+                    CreateEvent::MsgsChanged => EventType::MsgsChanged { msg_id, chat_id },
+                    CreateEvent::IncomingMsg => EventType::IncomingMsg { msg_id, chat_id },
                 };
                 context.emit_event(event);
             }
@@ -110,7 +122,7 @@ pub async fn dc_receive_imf(
     // or if From: is equal to SELF (in this case, it is any outgoing messages,
     // we do not check Return-Path any more as this is unreliable, see
     // https://github.com/deltachat/deltachat-core/issues/150)
-    let (from_id, from_id_blocked, incoming_origin) =
+    let (from_id, _from_id_blocked, incoming_origin) =
         from_field_to_contact_id(context, &mime_parser.from).await?;
 
     let incoming = from_id != DC_CONTACT_ID_SELF;
@@ -161,7 +173,6 @@ pub async fn dc_receive_imf(
             &rfc724_mid,
             &mut sent_timestamp,
             from_id,
-            from_id_blocked,
             &mut hidden,
             &mut chat_id,
             seen,
@@ -169,6 +180,7 @@ pub async fn dc_receive_imf(
             &mut insert_msg_id,
             &mut created_db_entries,
             &mut create_event_to_send,
+            fetching_existing_messages,
         )
         .await
         {
@@ -196,9 +208,16 @@ pub async fn dc_receive_imf(
     }
 
     if let Some(avatar_action) = &mime_parser.user_avatar {
-        match contact::set_profile_image(&context, from_id, avatar_action).await {
+        match contact::set_profile_image(
+            &context,
+            from_id,
+            avatar_action,
+            mime_parser.was_encrypted(),
+        )
+        .await
+        {
             Ok(()) => {
-                context.emit_event(Event::ChatModified(chat_id));
+                context.emit_event(EventType::ChatModified(chat_id));
             }
             Err(err) => {
                 warn!(context, "reveive_imf cannot update profile image: {}", err);
@@ -223,24 +242,29 @@ pub async fn dc_receive_imf(
                 )
                 .await;
             }
-        } else {
+        } else if insert_msg_id
+            .needs_move(context, server_folder.as_ref())
+            .await
+            .unwrap_or_default()
+        {
             // Move message if we don't delete it immediately.
-            context
-                .do_heuristics_moves(server_folder.as_ref(), insert_msg_id)
-                .await;
-            if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
-                // This is a Delta Chat MDN. Mark as read.
-                job::add(
-                    context,
-                    job::Job::new(
-                        Action::MarkseenMsgOnImap,
-                        insert_msg_id.to_u32(),
-                        Params::new(),
-                        0,
-                    ),
-                )
-                .await;
-            }
+            job::add(
+                context,
+                job::Job::new(Action::MoveMsg, insert_msg_id.to_u32(), Params::new(), 0),
+            )
+            .await;
+        } else if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
+            // This is a Delta Chat MDN. Mark as read.
+            job::add(
+                context,
+                job::Job::new(
+                    Action::MarkseenMsgOnImap,
+                    insert_msg_id.to_u32(),
+                    Params::new(),
+                    0,
+                ),
+            )
+            .await;
         }
     }
 
@@ -316,7 +340,6 @@ async fn add_parts(
     rfc724_mid: &str,
     sent_timestamp: &mut i64,
     from_id: u32,
-    from_id_blocked: bool,
     hidden: &mut bool,
     chat_id: &mut ChatId,
     seen: bool,
@@ -324,6 +347,7 @@ async fn add_parts(
     insert_msg_id: &mut MsgId,
     created_db_entries: &mut Vec<(ChatId, MsgId)>,
     create_event_to_send: &mut Option<CreateEvent>,
+    fetching_existing_messages: bool,
 ) -> Result<()> {
     let mut state: MessageState;
     let mut chat_id_blocked = Blocked::Not;
@@ -363,6 +387,7 @@ async fn add_parts(
         // this message is a classic email not a chat-message nor a reply to one
         match show_emails {
             ShowEmails::Off => {
+                info!(context, "Classical email not shown (TRASH)");
                 *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
                 allow_creation = false;
             }
@@ -378,7 +403,7 @@ async fn add_parts(
     let to_id: u32;
 
     if incoming {
-        state = if seen {
+        state = if seen || fetching_existing_messages {
             MessageState::InSeen
         } else {
             MessageState::InFresh
@@ -405,8 +430,6 @@ async fn add_parts(
                 }
                 Err(err) => {
                     *hidden = true;
-
-                    context.bob.write().await.status = 0; // secure-join failed
                     context.stop_ongoing().await;
                     warn!(context, "Error in Secure-Join message handling: {}", err);
                     return Ok(());
@@ -419,19 +442,15 @@ async fn add_parts(
                 .await
                 .unwrap_or_default();
 
-        if chat_id.is_unset() && mime_parser.failure_report.is_some() {
-            *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
-            info!(
-                context,
-                "Message belongs to an NDN and is not shown in a chat.",
-            );
-        }
-
         // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list,
         // it might also be blocked and displayed in the deaddrop as a result
+        if chat_id.is_unset() && mime_parser.failure_report.is_some() {
+            *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
+            info!(context, "Message belongs to an NDN (TRASH)",);
+        }
+
         if chat_id.is_unset() {
             // try to create a group
-            // (groups appear automatically only if the _sender_ is known, see core issue #54)
 
             let create_blocked =
                 if !test_normal_chat_id.is_unset() && test_normal_chat_id_blocked == Blocked::Not {
@@ -468,7 +487,7 @@ async fn add_parts(
             // check if the message belongs to a mailing list
             if mime_parser.is_mailinglist_message() {
                 *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
-                info!(context, "Message belongs to a mailing list and is ignored.",);
+                info!(context, "Message belongs to a mailing list (TRASH)");
             }
         }
 
@@ -512,6 +531,7 @@ async fn add_parts(
         if chat_id.is_unset() {
             // maybe from_id is null or sth. else is suspicious, move message to trash
             *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
+            info!(context, "No chat id for incoming msg (TRASH)")
         }
 
         // if the chat_id is blocked,
@@ -524,6 +544,10 @@ async fn add_parts(
             && show_emails != ShowEmails::All
         {
             state = MessageState::InNoticed;
+        } else if fetching_existing_messages && Blocked::Deaddrop == chat_id_blocked {
+            // The fetched existing message should be shown in the chatlist-contact-request because
+            // a new user won't find the contact request in the menu
+            state = MessageState::InFresh;
         }
     } else {
         // Outgoing
@@ -617,22 +641,136 @@ async fn add_parts(
         }
         if chat_id.is_unset() {
             *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
+            info!(context, "No chat id for outgoing message (TRASH)")
         }
     }
+
+    if fetching_existing_messages && mime_parser.decrypting_failed {
+        *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
+        // We are only gathering old messages on first start. We do not want to add loads of non-decryptable messages to the chats.
+        info!(context, "Existing non-decipherable message. (TRASH)");
+    }
+
+    // Extract ephemeral timer from the message.
+    let mut ephemeral_timer = if let Some(value) = mime_parser.get(HeaderDef::EphemeralTimer) {
+        match value.parse::<EphemeralTimer>() {
+            Ok(timer) => timer,
+            Err(err) => {
+                warn!(
+                    context,
+                    "can't parse ephemeral timer \"{}\": {}", value, err
+                );
+                EphemeralTimer::Disabled
+            }
+        }
+    } else {
+        EphemeralTimer::Disabled
+    };
+
+    let location_kml_is = mime_parser.location_kml.is_some();
+    let is_mdn = !mime_parser.mdn_reports.is_empty();
+
+    // Apply ephemeral timer changes to the chat.
+    //
+    // Only non-hidden timers are applied now. Timers from hidden
+    // messages such as read receipts can be useful to detect
+    // ephemeral timer support, but timer changes without visible
+    // received messages may be confusing to the user.
+    if !*hidden
+        && !location_kml_is
+        && !is_mdn
+        && (*chat_id).get_ephemeral_timer(context).await? != ephemeral_timer
+    {
+        if let Err(err) = (*chat_id)
+            .inner_set_ephemeral_timer(context, ephemeral_timer)
+            .await
+        {
+            warn!(
+                context,
+                "failed to modify timer for chat {}: {}", chat_id, err
+            );
+        } else if mime_parser.is_system_message != SystemMessage::EphemeralTimerChanged {
+            chat::add_info_msg(
+                context,
+                *chat_id,
+                stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await,
+            )
+            .await;
+        }
+    }
+
+    if mime_parser.is_system_message == SystemMessage::EphemeralTimerChanged {
+        set_better_msg(
+            mime_parser,
+            stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await,
+        );
+
+        // Do not delete the system message itself.
+        //
+        // This prevents confusion when timer is changed
+        // to 1 week, and then changed to 1 hour: after 1
+        // hour, only the message about the change to 1
+        // week is left.
+        ephemeral_timer = EphemeralTimer::Disabled;
+    }
+
+    // if a chat is protected, check additional properties
+    if !chat_id.is_special() {
+        let chat = Chat::load_from_db(context, *chat_id).await?;
+        let new_status = match mime_parser.is_system_message {
+            SystemMessage::ChatProtectionEnabled => Some(ProtectionStatus::Protected),
+            SystemMessage::ChatProtectionDisabled => Some(ProtectionStatus::Unprotected),
+            _ => None,
+        };
+
+        if chat.is_protected() || new_status.is_some() {
+            if let Err(err) =
+                check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
+            {
+                warn!(context, "verification problem: {}", err);
+                let s = format!("{}. See 'Info' for more details", err);
+                mime_parser.repl_msg_by_error(s);
+            } else {
+                // change chat protection only when verification check passes
+                if let Some(new_status) = new_status {
+                    if let Err(e) = chat_id.inner_set_protection(context, new_status).await {
+                        chat::add_info_msg(
+                            context,
+                            *chat_id,
+                            format!("Cannot set protection: {}", e),
+                        )
+                        .await;
+                        return Ok(()); // do not return an error as this would result in retrying the message
+                    }
+                    set_better_msg(
+                        mime_parser,
+                        context.stock_protection_msg(new_status, from_id).await,
+                    );
+                }
+            }
+        }
+    }
+
     // correct message_timestamp, it should not be used before,
     // however, we cannot do this earlier as we need from_id to be set
+    let in_fresh = state == MessageState::InFresh;
     let rcvd_timestamp = time();
-    let sort_timestamp = calc_sort_timestamp(
-        context,
-        *sent_timestamp,
-        *chat_id,
-        state == MessageState::InFresh,
-    )
-    .await;
-    *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
+    let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, in_fresh).await;
 
-    // unarchive chat
-    chat_id.unarchive(context).await?;
+    // Ensure replies to messages are sorted after the parent message.
+    //
+    // This is useful in a case where sender clocks are not
+    // synchronized and parent message has a Date: header with a
+    // timestamp higher than reply timestamp.
+    //
+    // This does not help if parent message arrives later than the
+    // reply.
+    let parent_timestamp = mime_parser.get_parent_timestamp(context).await?;
+    let sort_timestamp = parent_timestamp.map_or(sort_timestamp, |parent_timestamp| {
+        std::cmp::max(sort_timestamp, parent_timestamp)
+    });
+
+    *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
 
     // if the mime-headers should be saved, find out its size
     // (the mime-header ends with an empty line)
@@ -655,7 +793,6 @@ async fn add_parts(
 
     let mut parts = std::mem::replace(&mut mime_parser.parts, Vec::new());
     let server_folder = server_folder.as_ref().to_string();
-    let location_kml_is = mime_parser.location_kml.is_some();
     let is_system_message = mime_parser.is_system_message;
     let mime_headers = if save_mime_headers {
         Some(String::from_utf8_lossy(imf_raw).to_string())
@@ -665,7 +802,6 @@ async fn add_parts(
     let sent_timestamp = *sent_timestamp;
     let is_hidden = *hidden;
     let chat_id = *chat_id;
-    let is_mdn = !mime_parser.mdn_reports.is_empty();
 
     // TODO: can this clone be avoided?
     let rfc724_mid = rfc724_mid.to_string();
@@ -682,8 +818,8 @@ async fn add_parts(
                     "INSERT INTO msgs \
          (rfc724_mid, server_folder, server_uid, chat_id, from_id, to_id, timestamp, \
          timestamp_sent, timestamp_rcvd, type, state, msgrmsg,  txt, txt_raw, param, \
-         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references, error) \
-         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?, ?);",
+         bytes, hidden, mime_headers,  mime_in_reply_to, mime_references, error, ephemeral_timer, ephemeral_timestamp) \
+         VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?, ?,?,?);",
                 )?;
 
                 let is_location_kml = location_kml_is
@@ -705,6 +841,15 @@ async fn add_parts(
                     part.param.set_int(Param::Cmd, is_system_message as i32);
                 }
 
+                let ephemeral_timestamp = if in_fresh {
+                    0
+                } else {
+                    match ephemeral_timer {
+                        EphemeralTimer::Disabled => 0,
+                        EphemeralTimer::Enabled { duration } => rcvd_timestamp + i64::from(duration)
+                    }
+                };
+
                 stmt.execute(paramsv![
                     rfc724_mid,
                     server_folder,
@@ -727,7 +872,9 @@ async fn add_parts(
                     mime_headers,
                     mime_in_reply_to,
                     mime_references,
-                    part.error,
+                    part.error.take().unwrap_or_default(),
+                    ephemeral_timer,
+                    ephemeral_timestamp
                 ])?;
 
                 drop(stmt);
@@ -746,6 +893,10 @@ async fn add_parts(
         *insert_msg_id = *id;
     }
 
+    if !is_hidden {
+        chat_id.unarchive(context).await?;
+    }
+
     *hidden = is_hidden;
     created_db_entries.extend(ids.iter().map(|id| (chat_id, *id)));
     mime_parser.parts = new_parts;
@@ -755,13 +906,16 @@ async fn add_parts(
         "Message has {} parts and is assigned to chat #{}.", icnt, chat_id,
     );
 
+    // new outgoing message from another device marks the chat as noticed.
+    if !incoming && !*hidden && !chat_id.is_special() {
+        chat::marknoticed_chat_if_older_than(context, chat_id, sort_timestamp).await?;
+    }
+
     // check event to send
     if chat_id.is_trash() || *hidden {
         *create_event_to_send = None;
     } else if incoming && state == MessageState::InFresh {
-        if from_id_blocked {
-            *create_event_to_send = None;
-        } else if Blocked::Not != chat_id_blocked {
+        if Blocked::Not != chat_id_blocked {
             *create_event_to_send = Some(CreateEvent::MsgsChanged);
         } else {
             *create_event_to_send = Some(CreateEvent::IncomingMsg);
@@ -783,15 +937,17 @@ async fn add_parts(
         chat.update_param(context).await?;
         Ok(())
     }
-    update_last_subject(context, chat_id, mime_parser)
-        .await
-        .unwrap_or_else(|e| {
-            warn!(
-                context,
-                "Could not update LastSubject of chat: {}",
-                e.to_string()
-            )
-        });
+    if !is_mdn {
+        update_last_subject(context, chat_id, mime_parser)
+            .await
+            .unwrap_or_else(|e| {
+                warn!(
+                    context,
+                    "Could not update LastSubject of chat: {}",
+                    e.to_string()
+                )
+            });
+    }
 
     Ok(())
 }
@@ -852,7 +1008,7 @@ async fn save_locations(
         }
     }
     if send_event {
-        context.emit_event(Event::LocationChanged(Some(from_id)));
+        context.emit_event(EventType::LocationChanged(Some(from_id)));
     }
 }
 
@@ -924,37 +1080,24 @@ async fn create_or_lookup_group(
         set_better_msg(mime_parser, &better_msg);
     }
 
-    let mut grpid = "".to_string();
-    if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupId) {
-        grpid = optional_field.clone();
-    }
+    let grpid = try_getting_grpid(mime_parser);
 
     if grpid.is_empty() {
-        if let Some(extracted_grpid) = mime_parser
-            .get(HeaderDef::MessageId)
-            .and_then(|value| dc_extract_grpid_from_rfc724_mid(&value))
-        {
-            grpid = extracted_grpid.to_string();
-        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
-            grpid = extracted_grpid.to_string();
-        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References) {
-            grpid = extracted_grpid.to_string();
-        } else {
-            return create_or_lookup_adhoc_group(
-                context,
-                mime_parser,
-                allow_creation,
-                create_blocked,
-                from_id,
-                to_ids,
-            )
-            .await
-            .map_err(|err| {
-                info!(context, "could not create adhoc-group: {:?}", err);
-                err
-            });
-        }
+        return create_or_lookup_adhoc_group(
+            context,
+            mime_parser,
+            allow_creation,
+            create_blocked,
+            from_id,
+            to_ids,
+        )
+        .await
+        .map_err(|err| {
+            info!(context, "could not create adhoc-group: {:?}", err);
+            err
+        });
     }
+
     // now we have a grpid that is non-empty
     // but we might not know about this group
 
@@ -1036,29 +1179,18 @@ async fn create_or_lookup_group(
     set_better_msg(mime_parser, &better_msg);
 
     // check, if we have a chat with this group ID
-    let (mut chat_id, chat_id_verified, _blocked) = chat::get_chat_id_by_grpid(context, &grpid)
+    let (mut chat_id, _, _blocked) = chat::get_chat_id_by_grpid(context, &grpid)
         .await
         .unwrap_or((ChatId::new(0), false, Blocked::Not));
-    if !chat_id.is_unset() {
-        if chat_id_verified {
-            if let Err(err) =
-                check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
-            {
-                warn!(context, "verification problem: {}", err);
-                let s = format!("{}. See 'Info' for more details", err);
-                mime_parser.repl_msg_by_error(s);
-            }
-        }
-        if !chat::is_contact_in_chat(context, chat_id, from_id as u32).await {
-            // The From-address is not part of this group.
-            // It could be a new user or a DSN from a mailer-daemon.
-            // in any case we do not want to recreate the member list
-            // but still show the message as part of the chat.
-            // After all, the sender has a reference/in-reply-to that
-            // points to this chat.
-            let s = context.stock_str(StockMessage::UnknownSenderForChat).await;
-            mime_parser.repl_msg_by_error(s.to_string());
-        }
+    if !chat_id.is_unset() && !chat::is_contact_in_chat(context, chat_id, from_id as u32).await {
+        // The From-address is not part of this group.
+        // It could be a new user or a DSN from a mailer-daemon.
+        // in any case we do not want to recreate the member list
+        // but still show the message as part of the chat.
+        // After all, the sender has a reference/in-reply-to that
+        // points to this chat.
+        let s = context.stock_str(StockMessage::UnknownSenderForChat).await;
+        mime_parser.repl_msg_by_error(s.to_string());
     }
 
     // check if the group does not exist but should be created
@@ -1081,7 +1213,7 @@ async fn create_or_lookup_group(
                 || X_MrAddToGrp.is_some() && addr_cmp(&self_addr, X_MrAddToGrp.as_ref().unwrap()))
     {
         // group does not exist but should be created
-        let create_verified = if mime_parser.get(HeaderDef::ChatVerified).is_some() {
+        let create_protected = if mime_parser.get(HeaderDef::ChatVerified).is_some() {
             if let Err(err) =
                 check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
             {
@@ -1089,9 +1221,9 @@ async fn create_or_lookup_group(
                 let s = format!("{}. See 'Info' for more details", err);
                 mime_parser.repl_msg_by_error(&s);
             }
-            VerifiedStatus::Verified
+            ProtectionStatus::Protected
         } else {
-            VerifiedStatus::Unverified
+            ProtectionStatus::Unprotected
         };
 
         if !allow_creation {
@@ -1104,32 +1236,40 @@ async fn create_or_lookup_group(
             &grpid,
             grpname.as_ref().unwrap(),
             create_blocked,
-            create_verified,
+            create_protected,
         )
         .await;
         chat_id_blocked = create_blocked;
         recreate_member_list = true;
+
+        // once, we have protected-chats explained in UI, we can uncomment the following lines.
+        // ("verified groups" did not add a message anyway)
+        //
+        //if create_protected == ProtectionStatus::Protected {
+        // set from_id=0 as it is not clear that the sender of this random group message
+        // actually really has enabled chat-protection at some point.
+        //chat_id
+        //    .add_protection_msg(context, ProtectionStatus::Protected, false, 0)
+        //    .await?;
+        //}
     }
 
     // again, check chat_id
     if chat_id.is_special() {
-        return if group_explicitly_left {
-            Ok((ChatId::new(DC_CHAT_ID_TRASH), chat_id_blocked))
+        if mime_parser.decrypting_failed {
+            // It is possible that the message was sent to a valid,
+            // yet unknown group, which was rejected because
+            // Chat-Group-Name, which is in the encrypted part, was
+            // not found. We can't create a properly named group in
+            // this case, so assign error message to 1:1 chat with the
+            // sender instead.
+            return Ok((ChatId::new(0), Blocked::Not));
         } else {
-            create_or_lookup_adhoc_group(
-                context,
-                mime_parser,
-                allow_creation,
-                create_blocked,
-                from_id,
-                to_ids,
-            )
-            .await
-            .map_err(|err| {
-                warn!(context, "failed to create ad-hoc group: {:?}", err);
-                err
-            })
-        };
+            // The message was decrypted successfully, but contains a late "quit" or otherwise
+            // unwanted message.
+            info!(context, "message belongs to unwanted group (TRASH)");
+            return Ok((ChatId::new(DC_CHAT_ID_TRASH), chat_id_blocked));
+        }
     }
 
     // We have a valid chat_id > DC_CHAT_ID_LAST_SPECIAL.
@@ -1160,11 +1300,14 @@ async fn create_or_lookup_group(
                     .await
                     .is_ok()
                 {
-                    context.emit_event(Event::ChatModified(chat_id));
+                    context.emit_event(EventType::ChatModified(chat_id));
                 }
             }
         }
+    } else if mime_parser.is_system_message == SystemMessage::ChatProtectionEnabled {
+        recreate_member_list = true;
     }
+
     if let Some(avatar_action) = &mime_parser.group_avatar {
         info!(context, "group-avatar change for {}", chat_id);
         if let Ok(mut chat) = Chat::load_from_db(context, chat_id).await {
@@ -1219,11 +1362,32 @@ async fn create_or_lookup_group(
     }
 
     if send_EVENT_CHAT_MODIFIED {
-        context.emit_event(Event::ChatModified(chat_id));
+        context.emit_event(EventType::ChatModified(chat_id));
     }
     Ok((chat_id, chat_id_blocked))
 }
 
+fn try_getting_grpid(mime_parser: &MimeMessage) -> String {
+    if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupId) {
+        return optional_field.clone();
+    }
+
+    if let Some(extracted_grpid) = mime_parser
+        .get(HeaderDef::MessageId)
+        .and_then(|value| dc_extract_grpid_from_rfc724_mid(&value))
+    {
+        return extracted_grpid.to_string();
+    }
+    if !mime_parser.has_chat_version() {
+        if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
+            return extracted_grpid.to_string();
+        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References) {
+            return extracted_grpid.to_string();
+        }
+    }
+    "".to_string()
+}
+
 /// try extract a grpid from a message-id list header value
 fn extract_grpid(mime_parser: &MimeMessage, headerdef: HeaderDef) -> Option<&str> {
     let header = mime_parser.get(headerdef)?;
@@ -1314,12 +1478,10 @@ async fn create_or_lookup_adhoc_group(
         // decrypted.
         //
         // The subject may be encrypted and contain a placeholder such
-        // as "...". Besides that, it is possible that the message was
-        // sent to a valid, yet unknown group, which was rejected
-        // because Chat-Group-Name, which is in the encrypted part,
-        // was not found. Generating a new ID in this case would
-        // result in creation of a twin group with a different group
-        // ID.
+        // as "...". It can also be a COI group, with encrypted
+        // Chat-Group-ID and incompatible Message-ID format.
+        //
+        // Instead, assign the message to 1:1 chat with the sender.
         warn!(
             context,
             "not creating ad-hoc group for message that cannot be decrypted"
@@ -1351,14 +1513,14 @@ async fn create_or_lookup_adhoc_group(
         &grpid,
         grpname,
         create_blocked,
-        VerifiedStatus::Unverified,
+        ProtectionStatus::Unprotected,
     )
     .await;
     for &member_id in &member_ids {
         chat::add_to_chat_contacts_table(context, new_chat_id, member_id).await;
     }
 
-    context.emit_event(Event::ChatModified(new_chat_id));
+    context.emit_event(EventType::ChatModified(new_chat_id));
 
     Ok((new_chat_id, create_blocked))
 }
@@ -1368,20 +1530,17 @@ async fn create_group_record(
     grpid: impl AsRef<str>,
     grpname: impl AsRef<str>,
     create_blocked: Blocked,
-    create_verified: VerifiedStatus,
+    create_protected: ProtectionStatus,
 ) -> ChatId {
     if context.sql.execute(
-        "INSERT INTO chats (type, name, grpid, blocked, created_timestamp) VALUES(?, ?, ?, ?, ?);",
+        "INSERT INTO chats (type, name, grpid, blocked, created_timestamp, protected) VALUES(?, ?, ?, ?, ?, ?);",
         paramsv![
-            if VerifiedStatus::Unverified != create_verified {
-                Chattype::VerifiedGroup
-            } else {
-                Chattype::Group
-            },
+            Chattype::Group,
             grpname.as_ref(),
             grpid.as_ref(),
             create_blocked,
             time(),
+            create_protected,
         ],
     ).await
     .is_err()
@@ -1446,11 +1605,12 @@ async fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> String {
             },
         )
         .await
-        .unwrap_or_else(|_| member_cs);
+        .unwrap_or(member_cs);
 
     hex_hash(&members)
 }
 
+#[allow(clippy::indexing_slicing)]
 fn hex_hash(s: impl AsRef<str>) -> String {
     let bytes = s.as_ref().as_bytes();
     let result = Sha256::digest(bytes);
@@ -1473,7 +1633,7 @@ async fn search_chat_ids_by_contact_ids(
             }
         }
         if !contact_ids.is_empty() {
-            contact_ids.sort();
+            contact_ids.sort_unstable();
             let contact_ids_str = join(contact_ids.iter().map(|x| x.to_string()), ",");
             context.sql.query_map(
                 format!(
@@ -1503,7 +1663,7 @@ async fn search_chat_ids_by_contact_ids(
                             matches = 0;
                             mismatches = 0;
                         }
-                        if matches < contact_ids.len() && contact_id == contact_ids[matches] {
+                        if contact_ids.get(matches) == Some(&contact_id) {
                             matches += 1;
                         } else {
                             mismatches += 1;
@@ -1532,12 +1692,23 @@ async fn check_verified_properties(
 
     ensure!(mimeparser.was_encrypted(), "This message is not encrypted.");
 
+    if mimeparser.get(HeaderDef::ChatVerified).is_none() {
+        // we do not fail here currently, this would exclude (a) non-deltas
+        // and (b) deltas with different protection views across multiple devices.
+        // for group creation or protection enabled/disabled, however, Chat-Verified is respected.
+        warn!(
+            context,
+            "{} did not mark message as protected.",
+            contact.get_addr()
+        );
+    }
+
     // ensure, the contact is verified
     // and the message is signed with a verified key of the sender.
     // this check is skipped for SELF as there is no proper SELF-peerstate
     // and results in group-splits otherwise.
     if from_id != DC_CONTACT_ID_SELF {
-        let peerstate = Peerstate::from_addr(context, contact.get_addr()).await;
+        let peerstate = Peerstate::from_addr(context, contact.get_addr()).await?;
 
         if peerstate.is_none()
             || contact.is_verified_ex(context, peerstate.as_ref()).await
@@ -1591,7 +1762,7 @@ async fn check_verified_properties(
             context.is_self_addr(&to_addr).await
         );
         let mut is_verified = _is_verified != 0;
-        let peerstate = Peerstate::from_addr(context, &to_addr).await;
+        let peerstate = Peerstate::from_addr(context, &to_addr).await?;
 
         // mark gossiped keys (if any) as verified
         if mimeparser.gossipped_addr.contains(&to_addr) {
@@ -1621,7 +1792,7 @@ async fn check_verified_properties(
         }
         if !is_verified {
             bail!(
-                "{} is not a member of this verified group",
+                "{} is not a member of this protected chat",
                 to_addr.to_string()
             );
         }
@@ -1631,10 +1802,11 @@ async fn check_verified_properties(
 
 fn set_better_msg(mime_parser: &mut MimeMessage, better_msg: impl AsRef<str>) {
     let msg = better_msg.as_ref();
-    if !msg.is_empty() && !mime_parser.parts.is_empty() {
-        let part = &mut mime_parser.parts[0];
-        if part.typ == Viewtype::Text {
-            part.msg = msg.to_string();
+    if !msg.is_empty() {
+        if let Some(part) = mime_parser.parts.get_mut(0) {
+            if part.typ == Viewtype::Text {
+                part.msg = msg.to_string();
+            }
         }
     }
 }
@@ -1793,7 +1965,7 @@ fn dc_create_incoming_rfc724_mid(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat::ChatVisibility;
+    use crate::chat::{ChatItem, ChatVisibility};
     use crate::chatlist::Chatlist;
     use crate::message::Message;
     use crate::test_utils::*;
@@ -2056,7 +2228,7 @@ mod tests {
         let t = TestContext::new_alice().await;
 
         // create one-to-one with bob, archive one-to-one
-        let bob_id = Contact::create(&t.ctx, "bob", "bob@exampel.org")
+        let bob_id = Contact::create(&t.ctx, "bob", "bob@example.com")
             .await
             .unwrap();
         let one2one_id = chat::create_by_contact_id(&t.ctx, bob_id).await.unwrap();
@@ -2068,7 +2240,7 @@ mod tests {
         assert!(one2one.get_visibility() == ChatVisibility::Archived);
 
         // create a group with bob, archive group
-        let group_id = chat::create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
+        let group_id = chat::create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "foo")
             .await
             .unwrap();
         chat::add_contact_to_chat(&t.ctx, group_id, bob_id).await;
@@ -2118,8 +2290,12 @@ mod tests {
         .unwrap();
         let msgs = chat::get_chat_msgs(&t.ctx, group_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
-        let msg_id = msgs.first().unwrap();
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.first().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
+        let msg = message::Message::load_from_db(&t.ctx, *msg_id)
             .await
             .unwrap();
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
@@ -2170,7 +2346,7 @@ mod tests {
             chat::get_chat_msgs(&t.ctx, group_id, 0, None).await.len(),
             1
         );
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+        let msg = message::Message::load_from_db(&t.ctx, *msg_id)
             .await
             .unwrap();
         assert_eq!(msg.state, MessageState::OutMdnRcvd);
@@ -2253,8 +2429,12 @@ mod tests {
         );
         let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
-        let msg_id = msgs.first().unwrap();
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.first().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
+        let msg = message::Message::load_from_db(&t.ctx, *msg_id)
             .await
             .unwrap();
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
@@ -2361,7 +2541,7 @@ mod tests {
             "shenauithz@testrun.org",
             "Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it",
             include_bytes!("../test-data/message/tiscali_ndn.eml"),
-            "",
+            None,
         )
         .await;
     }
@@ -2373,7 +2553,7 @@ mod tests {
             "hcksocnsofoejx@five.chat",
             "Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org",
             include_bytes!("../test-data/message/testrun_ndn.eml"),
-            "Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"
+            Some("Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"),
         )
         .await;
     }
@@ -2385,7 +2565,7 @@ mod tests {
             "haeclirth.sinoenrat@yahoo.com",
             "1680295672.3657931.1591783872936@mail.yahoo.com",
             include_bytes!("../test-data/message/yahoo_ndn.eml"),
-            "Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com"
+            Some("Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com [...]"),
         )
         .await;
     }
@@ -2397,7 +2577,7 @@ mod tests {
             "assidhfaaspocwaeofi@gmail.com",
             "CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com",
             include_bytes!("../test-data/message/gmail_ndn.eml"),
-            "Delivery Status Notification (Failure) – ** Die Adresse wurde nicht gefunden **\n\nIhre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.\n\nHier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser\n\nAntwort:\n\n550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient\'s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp",
+            Some("Delivery Status Notification (Failure) – ** Die Adresse wurde nicht gefunden **\n\nIhre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.\n\nHier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser\n\nAntwort:\n\n550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient\'s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp"),
         )
         .await;
     }
@@ -2409,7 +2589,7 @@ mod tests {
             "snaerituhaeirns@gmail.com",
             "9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de",
             include_bytes!("../test-data/message/gmx_ndn.eml"),
-            "Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp"
+            Some("Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp [...]"),
         )
         .await;
     }
@@ -2421,7 +2601,7 @@ mod tests {
             "hanerthaertidiuea@gmx.de",
             "04422840-f884-3e37-5778-8192fe22d8e1@posteo.de",
             include_bytes!("../test-data/message/posteo_ndn.eml"),
-            "Undelivered Mail Returned to Sender – This is the mail system at host mout01.posteo.de.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550\n    Requested action not taken: mailbox unavailable (in reply to RCPT TO\n    command)",
+            Some("Undelivered Mail Returned to Sender – This is the mail system at host mout01.posteo.de.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550\n    Requested action not taken: mailbox unavailable (in reply to RCPT TO\n    command)"),
         )
         .await;
     }
@@ -2432,7 +2612,7 @@ mod tests {
         foreign_addr: &str,
         rfc724_mid_outgoing: &str,
         raw_ndn: &[u8],
-        error_msg: &str,
+        error_msg: Option<&str>,
     ) {
         let t = TestContext::new().await;
         t.configure_addr(self_addr).await;
@@ -2475,7 +2655,8 @@ mod tests {
         let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
 
         assert_eq!(msg.state, MessageState::OutFailed);
-        assert_eq!(msg.error, error_msg);
+
+        assert_eq!(msg.error(), error_msg.map(|error| error.to_string()));
     }
 
     #[async_std::test]
@@ -2516,9 +2697,12 @@ mod tests {
         assert_eq!(msg.state, MessageState::OutFailed);
 
         let msgs = chat::get_chat_msgs(&t.ctx, msg.chat_id, 0, None).await;
-        let last_msg = Message::load_from_db(&t.ctx, *msgs.last().unwrap())
-            .await
-            .unwrap();
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
+        let last_msg = Message::load_from_db(&t.ctx, *msg_id).await.unwrap();
 
         assert_eq!(
             last_msg.text,
@@ -2533,4 +2717,26 @@ mod tests {
         );
         assert_eq!(last_msg.from_id, DC_CONTACT_ID_INFO);
     }
+
+    #[async_std::test]
+    async fn test_html_only_mail() {
+        let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+        dc_receive_imf(
+            &t.ctx,
+            include_bytes!("../test-data/message/wrong-html.eml"),
+            "INBOX",
+            0,
+            false,
+        )
+        .await
+        .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+        assert_eq!(msg.text.unwrap(), "   Guten Abend,   \n\n   Lots of text   \n\n   text with Umlaut ä...   \n\n   MfG    [...]");
+    }
 }

src/dc_tools.rs

@@ -4,24 +4,29 @@
 use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
+use std::io::Cursor;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use async_std::path::{Path, PathBuf};
+use async_std::prelude::*;
 use async_std::{fs, io};
+
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
+use crate::chat::{add_device_msg, add_device_msg_with_importance};
+use crate::constants::{Viewtype, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::error::{bail, Error};
-use crate::events::Event;
-
-pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
-    0 != v && 0 == v & (v - 1)
-}
+use crate::events::EventType;
+use crate::message::Message;
+use crate::provider::get_provider_update_timestamp;
+use crate::stock::StockMessage;
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
     let ellipse = "[...]";
 
@@ -54,6 +59,7 @@ const COLORS: [u32; 16] = [
     0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
 ];
 
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let str_lower = s.as_ref().to_lowercase();
     let mut checksum = 0;
@@ -150,6 +156,73 @@ pub(crate) async fn dc_create_smeared_timestamps(context: &Context, count: usize
     start
 }
 
+// if the system time is not plausible, once a day, add a device message.
+// for testing we're using time() as that is also used for message timestamps.
+// moreover, add a warning if the app is outdated.
+pub(crate) async fn maybe_add_time_based_warnings(context: &Context) {
+    if !maybe_warn_on_bad_time(context, time(), get_provider_update_timestamp()).await {
+        maybe_warn_on_outdated(context, time(), get_provider_update_timestamp()).await;
+    }
+}
+
+async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestamp: i64) -> bool {
+    if now < known_past_timestamp {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(
+            context
+                .stock_string_repl_str(
+                    StockMessage::BadTimeMsgBody,
+                    Local
+                        .timestamp(now, 0)
+                        .format("%Y-%m-%d %H:%M:%S")
+                        .to_string(),
+                )
+                .await,
+        );
+        add_device_msg_with_importance(
+            context,
+            Some(
+                format!(
+                    "bad-time-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m-%d") // repeat every day
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+            true,
+        )
+        .await
+        .ok();
+        return true;
+    }
+    false
+}
+
+async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time: i64) {
+    if now > approx_compile_time + DC_OUTDATED_WARNING_DAYS * 24 * 60 * 60 {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(
+            context
+                .stock_str(StockMessage::UpdateReminderMsgBody)
+                .await
+                .into(),
+        );
+        add_device_msg(
+            context,
+            Some(
+                format!(
+                    "outdated-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m") // repeat every month
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+        )
+        .await
+        .ok();
+    }
+}
+
 /* Message-ID tools */
 pub(crate) fn dc_create_id() -> String {
     /* generate an id. the generated ID should be as short and as unique as possible:
@@ -198,7 +271,7 @@ fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
 pub(crate) fn dc_create_outgoing_rfc724_mid(grpid: Option<&str>, from_addr: &str) -> String {
     let hostname = from_addr
         .find('@')
-        .map(|k| &from_addr[k..])
+        .and_then(|k| from_addr.get(k..))
         .unwrap_or("@nohost");
     match grpid {
         Some(grpid) => format!("Gr.{}.{}{}", grpid, dc_create_id(), hostname),
@@ -240,9 +313,9 @@ pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
 
 /// Returns the `(width, height)` of the given image buffer.
 pub fn dc_get_filemeta(buf: &[u8]) -> Result<(u32, u32), Error> {
-    let meta = image_meta::load_from_buf(buf)?;
-
-    Ok((meta.dimensions.width, meta.dimensions.height))
+    let image = image::io::Reader::new(Cursor::new(buf)).with_guessed_format()?;
+    let dimensions = image.into_dimensions()?;
+    Ok(dimensions)
 }
 
 /// Expand paths relative to $BLOBDIR into absolute paths.
@@ -283,7 +356,7 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     let dpath = format!("{}", path.as_ref().to_string_lossy());
     match fs::remove_file(path_abs).await {
         Ok(_) => {
-            context.emit_event(Event::DeletedBlobFile(dpath));
+            context.emit_event(EventType::DeletedBlobFile(dpath));
             true
         }
         Err(err) => {
@@ -293,6 +366,23 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     }
 }
 
+pub async fn dc_delete_files_in_dir(context: &Context, path: impl AsRef<Path>) {
+    match async_std::fs::read_dir(path).await {
+        Ok(mut read_dir) => {
+            while let Some(entry) = read_dir.next().await {
+                match entry {
+                    Ok(file) => {
+                        dc_delete_file(context, file.file_name()).await;
+                    }
+                    Err(e) => warn!(context, "Could not read file to delete: {}", e),
+                }
+            }
+        }
+
+        Err(e) => warn!(context, "Could not read dir to delete: {}", e),
+    }
+}
+
 pub(crate) async fn dc_copy_file(
     context: &Context,
     src_path: impl AsRef<Path>,
@@ -447,7 +537,7 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn dc_get_next_backup_path(
+pub(crate) async fn get_next_backup_path_old(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<PathBuf, Error> {
@@ -467,6 +557,32 @@ pub(crate) async fn dc_get_next_backup_path(
     bail!("could not create backup file, disk full?");
 }
 
+/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
+/// it can be renamed to dest_path. This guarantees that the backup is complete.
+pub(crate) async fn get_next_backup_path_new(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<(PathBuf, PathBuf), Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        .format("delta-chat-backup-%Y-%m-%d")
+        .to_string();
+
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut tempfile = folder.clone();
+        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
+
+        let mut destfile = folder.clone();
+        destfile.push(format!("{}-{:02}.tar", stem, i));
+
+        if !tempfile.exists().await && !destfile.exists().await {
+            return Ok((tempfile, destfile));
+        }
+    }
+    bail!("could not create backup file, disk full?");
+}
+
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
@@ -583,8 +699,32 @@ pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
     (listflags & bitindex) == bitindex
 }
 
+/// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
+pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
+    input
+        .as_ref()
+        .replace("\n", " ")
+        .replace("\r", " ")
+        .trim()
+        .to_string()
+}
+
+pub(crate) trait IsNoneOrEmpty<T> {
+    fn is_none_or_empty(&self) -> bool;
+}
+impl<T> IsNoneOrEmpty<T> for Option<T>
+where
+    T: AsRef<str>,
+{
+    fn is_none_or_empty(&self) -> bool {
+        !matches!(self, Some(s) if !s.as_ref().is_empty())
+    }
+}
+
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use std::convert::TryInto;
 
@@ -744,6 +884,9 @@ mod tests {
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
+    use crate::chat;
+    use crate::chatlist::Chatlist;
+    use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
     use proptest::prelude::*;
 
     proptest! {
@@ -913,4 +1056,156 @@ mod tests {
             "3h 1m 0s"
         );
     }
+
+    #[test]
+    fn test_get_filemeta() {
+        let data = include_bytes!("../test-data/image/avatar900x900.png");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 900);
+        assert_eq!(h, 900);
+
+        let data = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 1000);
+        assert_eq!(h, 1000);
+
+        let data = include_bytes!("../test-data/image/image100x50.gif");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 100);
+        assert_eq!(h, 50);
+    }
+
+    #[test]
+    fn test_improve_single_line_input() {
+        assert_eq!(improve_single_line_input("Hi\naiae "), "Hi aiae");
+        assert_eq!(improve_single_line_input("\r\nahte\n\r"), "ahte");
+    }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_bad_time() {
+        let t = TestContext::new().await;
+        let timestamp_now = time();
+        let timestamp_future = timestamp_now + 60 * 60 * 24 * 7;
+        let timestamp_past = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 9, 1),
+            NaiveTime::from_hms(0, 0, 0),
+        )
+        .timestamp_millis()
+            / 1_000;
+
+        // a correct time must not add a device message
+        maybe_warn_on_bad_time(&t.ctx, timestamp_now, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // we cannot find out if a date in the future is wrong - a device message is not added
+        maybe_warn_on_bad_time(&t.ctx, timestamp_future, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // a date in the past must add a device message
+        maybe_warn_on_bad_time(&t.ctx, timestamp_past, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // the message should be added only once a day - test that an hour later and nearly a day later
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60 * 24 - 1,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // next day, there should be another device message
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60 * 24,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        assert_eq!(device_chat_id, chats.get_chat_id(0));
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 2);
+    }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_outdated() {
+        let t = TestContext::new().await;
+        let timestamp_now: i64 = time();
+
+        // in about 6 months, the app should not be outdated
+        // (if this fails, provider-db is not updated since 6 months)
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + 180 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // in 1 year, the app should be considered as outdated
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + 365 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // do not repeat the warning every day ...
+        // (we test that for the 2 subsequent days, this may be the next month, so the result should be 1 or 2 device message)
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + (365 + 1) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + (365 + 2) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let test_len = msgs.len();
+        assert!(test_len == 1 || test_len == 2);
+
+        // ... but every month
+        // (forward generous 33 days to avoid being in the same month as in the previous check)
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + (365 + 33) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), test_len + 1);
+    }
 }

src/dehtml.rs

@@ -2,12 +2,10 @@
 //!
 //! A module to remove HTML tags from the email text
 
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
-lazy_static! {
-    static ref LINE_RE: regex::Regex = regex::Regex::new(r"(\r?\n)+").unwrap();
-}
+static LINE_RE: Lazy<regex::Regex> = Lazy::new(|| regex::Regex::new(r"(\r?\n)+").unwrap());
 
 struct Dehtml {
     strbuilder: String,
@@ -24,8 +22,20 @@ enum AddText {
 
 // dehtml() returns way too many newlines; however, an optimisation on this issue is not needed as
 // the newlines are typically removed in further processing by the caller
-pub fn dehtml(buf: &str) -> String {
-    let buf = buf.trim();
+pub fn dehtml(buf: &str) -> Option<String> {
+    let s = dehtml_quick_xml(buf);
+    if !s.trim().is_empty() {
+        return Some(s);
+    }
+    let s = dehtml_manually(buf);
+    if !s.trim().is_empty() {
+        return Some(s);
+    }
+    None
+}
+
+pub fn dehtml_quick_xml(buf: &str) -> String {
+    let buf = buf.trim().trim_start_matches("<!doctype html>");
 
     let mut dehtml = Dehtml {
         strbuilder: String::with_capacity(buf.len()),
@@ -46,6 +56,12 @@ pub fn dehtml(buf: &str) -> String {
             Ok(quick_xml::events::Event::End(ref e)) => dehtml_endtag_cb(e, &mut dehtml),
             Ok(quick_xml::events::Event::Text(ref e)) => dehtml_text_cb(e, &mut dehtml),
             Ok(quick_xml::events::Event::CData(ref e)) => dehtml_cdata_cb(e, &mut dehtml),
+            Ok(quick_xml::events::Event::Empty(ref e)) => {
+                // Handle empty tags as a start tag immediately followed by end tag.
+                // For example, `<p/>` is treated as `<p></p>`.
+                dehtml_starttag_cb(e, &mut dehtml, &reader);
+                dehtml_endtag_cb(&BytesEnd::borrowed(e.name()), &mut dehtml);
+            }
             Err(e) => {
                 eprintln!(
                     "Parse html error: Error at position {}: {:?}",
@@ -165,9 +181,28 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     }
 }
 
+pub fn dehtml_manually(buf: &str) -> String {
+    // Just strip out everything between "<" and ">"
+    let mut strbuilder = String::new();
+    let mut show_next_chars = true;
+    for c in buf.chars() {
+        match c {
+            '<' => show_next_chars = false,
+            '>' => show_next_chars = true,
+            _ => {
+                if show_next_chars {
+                    strbuilder.push(c)
+                }
+            }
+        }
+    }
+    strbuilder
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::simplify::simplify;
 
     #[test]
     fn test_dehtml() {
@@ -176,31 +211,40 @@ mod tests {
                 "<a href='https://example.com'> Foo </a>",
                 "[ Foo ](https://example.com)",
             ),
-            ("<img href='/foo.png'>", ""),
             ("<b> bar </b>", "* bar *"),
             ("<b> bar <i> foo", "* bar _ foo"),
             ("&amp; bar", "& bar"),
-            // Note missing '
-            ("<a href='/foo.png>Hi</a> ", ""),
-            ("", ""),
+            // Despite missing ', this should be shown:
+            ("<a href='/foo.png>Hi</a> ", "Hi "),
+            (
+                "<a href='https://get.delta.chat/'/>",
+                "[](https://get.delta.chat/)",
+            ),
+            ("<!doctype html>\n<b>fat text</b>", "*fat text*"),
+            // Invalid html (at least DC should show the text if the html is invalid):
+            ("<!some invalid html code>\n<b>some text</b>", "some text"),
         ];
         for (input, output) in cases {
-            assert_eq!(dehtml(input), output);
+            assert_eq!(simplify(dehtml(input).unwrap(), true).0, output);
+        }
+        let none_cases = vec!["<html> </html>", ""];
+        for input in none_cases {
+            assert_eq!(dehtml(input), None);
         }
     }
 
     #[test]
     fn test_dehtml_parse_br() {
-        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
-        let plain = dehtml(html);
+        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2<br/>line3\n\r";
+        let plain = dehtml(html).unwrap();
 
-        assert_eq!(plain, "line1\n\r\r\rline2");
+        assert_eq!(plain, "line1\n\r\r\rline2\nline3");
     }
 
     #[test]
     fn test_dehtml_parse_href() {
         let html = "<a href=url>text</a";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "[text](url)");
     }
@@ -208,7 +252,7 @@ mod tests {
     #[test]
     fn test_dehtml_bold_text() {
         let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "text *bold*<>");
     }
@@ -218,7 +262,7 @@ mod tests {
         let html =
                 "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
 
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(
             plain,
@@ -241,7 +285,7 @@ mod tests {
         </body>
         </html>
         "##;
-        let txt = dehtml(input);
+        let txt = dehtml(input).unwrap();
         assert_eq!(txt.trim(), "lots of text");
     }
 }

src/e2ee.rs

@@ -15,7 +15,6 @@ use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
-use crate::securejoin::handle_degrade_event;
 
 #[derive(Debug)]
 pub struct EncryptHelper {
@@ -52,23 +51,42 @@ impl EncryptHelper {
     }
 
     /// Determines if we can and should encrypt.
+    ///
+    /// For encryption to be enabled, `e2ee_guaranteed` should be true, or strictly more than a half
+    /// of peerstates should prefer encryption. Own preference is counted equally to peer
+    /// preferences, even if message copy is not sent to self.
+    ///
+    /// `e2ee_guaranteed` should be set to true for replies to encrypted messages (as required by
+    /// Autocrypt Level 1, version 1.1) and for messages sent in protected groups.
+    ///
+    /// Returns an error if `e2ee_guaranteed` is true, but one or more keys are missing.
     pub fn should_encrypt(
         &self,
         context: &Context,
         e2ee_guaranteed: bool,
         peerstates: &[(Option<Peerstate>, &str)],
     ) -> Result<bool> {
-        if !(self.prefer_encrypt == EncryptPreference::Mutual || e2ee_guaranteed) {
-            return Ok(false);
-        }
-
+        let mut prefer_encrypt_count = if self.prefer_encrypt == EncryptPreference::Mutual {
+            1
+        } else {
+            0
+        };
         for (peerstate, addr) in peerstates {
             match peerstate {
                 Some(peerstate) => {
-                    if peerstate.prefer_encrypt != EncryptPreference::Mutual && !e2ee_guaranteed {
-                        info!(context, "peerstate for {:?} is no-encrypt", addr);
-                        return Ok(false);
-                    }
+                    info!(
+                        context,
+                        "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
+                    );
+                    match peerstate.prefer_encrypt {
+                        EncryptPreference::NoPreference => {}
+                        EncryptPreference::Mutual => prefer_encrypt_count += 1,
+                        EncryptPreference::Reset => {
+                            if !e2ee_guaranteed {
+                                return Ok(false);
+                            }
+                        }
+                    };
                 }
                 None => {
                     let msg = format!("peerstate for {:?} missing, cannot encrypt", addr);
@@ -82,7 +100,11 @@ impl EncryptHelper {
             }
         }
 
-        Ok(true)
+        // Count number of recipients, including self.
+        // This does not depend on whether we send a copy to self or not.
+        let recipients_count = peerstates.len() + 1;
+
+        Ok(e2ee_guaranteed || 2 * prefer_encrypt_count > recipients_count)
     }
 
     /// Tries to encrypt the passed in `mail`.
@@ -115,6 +137,14 @@ impl EncryptHelper {
     }
 }
 
+/// Tries to decrypt a message, but only if it is structured as an
+/// Autocrypt message.
+///
+/// Returns decrypted body and a set of valid signature fingerprints
+/// if successful.
+///
+/// If the message is wrongly signed, this will still return the decrypted
+/// message but the HashSet will be empty.
 pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
@@ -128,44 +158,31 @@ pub async fn try_decrypt(
         .map(|from| from.addr)
         .unwrap_or_default();
 
-    let mut peerstate = None;
-    let autocryptheader = Aheader::from_headers(context, &from, &mail.headers);
-
-    if message_time > 0 {
-        peerstate = Peerstate::from_addr(context, &from).await;
+    let mut peerstate = Peerstate::from_addr(context, &from).await?;
 
+    // Apply Autocrypt header
+    if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
         if let Some(ref mut peerstate) = peerstate {
-            if let Some(ref header) = autocryptheader {
-                peerstate.apply_header(&header, message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            } else if message_time > peerstate.last_seen_autocrypt && !contains_report(mail) {
-                peerstate.degrade_encryption(message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            }
-        } else if let Some(ref header) = autocryptheader {
+            peerstate.apply_header(&header, message_time);
+            peerstate.save_to_db(&context.sql, false).await?;
+        } else {
             let p = Peerstate::from_header(context, header, message_time);
             p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
     }
 
-    /* possibly perform decryption */
+    // Possibly perform decryption
     let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
     let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
     let mut signatures = HashSet::default();
 
-    if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-        peerstate = Peerstate::from_addr(&context, &from).await;
-    }
-    if let Some(peerstate) = peerstate {
-        if peerstate.degrade_event.is_some() {
-            handle_degrade_event(context, &peerstate).await?;
-        }
-        if let Some(key) = peerstate.gossip_key {
-            public_keyring_for_validate.add(key);
-        }
-        if let Some(key) = peerstate.public_key {
-            public_keyring_for_validate.add(key);
+    if let Some(ref mut peerstate) = peerstate {
+        peerstate.handle_fingerprint_change(context).await?;
+        if let Some(key) = &peerstate.public_key {
+            public_keyring_for_validate.add(key.clone());
+        } else if let Some(key) = &peerstate.gossip_key {
+            public_keyring_for_validate.add(key.clone());
         }
     }
 
@@ -177,6 +194,18 @@ pub async fn try_decrypt(
         &mut signatures,
     )
     .await?;
+
+    if let Some(mut peerstate) = peerstate {
+        // If message is not encrypted and it is not a read receipt, degrade encryption.
+        if out_mail.is_none()
+            && message_time > peerstate.last_seen_autocrypt
+            && !contains_report(mail)
+        {
+            peerstate.degrade_encryption(message_time);
+            peerstate.save_to_db(&context.sql, false).await?;
+        }
+    }
+
     Ok((out_mail, signatures))
 }
 
@@ -187,40 +216,32 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
         "Not a multipart/encrypted message: {}",
         mail.ctype.mimetype
     );
-    ensure!(
-        mail.subparts.len() == 2,
-        "Invalid Autocrypt Level 1 Mime Parts"
-    );
+    if let [first_part, second_part] = &mail.subparts[..] {
+        ensure!(
+            first_part.ctype.mimetype == "application/pgp-encrypted",
+            "Invalid Autocrypt Level 1 version part: {:?}",
+            first_part.ctype,
+        );
 
-    ensure!(
-        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
-        "Invalid Autocrypt Level 1 version part: {:?}",
-        mail.subparts[0].ctype,
-    );
+        ensure!(
+            second_part.ctype.mimetype == "application/octet-stream",
+            "Invalid Autocrypt Level 1 encrypted part: {:?}",
+            second_part.ctype
+        );
 
-    ensure!(
-        mail.subparts[1].ctype.mimetype == "application/octet-stream",
-        "Invalid Autocrypt Level 1 encrypted part: {:?}",
-        mail.subparts[1].ctype
-    );
-
-    Ok(&mail.subparts[1])
+        Ok(second_part)
+    } else {
+        bail!("Invalid Autocrypt Level 1 Mime Parts")
+    }
 }
 
-async fn decrypt_if_autocrypt_message<'a>(
+async fn decrypt_if_autocrypt_message(
     context: &Context,
-    mail: &ParsedMail<'a>,
+    mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
     ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
-    //  The returned bool is true if we detected an Autocrypt-encrypted
-    // message and successfully decrypted it. Decryption then modifies the
-    // passed in mime structure in place. The returned bool is false
-    // if it was not an Autocrypt message.
-    //
-    // Errors are returned for failures related to decryption of AC-messages.
-
     let encrypted_data_part = match get_autocrypt_mime(mail) {
         Err(_) => {
             // not an autocrypt mime message, abort and ignore
@@ -260,13 +281,16 @@ async fn decrypt_part(
         )
         .await?;
 
-        ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
+        // If the message was wrongly or not signed, still return the plain text.
+        // The caller has to check the signatures then.
+
         return Ok(Some(plain));
     }
 
     Ok(None)
 }
 
+#[allow(clippy::indexing_slicing)]
 fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     if let Some(index) = input.iter().position(|b| *b > b' ') {
         if input.len() - index > 26 {
@@ -320,6 +344,11 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
 mod tests {
     use super::*;
 
+    use crate::chat;
+    use crate::constants::Viewtype;
+    use crate::contact::{Contact, Origin};
+    use crate::message::Message;
+    use crate::param::Param;
     use crate::test_utils::*;
 
     mod ensure_secret_key_exists {
@@ -380,4 +409,161 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         let data = b"blas";
         assert_eq!(has_decrypted_pgp_armor(data), false);
     }
+
+    #[async_std::test]
+    async fn test_encrypted_no_autocrypt() -> crate::error::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let (contact_alice_id, _modified) = Contact::add_or_lookup(
+            &bob.ctx,
+            "Alice",
+            "alice@example.com",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+        let (contact_bob_id, _modified) = Contact::add_or_lookup(
+            &alice.ctx,
+            "Bob",
+            "bob@example.net",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+
+        let chat_alice = chat::create_by_contact_id(&alice.ctx, contact_bob_id).await?;
+        let chat_bob = chat::create_by_contact_id(&bob.ctx, contact_alice_id).await?;
+
+        // Alice sends unencrypted message to Bob
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        // Bob receives unencrypted message from Alice
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+
+        // Parsing a message is enough to update peerstate
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Bob sends encrypted message to Alice
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        chat::send_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        let sent = bob.pop_sent_msg().await;
+
+        // Alice receives encrypted message from Bob
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+
+        let peerstate_bob = Peerstate::from_addr(&alice.ctx, "bob@example.net")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_bob.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Now Alice and Bob have established keys.
+
+        // Alice sends encrypted message without Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::SkipAutocrypt, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Alice sends plaintext message with Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::ForcePlaintext, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Alice sends plaintext message without Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::ForcePlaintext, 1);
+        msg.param.set_int(Param::SkipAutocrypt, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Reset);
+
+        Ok(())
+    }
+
+    fn new_peerstates(
+        ctx: &Context,
+        prefer_encrypt: EncryptPreference,
+    ) -> Vec<(Option<Peerstate<'_>>, &str)> {
+        let addr = "bob@foo.bar";
+        let pub_key = bob_keypair().public;
+        let peerstate = Peerstate {
+            context: &ctx,
+            addr: addr.into(),
+            last_seen: 13,
+            last_seen_autocrypt: 14,
+            prefer_encrypt,
+            public_key: Some(pub_key.clone()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
+            gossip_key: Some(pub_key.clone()),
+            gossip_timestamp: 15,
+            gossip_key_fingerprint: Some(pub_key.fingerprint()),
+            verified_key: Some(pub_key.clone()),
+            verified_key_fingerprint: Some(pub_key.fingerprint()),
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        };
+        let mut peerstates = Vec::new();
+        peerstates.push((Some(peerstate), addr));
+        peerstates
+    }
+
+    #[async_std::test]
+    async fn test_should_encrypt() {
+        let t = TestContext::new_alice().await;
+        let encrypt_helper = EncryptHelper::new(&t.ctx).await.unwrap();
+
+        // test with EncryptPreference::NoPreference:
+        // if e2ee_eguaranteed is unset, there is no encryption as not more than half of peers want encryption
+        let ps = new_peerstates(&t.ctx, EncryptPreference::NoPreference);
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+
+        // test with EncryptPreference::Reset
+        let ps = new_peerstates(&t.ctx, EncryptPreference::Reset);
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+
+        // test with EncryptPreference::Mutual (self is also Mutual)
+        let ps = new_peerstates(&t.ctx, EncryptPreference::Mutual);
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+
+        // test with missing peerstate
+        let mut ps = Vec::new();
+        ps.push((None, "bob@foo.bar"));
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).is_err());
+        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+    }
 }

src/ephemeral.rs

@@ -0,0 +1,528 @@
+//! # Ephemeral messages
+//!
+//! Ephemeral messages are messages that have an Ephemeral-Timer
+//! header attached to them, which specifies time in seconds after
+//! which the message should be deleted both from the device and from
+//! the server. The timer is started when the message is marked as
+//! seen, which usually happens when its contents is displayed on
+//! device screen.
+//!
+//! Each chat, including 1:1, group chats and "saved messages" chat,
+//! has its own ephemeral timer setting, which is applied to all
+//! messages sent to the chat. The setting is synchronized to all the
+//! devices participating in the chat by applying the timer value from
+//! all received messages, including BCC-self ones, to the chat. This
+//! way the setting is eventually synchronized among all participants.
+//!
+//! When user changes ephemeral timer setting for the chat, a system
+//! message is automatically sent to update the setting for all
+//! participants. This allows changing the setting for a chat like any
+//! group chat setting, e.g. name and avatar, without the need to
+//! write an actual message.
+//!
+//! ## Device settings
+//!
+//! In addition to per-chat ephemeral message setting, each device has
+//! two global user-configured settings that complement per-chat
+//! settings: `delete_device_after` and `delete_server_after`. These
+//! settings are not synchronized among devices and apply to all
+//! messages known to the device, including messages sent or received
+//! before configuring the setting.
+//!
+//! `delete_device_after` configures the maximum time device is
+//! storing the messages locally. `delete_server_after` configures the
+//! time after which device will delete the messages it knows about
+//! from the server.
+//!
+//! ## How messages are deleted
+//!
+//! When the message is deleted locally, its contents is removed and
+//! it is moved to the trash chat. This database entry is then used to
+//! track the Message-ID and corresponding IMAP folder and UID until
+//! the message is deleted from the server. Vice versa, when device
+//! deletes the message from the server, it removes IMAP folder and
+//! UID information, but keeps the message contents. When database
+//! entry is both moved to trash chat and does not contain UID
+//! information, it is deleted from the database, leaving no trace of
+//! the message.
+//!
+//! ## When messages are deleted
+//!
+//! Local deletion happens when the chatlist or chat is loaded. A
+//! `MsgsChanged` event is emitted when a message deletion is due, to
+//! make UI reload displayed messages and cause actual deletion.
+//!
+//! Server deletion happens by generating IMAP deletion jobs based on
+//! the database entries which are expired either according to their
+//! ephemeral message timers or global `delete_server_after` setting.
+
+use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
+use crate::constants::{
+    Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
+};
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::error::{ensure, Error};
+use crate::events::EventType;
+use crate::message::{Message, MessageState, MsgId};
+use crate::mimeparser::SystemMessage;
+use crate::sql;
+use crate::stock::StockMessage;
+use async_std::task;
+use serde::{Deserialize, Serialize};
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
+pub enum Timer {
+    Disabled,
+    Enabled { duration: u32 },
+}
+
+impl Timer {
+    pub fn to_u32(self) -> u32 {
+        match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => duration,
+        }
+    }
+
+    pub fn from_u32(duration: u32) -> Self {
+        if duration == 0 {
+            Self::Disabled
+        } else {
+            Self::Enabled { duration }
+        }
+    }
+}
+
+impl Default for Timer {
+    fn default() -> Self {
+        Self::Disabled
+    }
+}
+
+impl ToString for Timer {
+    fn to_string(&self) -> String {
+        self.to_u32().to_string()
+    }
+}
+
+impl FromStr for Timer {
+    type Err = ParseIntError;
+
+    fn from_str(input: &str) -> Result<Timer, ParseIntError> {
+        input.parse::<u32>().map(Self::from_u32)
+    }
+}
+
+impl rusqlite::types::ToSql for Timer {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Integer(match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => i64::from(*duration),
+        });
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+impl rusqlite::types::FromSql for Timer {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|value| {
+            if value == 0 {
+                Ok(Self::Disabled)
+            } else if let Ok(duration) = u32::try_from(value) {
+                Ok(Self::Enabled { duration })
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(value))
+            }
+        })
+    }
+}
+
+impl ChatId {
+    /// Get ephemeral message timer value in seconds.
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
+        let timer = context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM chats WHERE id=?;",
+                paramsv![self],
+            )
+            .await?;
+        Ok(timer.unwrap_or_default())
+    }
+
+    /// Set ephemeral timer value without sending a message.
+    ///
+    /// Used when a message arrives indicating that someone else has
+    /// changed the timer value for a chat.
+    pub(crate) async fn inner_set_ephemeral_timer(
+        self,
+        context: &Context,
+        timer: Timer,
+    ) -> Result<(), Error> {
+        ensure!(!self.is_special(), "Invalid chat ID");
+
+        context
+            .sql
+            .execute(
+                "UPDATE chats
+             SET ephemeral_timer=?
+             WHERE id=?;",
+                paramsv![timer, self],
+            )
+            .await?;
+
+        context.emit_event(EventType::ChatEphemeralTimerModified {
+            chat_id: self,
+            timer,
+        });
+        Ok(())
+    }
+
+    /// Set ephemeral message timer value in seconds.
+    ///
+    /// If timer value is 0, disable ephemeral message timer.
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
+        if timer == self.get_ephemeral_timer(context).await? {
+            return Ok(());
+        }
+        self.inner_set_ephemeral_timer(context, timer).await?;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
+        msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
+        if let Err(err) = send_msg(context, self, &mut msg).await {
+            error!(
+                context,
+                "Failed to send a message about ephemeral message timer change: {:?}", err
+            );
+        }
+        Ok(())
+    }
+}
+
+/// Returns a stock message saying that ephemeral timer is changed to `timer` by `from_id`.
+pub(crate) async fn stock_ephemeral_timer_changed(
+    context: &Context,
+    timer: Timer,
+    from_id: u32,
+) -> String {
+    let stock_message = match timer {
+        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+        Timer::Enabled { duration } => match duration {
+            60 => StockMessage::MsgEphemeralTimerMinute,
+            3600 => StockMessage::MsgEphemeralTimerHour,
+            86400 => StockMessage::MsgEphemeralTimerDay,
+            604_800 => StockMessage::MsgEphemeralTimerWeek,
+            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
+            _ => StockMessage::MsgEphemeralTimerEnabled,
+        },
+    };
+
+    context
+        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
+        .await
+}
+
+impl MsgId {
+    /// Returns ephemeral message timer value for the message.
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> crate::sql::Result<Timer> {
+        let res = match context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM msgs WHERE id=?",
+                paramsv![self],
+            )
+            .await?
+        {
+            None | Some(0) => Timer::Disabled,
+            Some(duration) => Timer::Enabled { duration },
+        };
+        Ok(res)
+    }
+
+    /// Starts ephemeral message timer for the message if it is not started yet.
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> crate::sql::Result<()> {
+        if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
+            let ephemeral_timestamp = time() + i64::from(duration);
+
+            context
+                .sql
+                .execute(
+                    "UPDATE msgs SET ephemeral_timestamp = ? \
+                WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?) \
+                AND id = ?",
+                    paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
+                )
+                .await?;
+            schedule_ephemeral_task(context).await;
+        }
+        Ok(())
+    }
+}
+
+/// Deletes messages which are expired according to
+/// `delete_device_after` setting or `ephemeral_timestamp` column.
+///
+/// Returns true if any message is deleted, so caller can emit
+/// MsgsChanged event. If nothing has been deleted, returns
+/// false. This function does not emit the MsgsChanged event itself,
+/// because it is also called when chatlist is reloaded, and emitting
+/// MsgsChanged there will cause infinite reload loop.
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
+    let mut updated = context
+        .sql
+        .execute(
+            "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE \
+             ephemeral_timestamp != 0 \
+             AND ephemeral_timestamp < ? \
+             AND chat_id != ?",
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
+        )
+        .await?
+        > 0;
+
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
+        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+            .await
+            .unwrap_or_default()
+            .0;
+        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+            .await
+            .unwrap_or_default()
+            .0;
+
+        let threshold_timestamp = time() - delete_device_after;
+
+        // Delete expired messages
+        //
+        // Only update the rows that have to be updated, to avoid emitting
+        // unnecessary "chat modified" events.
+        let rows_modified = context
+            .sql
+            .execute(
+                "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE timestamp < ? \
+             AND chat_id > ? \
+             AND chat_id != ? \
+             AND chat_id != ?",
+                paramsv![
+                    DC_CHAT_ID_TRASH,
+                    threshold_timestamp,
+                    DC_CHAT_ID_LAST_SPECIAL,
+                    self_chat_id,
+                    device_chat_id
+                ],
+            )
+            .await?;
+
+        updated |= rows_modified > 0;
+    }
+
+    schedule_ephemeral_task(context).await;
+    Ok(updated)
+}
+
+/// Schedule a task to emit MsgsChanged event when the next local
+/// deletion happens. Existing task is cancelled to make sure at most
+/// one such task is scheduled at a time.
+///
+/// UI is expected to reload the chatlist or the chat in response to
+/// MsgsChanged event, this will trigger actual deletion.
+///
+/// This takes into account only per-chat timeouts, because global device
+/// timeouts are at least one hour long and deletion is triggered often enough
+/// by user actions.
+pub async fn schedule_ephemeral_task(context: &Context) {
+    let ephemeral_timestamp: Option<i64> = match context
+        .sql
+        .query_get_value_result(
+            "SELECT ephemeral_timestamp \
+         FROM msgs \
+         WHERE ephemeral_timestamp != 0 \
+           AND chat_id != ? \
+         ORDER BY ephemeral_timestamp ASC \
+         LIMIT 1",
+            paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
+        )
+        .await
+    {
+        Err(err) => {
+            warn!(context, "Can't calculate next ephemeral timeout: {}", err);
+            return;
+        }
+        Ok(ephemeral_timestamp) => ephemeral_timestamp,
+    };
+
+    // Cancel existing task, if any
+    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
+        ephemeral_task.cancel().await;
+    }
+
+    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
+        let now = SystemTime::now();
+        let until = UNIX_EPOCH
+            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+            + Duration::from_secs(1);
+
+        if let Ok(duration) = until.duration_since(now) {
+            // Schedule a task, ephemeral_timestamp is in the future
+            let context1 = context.clone();
+            let ephemeral_task = task::spawn(async move {
+                async_std::task::sleep(duration).await;
+                emit_event!(
+                    context1,
+                    EventType::MsgsChanged {
+                        chat_id: ChatId::new(0),
+                        msg_id: MsgId::new(0)
+                    }
+                );
+            });
+            *context.ephemeral_task.write().await = Some(ephemeral_task);
+        } else {
+            // Emit event immediately
+            emit_event!(
+                context,
+                EventType::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0)
+                }
+            );
+        }
+    }
+}
+
+/// Returns ID of any expired message that should be deleted from the server.
+///
+/// It looks up the trash chat too, to find messages that are already
+/// deleted locally, but not deleted on the server.
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+    let now = time();
+
+    let threshold_timestamp = match context.get_config_delete_server_after().await {
+        None => 0,
+        Some(delete_server_after) => now - delete_server_after,
+    };
+
+    context
+        .sql
+        .query_row_optional(
+            "SELECT id FROM msgs \
+         WHERE ( \
+         timestamp < ? \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         ) \
+         AND server_uid != 0 \
+         LIMIT 1",
+            paramsv![threshold_timestamp, now],
+            |row| row.get::<_, MsgId>(0),
+        )
+        .await
+}
+
+/// Start ephemeral timers for seen messages if they are not started
+/// yet.
+///
+/// It is possible that timers are not started due to a missing or
+/// failed `MsgId.start_ephemeral_timer()` call, either in the current
+/// or previous version of Delta Chat.
+///
+/// This function is supposed to be called in the background,
+/// e.g. from housekeeping task.
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
+    context
+        .sql
+        .execute(
+            "UPDATE msgs \
+    SET ephemeral_timestamp = ? + ephemeral_timer \
+    WHERE ephemeral_timer > 0 \
+    AND ephemeral_timestamp = 0 \
+    AND state NOT IN (?, ?, ?)",
+            paramsv![
+                time(),
+                MessageState::InFresh,
+                MessageState::InNoticed,
+                MessageState::OutDraft
+            ],
+        )
+        .await?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::*;
+
+    #[async_std::test]
+    async fn test_stock_ephemeral_messages() {
+        let context = TestContext::new().await.ctx;
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
+            "Message deletion timer is disabled by me."
+        );
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
+            "Message deletion timer is disabled."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
+            "Message deletion timer is set to 1 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
+            "Message deletion timer is set to 30 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
+            "Message deletion timer is set to 1 minute."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
+            "Message deletion timer is set to 1 hour."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 day."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 week."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 4 * 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 4 weeks."
+        );
+    }
+}

src/events.rs

@@ -1,10 +1,13 @@
 //! # Events specification
 
+use std::ops::Deref;
+
 use async_std::path::PathBuf;
 use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
 
 #[derive(Debug)]
@@ -53,14 +56,38 @@ impl EventEmitter {
         async_std::task::block_on(self.recv())
     }
 
-    /// Blocking async recv of an event. Return `None` if the `Sender` has been droped.
+    /// Async recv of an event. Return `None` if the `Sender` has been droped.
     pub async fn recv(&self) -> Option<Event> {
-        // TODO: change once we can use async channels internally.
         self.0.recv().await.ok()
     }
 }
 
-impl Event {
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Event {
+    pub id: u32,
+    pub typ: EventType,
+}
+
+impl Deref for Event {
+    type Target = EventType;
+
+    fn deref(&self) -> &EventType {
+        &self.typ
+    }
+}
+
+impl EventType {
     /// Returns the corresponding Event id.
     pub fn as_id(&self) -> i32 {
         self.get_str("id")
@@ -71,7 +98,7 @@ impl Event {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, EnumProperty)]
-pub enum Event {
+pub enum EventType {
     /// The library-user may write an informational string to the log.
     /// Passed to the callback given to dc_context_new().
     /// This event should not be reported to the end-user using a popup or something like that.
@@ -98,15 +125,11 @@ pub enum Event {
     #[strum(props(id = "105"))]
     ImapMessageMoved(String),
 
-    /// Emitted when an IMAP folder was emptied
-    #[strum(props(id = "106"))]
-    ImapFolderEmptied(String),
-
     /// Emitted when an new file in the $BLOBDIR was created
     #[strum(props(id = "150"))]
     NewBlobFile(String),
 
-    /// Emitted when an new file in the $BLOBDIR was created
+    /// Emitted when an file in the $BLOBDIR was deleted
     #[strum(props(id = "151"))]
     DeletedBlobFile(String),
 
@@ -139,7 +162,6 @@ pub enum Event {
     /// Network errors should be reported to users in a non-disturbing way,
     /// however, as network errors may come in a sequence,
     /// it is not useful to raise each an every error to the user.
-    /// For this purpose, data1 is set to 1 if the error is probably worth reporting.
     ///
     /// Moreover, if the UI detects that the device is offline,
     /// it is probably more useful to report this to the user
@@ -170,6 +192,11 @@ pub enum Event {
     #[strum(props(id = "2005"))]
     IncomingMsg { chat_id: ChatId, msg_id: MsgId },
 
+    /// Messages were seen or noticed.
+    /// chat id is always set.
+    #[strum(props(id = "2008"))]
+    MsgsNoticed(ChatId),
+
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
     #[strum(props(id = "2010"))]
@@ -189,9 +216,19 @@ pub enum Event {
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
+    ///
+    /// This event does not include ephemeral timer modification, which
+    /// is a separate event.
     #[strum(props(id = "2020"))]
     ChatModified(ChatId),
 
+    /// Chat ephemeral timer changed.
+    #[strum(props(id = "2021"))]
+    ChatEphemeralTimerModified {
+        chat_id: ChatId,
+        timer: EphemeralTimer,
+    },
+
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
@@ -207,10 +244,16 @@ pub enum Event {
     LocationChanged(Option<u32>),
 
     /// Inform about the configuration progress started by configure().
-    ///
-    /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     #[strum(props(id = "2041"))]
-    ConfigureProgress(usize),
+    ConfigureProgress {
+        /// Progress.
+        ///
+        /// 0=error, 1-999=progress in permille, 1000=success and done
+        progress: usize,
+
+        /// Progress comment or error, something to display to the user.
+        comment: Option<String>,
+    },
 
     /// Inform about the import/export progress started by imex().
     ///

src/format_flowed.rs

@@ -0,0 +1,177 @@
+///! # format=flowed support
+///!
+///! Format=flowed is defined in
+///! [RFC 3676](https://tools.ietf.org/html/rfc3676).
+///!
+///! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
+///! during formatting, i.e., DelSp parameter introduced in RFC 3676
+///! is assumed to be set to "no".
+///!
+///! For received messages, DelSp parameter is honoured.
+
+/// Wraps line to 72 characters using format=flowed soft breaks.
+///
+/// 72 characters is the limit recommended by RFC 3676.
+///
+/// The function breaks line only after SP and before non-whitespace
+/// characters. It also does not insert breaks before ">" to avoid the
+/// need to do space stuffing (see RFC 3676) for quotes.
+///
+/// If there are long words, line may still exceed the limits on line
+/// length. However, this should be rare and should not result in
+/// immediate mail rejection: SMTP (RFC 2821) limit is 998 characters,
+/// and Spam Assassin limit is 78 characters.
+fn format_line_flowed(line: &str, prefix: &str) -> String {
+    let mut result = String::new();
+    let mut buffer = prefix.to_string();
+    let mut after_space = false;
+
+    for c in line.chars() {
+        if c == ' ' {
+            buffer.push(c);
+            after_space = true;
+        } else if c == '>' {
+            if buffer.is_empty() {
+                // Space stuffing, see RFC 3676
+                buffer.push(' ');
+            }
+            buffer.push(c);
+            after_space = false;
+        } else {
+            if after_space && buffer.len() >= 72 && !c.is_whitespace() {
+                // Flush the buffer and insert soft break (SP CRLF).
+                result += &buffer;
+                result += "\r\n";
+                buffer = prefix.to_string();
+            }
+            buffer.push(c);
+            after_space = false;
+        }
+    }
+    result + &buffer
+}
+
+fn format_flowed_prefix(text: &str, prefix: &str) -> String {
+    let mut result = String::new();
+
+    for line in text.split('\n') {
+        if !result.is_empty() {
+            result += "\r\n";
+        }
+        let line = line.trim_end();
+        if prefix.len() + line.len() > 78 {
+            result += &format_line_flowed(line, prefix);
+        } else {
+            result += prefix;
+            if prefix.is_empty() && line.starts_with('>') {
+                // Space stuffing, see RFC 3676
+                result.push(' ');
+            }
+            result += line;
+        }
+    }
+    result
+}
+
+/// Returns text formatted according to RFC 3767 (format=flowed).
+///
+/// This function accepts text separated by LF, but returns text
+/// separated by CRLF.
+///
+/// RFC 2646 technique is used to insert soft line breaks, so DelSp
+/// SHOULD be set to "no" when sending.
+pub fn format_flowed(text: &str) -> String {
+    format_flowed_prefix(text, "")
+}
+
+/// Same as format_flowed(), but adds "> " prefix to each line.
+pub fn format_flowed_quote(text: &str) -> String {
+    format_flowed_prefix(text, "> ")
+}
+
+/// Joins lines in format=flowed text.
+///
+/// Lines must be separated by single LF.
+///
+/// Quote processing is not supported, it is assumed that they are
+/// deleted during simplification.
+///
+/// Signature separator line is not processed here, it is assumed to
+/// be stripped beforehand.
+pub fn unformat_flowed(text: &str, delsp: bool) -> String {
+    let mut result = String::new();
+    let mut skip_newline = true;
+
+    for line in text.split('\n') {
+        // Revert space-stuffing
+        let line = line.strip_prefix(" ").unwrap_or(line);
+
+        if !skip_newline {
+            result.push('\n');
+        }
+
+        if let Some(line) = line.strip_suffix(" ") {
+            // Flowed line
+            result += line;
+            if !delsp {
+                result.push(' ');
+            }
+            skip_newline = true;
+        } else {
+            // Fixed line
+            result += line;
+            skip_newline = false;
+        }
+    }
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_format_flowed() {
+        let text = "Foo bar baz";
+        assert_eq!(format_flowed(text), "Foo bar baz");
+
+        let text = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\
+                    \n\
+                    To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
+        let expected = "This is the Autocrypt Setup Message used to transfer your key between clients.\r\n\
+                        \r\n\
+                        To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        client and enter the setup code presented on the generating device.";
+        assert_eq!(format_flowed(text), expected);
+
+        let text = "> Not a quote";
+        assert_eq!(format_flowed(text), " > Not a quote");
+    }
+
+    #[test]
+    fn test_unformat_flowed() {
+        let text = "this is a very long message that should be wrapped using format=flowed and \n\
+            unwrapped on the receiver";
+        let expected =
+            "this is a very long message that should be wrapped using format=flowed and \
+                        unwrapped on the receiver";
+        assert_eq!(unformat_flowed(text, false), expected);
+    }
+
+    #[test]
+    fn test_format_flowed_quote() {
+        let quote = "this is a quoted line";
+        let expected = "> this is a quoted line";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "> foo bar baz";
+        let expected = "> > foo bar baz";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "this is a very long quote that should be wrapped using format=flowed and unwrapped on the receiver";
+        let expected =
+            "> this is a very long quote that should be wrapped using format=flowed and \r\n\
+            > unwrapped on the receiver";
+        assert_eq!(format_flowed_quote(quote), expected);
+    }
+}

src/headerdef.rs

@@ -3,7 +3,6 @@ use mailparse::{MailHeader, MailHeaderMap};
 
 #[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
 #[strum(serialize_all = "kebab_case")]
-#[allow(dead_code)]
 pub enum HeaderDef {
     MessageId,
     Subject,
@@ -35,6 +34,7 @@ pub enum HeaderDef {
     ChatContent,
     ChatDuration,
     ChatDispositionNotificationTo,
+    ChatWebrtcRoom,
     Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,
@@ -42,6 +42,7 @@ pub enum HeaderDef {
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
     SecureJoinAuth,
+    EphemeralTimer,
     _TestHeader,
 }
 

src/imap/client.rs

@@ -56,7 +56,7 @@ impl Client {
     pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
         self,
         auth_type: S,
-        authenticator: &A,
+        authenticator: A,
     ) -> std::result::Result<Session, (ImapError, Self)> {
         let Client { inner, is_secure } = self;
         let session =
@@ -85,9 +85,6 @@ impl Client {
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
 
         let _greeting = client
             .read_response()
@@ -104,9 +101,6 @@ impl Client {
         let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
         let _greeting = client
             .read_response()
             .await

src/imap/idle.rs

@@ -1,34 +1,15 @@
 use super::Imap;
 
 use async_imap::extensions::idle::IdleResponse;
+use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
+use crate::error::{bail, format_err, Result};
 use crate::{context::Context, scheduler::InterruptInfo};
 
-use super::select_folder;
 use super::session::Session;
 
-type Result<T> = std::result::Result<T, Error>;
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete")]
-    IdleProtocolFailed(#[from] async_imap::error::Error),
-
-    #[error("IMAP IDLE protocol timed out")]
-    IdleTimeout(#[from] async_std::future::TimeoutError),
-
-    #[error("IMAP server does not have IDLE capability")]
-    IdleAbilityMissing,
-
-    #[error("IMAP select folder error")]
-    SelectFolderError(#[from] select_folder::Error),
-
-    #[error("Setup handle error")]
-    SetupHandleError(#[from] super::Error),
-}
-
 impl Imap {
     pub fn can_idle(&self) -> bool {
         self.config.can_idle
@@ -42,20 +23,36 @@ impl Imap {
         use futures::future::FutureExt;
 
         if !self.can_idle() {
-            return Err(Error::IdleAbilityMissing);
+            bail!("IMAP server does not have IDLE capability");
         }
-        self.setup_handle_if_needed(context).await?;
+        self.setup_handle(context).await?;
 
         self.select_folder(context, watch_folder.clone()).await?;
 
-        let session = self.session.take();
         let timeout = Duration::from_secs(23 * 60);
         let mut info = Default::default();
 
-        if let Some(session) = session {
+        if let Some(session) = self.session.take() {
+            // if we have unsolicited responses we directly return
+            let mut unsolicited_exists = false;
+            while let Ok(response) = session.unsolicited_responses.try_recv() {
+                match response {
+                    UnsolicitedResponse::Exists(_) => {
+                        warn!(context, "skip idle, got unsolicited EXISTS {:?}", response);
+                        unsolicited_exists = true;
+                    }
+                    _ => info!(context, "ignoring unsolicited response {:?}", response),
+                }
+            }
+
+            if unsolicited_exists {
+                self.session = Some(session);
+                return Ok(info);
+            }
+
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
-                return Err(Error::IdleProtocolFailed(err));
+                bail!("IMAP IDLE protocol failed to init/complete: {}", err);
             }
 
             let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
@@ -65,68 +62,43 @@ impl Imap {
                 Interrupt(InterruptInfo),
             }
 
-            if self.skip_next_idle_wait {
-                // interrupt_idle has happened before we
-                // provided self.interrupt
-                self.skip_next_idle_wait = false;
-                drop(idle_wait);
+            info!(context, "Idle entering wait-on-remote state");
+            let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
+                let probe_network = self.idle_interrupt.recv().await;
+
+                // cancel imap idle connection properly
                 drop(interrupt);
 
-                info!(context, "Idle wait was skipped");
-            } else {
-                info!(context, "Idle entering wait-on-remote state");
-                let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(
-                    self.idle_interrupt.recv().map(|probe_network| {
-                        Ok(Event::Interrupt(probe_network.unwrap_or_default()))
-                    }),
-                );
+                Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+            });
 
-                match fut.await {
-                    Ok(Event::IdleResponse(IdleResponse::NewData(_))) => {
-                        info!(context, "Idle has NewData");
-                    }
-                    // TODO: idle_wait does not distinguish manual interrupts
-                    // from Timeouts if we would know it's a Timeout we could bail
-                    // directly and reconnect .
-                    Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
-                        info!(context, "Idle-wait timeout or interruption");
-                    }
-                    Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Ok(Event::Interrupt(i)) => {
-                        info = i;
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Err(err) => {
-                        warn!(context, "Idle wait errored: {:?}", err);
-                    }
+            match fut.await {
+                Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
+                    info!(context, "Idle has NewData {:?}", x);
+                }
+                Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
+                    info!(context, "Idle-wait timeout or interruption");
+                }
+                Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
+                    info!(context, "Idle wait was interrupted");
+                }
+                Ok(Event::Interrupt(i)) => {
+                    info = i;
+                    info!(context, "Idle wait was interrupted");
+                }
+                Err(err) => {
+                    warn!(context, "Idle wait errored: {:?}", err);
                 }
             }
 
-            // if we can't properly terminate the idle
-            // protocol let's break the connection.
-            let res = handle
+            let session = handle
                 .done()
                 .timeout(Duration::from_secs(15))
                 .await
-                .map_err(|err| {
-                    self.trigger_reconnect();
-                    Error::IdleTimeout(err)
-                })?;
-
-            match res {
-                Ok(session) => {
-                    self.session = Some(Session { inner: session });
-                }
-                Err(err) => {
-                    // if we cannot terminate IDLE it probably
-                    // means that we waited long (with idle_wait)
-                    // but the network went away/changed
-                    self.trigger_reconnect();
-                    return Err(Error::IdleProtocolFailed(err));
-                }
-            }
+                .map_err(|err| format_err!("IMAP IDLE protocol timed out: {}", err))??;
+            self.session = Some(Session { inner: session });
+        } else {
+            warn!(context, "Attempted to idle without a session");
         }
 
         Ok(info)
@@ -141,80 +113,74 @@ impl Imap {
         // in this case, we're waiting for a configure job (and an interrupt).
 
         let fake_idle_start_time = SystemTime::now();
-        info!(context, "IMAP-fake-IDLEing...");
 
         // Do not poll, just wait for an interrupt when no folder is passed in.
         if watch_folder.is_none() {
+            info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
             return self.idle_interrupt.recv().await.unwrap_or_default();
         }
+        info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
-        let mut info: InterruptInfo = Default::default();
-        if self.skip_next_idle_wait {
-            // interrupt_idle has happened before we
-            // provided self.interrupt
-            self.skip_next_idle_wait = false;
-            info!(context, "fake-idle wait was skipped");
-        } else {
-            // check every minute if there are new messages
-            // TODO: grow sleep durations / make them more flexible
-            let mut interval = async_std::stream::interval(Duration::from_secs(60));
+        // check every minute if there are new messages
+        // TODO: grow sleep durations / make them more flexible
+        let mut interval = async_std::stream::interval(Duration::from_secs(60));
 
-            enum Event {
-                Tick,
-                Interrupt(InterruptInfo),
-            }
-            // loop until we are interrupted or if we fetched something
-            info =
-                loop {
-                    use futures::future::FutureExt;
-                    match interval
-                        .next()
-                        .map(|_| Event::Tick)
-                        .race(self.idle_interrupt.recv().map(|probe_network| {
-                            Event::Interrupt(probe_network.unwrap_or_default())
-                        }))
-                        .await
-                    {
-                        Event::Tick => {
-                            // try to connect with proper login params
-                            // (setup_handle_if_needed might not know about them if we
-                            // never successfully connected)
-                            if let Err(err) = self.connect_configured(context).await {
-                                warn!(context, "fake_idle: could not connect: {}", err);
-                                continue;
-                            }
-                            if self.config.can_idle {
-                                // we only fake-idled because network was gone during IDLE, probably
-                                break InterruptInfo::new(false, None);
-                            }
-                            info!(context, "fake_idle is connected");
-                            // we are connected, let's see if fetching messages results
-                            // in anything.  If so, we behave as if IDLE had data but
-                            // will have already fetched the messages so perform_*_fetch
-                            // will not find any new.
+        enum Event {
+            Tick,
+            Interrupt(InterruptInfo),
+        }
+        // loop until we are interrupted or if we fetched something
+        let info = loop {
+            use futures::future::FutureExt;
+            match interval
+                .next()
+                .map(|_| Event::Tick)
+                .race(
+                    self.idle_interrupt
+                        .recv()
+                        .map(|probe_network| Event::Interrupt(probe_network.unwrap_or_default())),
+                )
+                .await
+            {
+                Event::Tick => {
+                    // try to connect with proper login params
+                    // (setup_handle_if_needed might not know about them if we
+                    // never successfully connected)
+                    if let Err(err) = self.connect_configured(context).await {
+                        warn!(context, "fake_idle: could not connect: {}", err);
+                        continue;
+                    }
+                    if self.config.can_idle {
+                        // we only fake-idled because network was gone during IDLE, probably
+                        break InterruptInfo::new(false, None);
+                    }
+                    info!(context, "fake_idle is connected");
+                    // we are connected, let's see if fetching messages results
+                    // in anything.  If so, we behave as if IDLE had data but
+                    // will have already fetched the messages so perform_*_fetch
+                    // will not find any new.
 
-                            if let Some(ref watch_folder) = watch_folder {
-                                match self.fetch_new_messages(context, watch_folder).await {
-                                    Ok(res) => {
-                                        info!(context, "fetch_new_messages returned {:?}", res);
-                                        if res {
-                                            break InterruptInfo::new(false, None);
-                                        }
-                                    }
-                                    Err(err) => {
-                                        error!(context, "could not fetch from folder: {}", err);
-                                        self.trigger_reconnect()
-                                    }
+                    if let Some(ref watch_folder) = watch_folder {
+                        match self.fetch_new_messages(context, watch_folder, false).await {
+                            Ok(res) => {
+                                info!(context, "fetch_new_messages returned {:?}", res);
+                                if res {
+                                    break InterruptInfo::new(false, None);
                                 }
                             }
-                        }
-                        Event::Interrupt(info) => {
-                            // Interrupt
-                            break info;
+                            Err(err) => {
+                                error!(context, "could not fetch from folder: {}", err);
+                                self.trigger_reconnect()
+                            }
                         }
                     }
-                };
-        }
+                }
+                Event::Interrupt(info) => {
+                    // Interrupt
+                    break info;
+                }
+            }
+        };
 
         info!(
             context,

src/imap/select_folder.rs

@@ -51,6 +51,14 @@ impl Imap {
         Ok(())
     }
 
+    /// Issues a CLOSE command if selected folder needs expunge.
+    pub(crate) async fn maybe_close_folder(&mut self, context: &Context) -> Result<()> {
+        if self.config.selected_folder_needs_expunge {
+            self.close_folder(context).await?;
+        }
+        Ok(())
+    }
+
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
@@ -76,10 +84,7 @@ impl Imap {
         }
 
         // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
-        let needs_expunge = { self.config.selected_folder_needs_expunge };
-        if needs_expunge {
-            self.close_folder(context).await?;
-        }
+        self.maybe_close_folder(context).await?;
 
         // select new folder
         if let Some(ref folder) = folder {

src/imex.rs

@@ -1,10 +1,17 @@
 //! # Import/export module
 
 use std::any::Any;
-use std::cmp::{max, min};
+use std::{
+    cmp::{max, min},
+    ffi::OsStr,
+};
 
+use anyhow::Context as _;
 use async_std::path::{Path, PathBuf};
-use async_std::prelude::*;
+use async_std::{
+    fs::{self, File},
+    prelude::*,
+};
 use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
@@ -16,7 +23,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
-use crate::events::Event;
+use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
@@ -24,6 +31,12 @@ use crate::param::*;
 use crate::pgp;
 use crate::sql::{self, Sql};
 use crate::stock::StockMessage;
+use ::pgp::types::KeyTrait;
+use async_tar::Archive;
+
+// Name of the database file in the backup.
+const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite";
+const BLOBS_BACKUP_NAME: &str = "blobs_backup";
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
 #[repr(i32)]
@@ -42,8 +55,8 @@ pub enum ImexMode {
     /// Export a backup to the directory given as `param1`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
-    /// The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
-    /// the format is `delta-chat.<day>-<number>.bak`
+    /// The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
+    /// the format is `delta-chat-<day>-<number>.tar`
     ExportBackup = 11,
 
     /// `param1` is the file (not: directory) to import. The file is normally
@@ -66,25 +79,83 @@ pub enum ImexMode {
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
-pub async fn imex(
-    context: &Context,
-    what: ImexMode,
-    param1: Option<impl AsRef<Path>>,
-) -> Result<()> {
-    use futures::future::FutureExt;
-
+pub async fn imex(context: &Context, what: ImexMode, param1: impl AsRef<Path>) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
-    let res = imex_inner(context, what, param1)
-        .race(cancel.recv().map(|_| Err(format_err!("canceled"))))
-        .await;
+
+    let res = async {
+        let success = imex_inner(context, what, param1).await;
+        match success {
+            Ok(()) => {
+                info!(context, "IMEX successfully completed");
+                context.emit_event(EventType::ImexProgress(1000));
+                Ok(())
+            }
+            Err(err) => {
+                cleanup_aborted_imex(context, what).await;
+                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                error!(context, "{:#}", err);
+                context.emit_event(EventType::ImexProgress(0));
+                bail!("IMEX FAILED to complete: {}", err);
+            }
+        }
+    }
+    .race(async {
+        cancel.recv().await.ok();
+        cleanup_aborted_imex(context, what).await;
+        Err(format_err!("canceled"))
+    })
+    .await;
 
     context.free_ongoing().await;
 
     res
 }
 
+async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
+    if what == ImexMode::ImportBackup {
+        dc_delete_file(context, context.get_dbfile()).await;
+        dc_delete_files_in_dir(context, context.get_blobdir()).await;
+    }
+    if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
+        if let Err(e) = context.sql.open(context, context.get_dbfile(), false).await {
+            warn!(context, "Re-opening db after imex failed: {}", e);
+        }
+    }
+}
+
 /// Returns the filename of the backup found (otherwise an error)
 pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
+    let dir_name = dir_name.as_ref();
+    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
+    let mut newest_backup_name = "".to_string();
+    let mut newest_backup_path: Option<PathBuf> = None;
+
+    while let Some(dirent) = dir_iter.next().await {
+        if let Ok(dirent) = dirent {
+            let path = dirent.path();
+            let name = dirent.file_name();
+            let name: String = name.to_string_lossy().into();
+            if name.starts_with("delta-chat")
+                && name.ends_with(".tar")
+                && (newest_backup_name.is_empty() || name > newest_backup_name)
+            {
+                // We just use string comparison to determine which backup is newer.
+                // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
+                newest_backup_path = Some(path);
+                newest_backup_name = name;
+            }
+        }
+    }
+
+    match newest_backup_path {
+        Some(path) => Ok(path.to_string_lossy().into_owned()),
+        None => has_backup_old(context, dir_name).await,
+        // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`.
+    }
+}
+
+/// Returns the filename of the backup found (otherwise an error)
+pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
@@ -96,17 +167,23 @@ pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result
             let name = name.to_string_lossy();
             if name.starts_with("delta-chat") && name.ends_with(".bak") {
                 let sql = Sql::new();
-                if sql.open(context, &path, true).await {
-                    let curr_backup_time = sql
-                        .get_raw_config_int(context, "backup_time")
-                        .await
-                        .unwrap_or_default();
-                    if curr_backup_time > newest_backup_time {
-                        newest_backup_path = Some(path);
-                        newest_backup_time = curr_backup_time;
+                match sql.open(context, &path, true).await {
+                    Ok(_) => {
+                        let curr_backup_time = sql
+                            .get_raw_config_int(context, "backup_time")
+                            .await
+                            .unwrap_or_default();
+                        if curr_backup_time > newest_backup_time {
+                            newest_backup_path = Some(path);
+                            newest_backup_time = curr_backup_time;
+                        }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close().await;
                     }
-                    info!(context, "backup_time of {} is {}", name, curr_backup_time);
-                    sql.close().await;
+                    Err(e) => warn!(
+                        context,
+                        "Found backup file {} which could not be opened: {}", name, e
+                    ),
                 }
             }
         }
@@ -150,10 +227,8 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.param.set_int(
-        Param::ForcePlaintext,
-        ForcePlaintext::NoAutocryptHeader as i32,
-    );
+    msg.param.set_int(Param::ForcePlaintext, 1);
+    msg.param.set_int(Param::SkipAutocrypt, 1);
 
     let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
     info!(context, "Wait for setup message being sent ...",);
@@ -178,10 +253,11 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
 ///
 /// The `passphrase` must be at least 2 characters long.
 pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
-    ensure!(
-        passphrase.len() >= 2,
-        "Passphrase must be at least 2 chars long."
-    );
+    let passphrase_begin = if let Some(passphrase_begin) = passphrase.get(..2) {
+        passphrase_begin
+    } else {
+        bail!("Passphrase must be at least 2 chars long.");
+    };
     let private_key = SignedSecretKey::load_self(context).await?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
@@ -196,7 +272,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
             "Passphrase-Format: numeric9x4\r\n",
             "Passphrase-Begin: {}"
         ),
-        &passphrase[..2]
+        passphrase_begin
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
@@ -334,6 +410,8 @@ async fn set_self_key(
         },
     )
     .await?;
+
+    info!(context, "stored self key: {:?}", keypair.secret.key_id());
     Ok(())
 }
 
@@ -360,19 +438,11 @@ pub fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-async fn imex_inner(
-    context: &Context,
-    what: ImexMode,
-    param: Option<impl AsRef<Path>>,
-) -> Result<()> {
-    ensure!(param.is_some(), "No Import/export dir/file given.");
-
-    info!(context, "Import/export process started.");
-    context.emit_event(Event::ImexProgress(10));
-
+async fn imex_inner(context: &Context, what: ImexMode, path: impl AsRef<Path>) -> Result<()> {
+    info!(context, "Import/export dir: {}", path.as_ref().display());
     ensure!(context.sql.is_open().await, "Database not opened.");
+    context.emit_event(EventType::ImexProgress(10));
 
-    let path = param.unwrap();
     if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
         if e2ee::ensure_secret_key_exists(context).await.is_err() {
@@ -382,28 +452,87 @@ async fn imex_inner(
         }
     }
 
-    let success = match what {
+    match what {
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
-        ImexMode::ExportBackup => export_backup(context, path).await,
-        ImexMode::ImportBackup => import_backup(context, path).await,
-    };
 
-    match success {
-        Ok(()) => {
-            info!(context, "IMEX successfully completed");
-            context.emit_event(Event::ImexProgress(1000));
-            Ok(())
-        }
-        Err(err) => {
-            context.emit_event(Event::ImexProgress(0));
-            bail!("IMEX FAILED to complete: {}", err);
-        }
+        // TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old().
+        // (now is 07/2020)
+        ImexMode::ExportBackup => export_backup_old(context, path).await,
+        // import_backup() will call import_backup_old() if this is an old backup.
+        ImexMode::ImportBackup => import_backup(context, path).await,
     }
 }
 
 /// Import Backup
 async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+    if backup_to_import
+        .as_ref()
+        .to_string_lossy()
+        .ends_with(".bak")
+    {
+        // Backwards compability
+        return import_backup_old(context, backup_to_import).await;
+    }
+
+    info!(
+        context,
+        "Import \"{}\" to \"{}\".",
+        backup_to_import.as_ref().display(),
+        context.get_dbfile().display()
+    );
+
+    ensure!(
+        !context.is_configured().await,
+        "Cannot import backups to accounts in use."
+    );
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
+    ensure!(
+        !context.get_dbfile().exists().await,
+        "Cannot delete old database."
+    );
+
+    let backup_file = File::open(backup_to_import).await?;
+    let archive = Archive::new(backup_file);
+    let mut entries = archive.entries()?;
+    while let Some(file) = entries.next().await {
+        let f = &mut file?;
+        if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
+            // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
+            f.unpack_in(context.get_blobdir()).await?;
+            fs::rename(
+                context.get_blobdir().join(DBFILE_BACKUP_NAME),
+                context.get_dbfile(),
+            )
+            .await?;
+            context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
+        } else {
+            // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
+            f.unpack_in(context.get_blobdir()).await?;
+            let from_path = context.get_blobdir().join(f.path()?);
+            if from_path.is_file().await {
+                if let Some(name) = from_path.file_name() {
+                    fs::rename(&from_path, context.get_blobdir().join(name)).await?;
+                } else {
+                    warn!(context, "No file name");
+                }
+            }
+        }
+    }
+
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
+
+    delete_and_reset_all_device_msgs(&context).await?;
+
+    Ok(())
+}
+
+async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
     info!(
         context,
         "Import \"{}\" to \"{}\".",
@@ -428,13 +557,11 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     );
     /* error already logged */
     /* re-open copied database file */
-    ensure!(
-        context
-            .sql
-            .open(&context, &context.get_dbfile(), false)
-            .await,
-        "could not re-open db"
-    );
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
 
     delete_and_reset_all_device_msgs(&context).await?;
 
@@ -448,27 +575,33 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
     );
 
-    let files = context
+    // Load IDs only for now, without the file contents, to avoid
+    // consuming too much memory.
+    let file_ids = context
         .sql
         .query_map(
-            "SELECT file_name, file_content FROM backup_blobs ORDER BY id;",
+            "SELECT id FROM backup_blobs ORDER BY id",
             paramsv![],
-            |row| {
-                let name: String = row.get(0)?;
-                let blob: Vec<u8> = row.get(1)?;
-
-                Ok((name, blob))
-            },
-            |files| {
-                files
-                    .collect::<std::result::Result<Vec<_>, _>>()
+            |row| row.get(0),
+            |ids| {
+                ids.collect::<std::result::Result<Vec<i64>, _>>()
                     .map_err(Into::into)
             },
         )
         .await?;
 
     let mut all_files_extracted = true;
-    for (processed_files_cnt, (file_name, file_blob)) in files.into_iter().enumerate() {
+    for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
+        // Load a single blob into memory
+        let (file_name, file_blob) = context
+            .sql
+            .query_row(
+                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
+                paramsv![file_id],
+                |row| Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?)),
+            )
+            .await?;
+
         if context.shall_stop_ongoing().await {
             all_files_extracted = false;
             break;
@@ -480,7 +613,7 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         if permille > 990 {
             permille = 990
         }
-        context.emit_event(Event::ImexProgress(permille));
+        context.emit_event(EventType::ImexProgress(permille));
         if file_blob.is_empty() {
             continue;
         }
@@ -505,14 +638,90 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
 /*******************************************************************************
  * Export backup
  ******************************************************************************/
-/* the FILE_PROGRESS macro calls the callback with the permille of files processed.
-The macro avoids weird values of 0% or 100% while still working. */
+#[allow(unused)]
 async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
+    let now = time();
+    let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+    let _d = DeleteOnDrop(temp_path.clone());
+
+    context
+        .sql
+        .set_raw_config_int(context, "backup_time", now as i32)
+        .await?;
+    sql::housekeeping(context).await;
+
+    context
+        .sql
+        .execute("VACUUM;", paramsv![])
+        .await
+        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
+
+    // we close the database during the export
+    context.sql.close().await;
+
+    info!(
+        context,
+        "Backup '{}' to '{}'.",
+        context.get_dbfile().display(),
+        dest_path.display(),
+    );
+
+    let res = export_backup_inner(context, &temp_path).await;
+
+    // we re-open the database after export is finished
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await;
+
+    match &res {
+        Ok(_) => {
+            fs::rename(temp_path, &dest_path).await?;
+            context.emit_event(EventType::ImexFileWritten(dest_path));
+        }
+        Err(e) => {
+            error!(context, "backup failed: {}", e);
+        }
+    }
+
+    res
+}
+struct DeleteOnDrop(PathBuf);
+impl Drop for DeleteOnDrop {
+    fn drop(&mut self) {
+        let file = self.0.clone();
+        // Not using dc_delete_file() here because it would send a DeletedBlobFile event
+        async_std::task::block_on(async move { fs::remove_file(file).await.ok() });
+    }
+}
+
+async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
+    let file = File::create(temp_path).await?;
+
+    let mut builder = async_tar::Builder::new(file);
+
+    // append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument.
+    builder
+        .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
+        .await?;
+
+    context.emit_event(EventType::ImexProgress(500));
+
+    builder
+        .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
+        .await?;
+
+    builder.finish().await?;
+    Ok(())
+}
+
+async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
     // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
     let now = time();
-    let dest_path_filename = dc_get_next_backup_path(dir, now).await?;
+    let dest_path_filename = get_next_backup_path_old(dir, now).await?;
     let dest_path_string = dest_path_filename.to_string_lossy().to_string();
 
     sql::housekeeping(context).await;
@@ -531,7 +740,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     context
         .sql
         .open(&context, &context.get_dbfile(), false)
-        .await;
+        .await?;
 
     if !copied {
         bail!(
@@ -541,11 +750,11 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         );
     }
     let dest_sql = Sql::new();
-    ensure!(
-        dest_sql.open(context, &dest_path_filename, false).await,
-        "could not open exported database {}",
-        dest_path_string
-    );
+    dest_sql
+        .open(context, &dest_path_filename, false)
+        .await
+        .with_context(|| format!("could not open exported database {}", dest_path_string))?;
+
     let res = match add_files_to_export(context, &dest_sql).await {
         Err(err) => {
             dc_delete_file(context, &dest_path_filename).await;
@@ -556,7 +765,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
             dest_sql
                 .set_raw_config_int(context, "backup_time", now as i32)
                 .await?;
-            context.emit_event(Event::ImexFileWritten(dest_path_filename));
+            context.emit_event(EventType::ImexFileWritten(dest_path_filename));
             Ok(())
         }
     };
@@ -595,7 +804,7 @@ async fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
             }
             processed_files_cnt += 1;
             let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
-            context.emit_event(Event::ImexProgress(permille));
+            context.emit_event(EventType::ImexProgress(permille));
 
             let name_f = entry.file_name();
             let name = name_f.to_string_lossy();
@@ -657,6 +866,12 @@ async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
                 continue;
             }
         }
+        info!(
+            context,
+            "considering key file: {}",
+            path_plus_name.display()
+        );
+
         match dc_read_file(context, &path_plus_name).await {
             Ok(buf) => {
                 let armored = std::string::String::from_utf8_lossy(&buf);
@@ -746,7 +961,7 @@ where
         let any_key = key as &dyn Any;
         let kind = if any_key.downcast_ref::<SignedPublicKey>().is_some() {
             "public"
-        } else if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+        } else if any_key.downcast_ref::<SignedSecretKey>().is_some() {
             "private"
         } else {
             "unknown"
@@ -754,7 +969,12 @@ where
         let id = id.map_or("default".into(), |i| i.to_string());
         dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
     };
-    info!(context, "Exporting key {}", file_name.display());
+    info!(
+        context,
+        "Exporting key {:?} to {}",
+        key.key_id(),
+        file_name.display()
+    );
     dc_delete_file(context, &file_name).await;
 
     let content = key.to_asc(None).into_bytes();
@@ -762,7 +982,7 @@ where
     if res.is_err() {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
-        context.emit_event(Event::ImexFileWritten(file_name));
+        context.emit_event(EventType::ImexFileWritten(file_name));
     }
     res
 }
@@ -822,7 +1042,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_export_key_to_asc_file() {
+    async fn test_export_public_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
@@ -836,6 +1056,37 @@ mod tests {
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
+    #[async_std::test]
+    async fn test_export_private_key_to_asc_file() {
+        let context = TestContext::new().await;
+        let key = alice_keypair().secret;
+        let blobdir = "$BLOBDIR";
+        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
+            .await
+            .is_ok());
+        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        let filename = format!("{}/private-key-default.asc", blobdir);
+        let bytes = async_std::fs::read(&filename).await.unwrap();
+
+        assert_eq!(bytes, key.to_asc(None).into_bytes());
+    }
+
+    #[async_std::test]
+    async fn test_export_and_import_key() {
+        let context = TestContext::new().await;
+        context.configure_alice().await;
+        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir).await {
+            panic!("got error on export: {:?}", err);
+        }
+
+        let context2 = TestContext::new().await;
+        context2.configure_alice().await;
+        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir).await {
+            panic!("got error on import: {:?}", err);
+        }
+    }
+
     #[test]
     fn test_normalize_setup_code() {
         let norm = normalize_setup_code("123422343234423452346234723482349234");

src/job.rs

@@ -14,23 +14,22 @@ use async_smtp::smtp::response::Category;
 use async_smtp::smtp::response::Code;
 use async_smtp::smtp::response::Detail;
 
-use crate::blob::BlobObject;
 use crate::chat::{self, ChatId};
 use crate::config::Config;
-use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::load_imap_deletion_msgid;
 use crate::error::{bail, ensure, format_err, Error, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::imap::*;
 use crate::location;
-use crate::login_param::LoginParam;
 use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
+use crate::{blob::BlobObject, contact::normalize_name, contact::Modifier, contact::Origin};
 use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
@@ -93,8 +92,7 @@ pub enum Action {
 
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
-    EmptyServer = 107,
-    OldDeleteMsgOnImap = 110,
+    FetchExistingMsgs = 110,
     MarkseenMsgOnImap = 130,
 
     // Moving message is prioritized lower than deletion so we don't
@@ -102,6 +100,10 @@ pub enum Action {
     MoveMsg = 200,
     DeleteMsgOnImap = 210,
 
+    // UID synchronization is high-priority to make sure correct UIDs
+    // are used by message moving/deletion.
+    ResyncFolders = 300,
+
     // Jobs in the SMTP-thread, range from DC_SMTP_THREAD..DC_SMTP_THREAD+999
     MaybeSendLocations = 5005, // low priority ...
     MaybeSendLocationsEnded = 5007,
@@ -123,9 +125,9 @@ impl From<Action> for Thread {
             Unknown => Thread::Unknown,
 
             Housekeeping => Thread::Imap,
-            OldDeleteMsgOnImap => Thread::Imap,
+            FetchExistingMsgs => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
-            EmptyServer => Thread::Imap,
+            ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
 
@@ -254,40 +256,48 @@ impl Job {
 
                 let res = match err {
                     async_smtp::smtp::error::Error::Permanent(ref response) => {
-                        match response.code {
+                        // Workaround for incorrectly configured servers returning permanent errors
+                        // instead of temporary ones.
+                        let maybe_transient = match response.code {
                             // Sometimes servers send a permanent error when actually it is a temporary error
                             // For documentation see https://tools.ietf.org/html/rfc3463
-
-                            // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
                             Code {
                                 category: Category::MailSystem,
                                 detail: Detail::Zero,
                                 ..
-                            } => Status::RetryLater,
-
-                            _ => {
-                                // If we do not retry, add an info message to the chat
-                                // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM.
-                                match Message::load_from_db(context, MsgId::new(self.foreign_id))
-                                    .await
-                                {
-                                    Ok(message) => {
-                                        chat::add_info_msg(
-                                            context,
-                                            message.chat_id,
-                                            err.to_string(),
-                                        )
-                                        .await
-                                    }
-                                    Err(e) => warn!(
-                                        context,
-                                        "couldn't load chat_id to inform user about SMTP error: {}",
-                                        e
-                                    ),
-                                };
-
-                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } => {
+                                // Ignore status code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
+                                // Maybe incorrectly configured Postfix milter with "reject" instead of "tempfail", which returns
+                                // "550 5.5.0 Service unavailable" instead of "451 4.7.1 Service unavailable - try again later".
+                                //
+                                // Other enhanced status codes, such as Postfix
+                                // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
+                                // are not ignored.
+                                response.message.get(0) == Some(&"5.5.0".to_string())
                             }
+                            _ => false,
+                        };
+
+                        if maybe_transient {
+                            Status::RetryLater
+                        } else {
+                            // If we do not retry, add an info message to the chat.
+                            // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
+                            // should definitely go here, because user has to open the link to
+                            // resume message sending.
+                            let msg_id = MsgId::new(self.foreign_id);
+                            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+                            match Message::load_from_db(context, msg_id).await {
+                                Ok(message) => {
+                                    chat::add_info_msg(context, message.chat_id, err.to_string())
+                                        .await
+                                }
+                                Err(e) => error!(
+                                    context,
+                                    "couldn't load chat_id to inform user about SMTP error: {}", e
+                                ),
+                            };
+                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                         }
                     }
                     async_smtp::smtp::error::Error::Transient(_) => {
@@ -331,12 +341,9 @@ impl Job {
 
     pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
         //  SMTP server, if not yet done
-        if !smtp.is_connected().await {
-            let loginparam = LoginParam::from_database(context, "configured_").await;
-            if let Err(err) = smtp.connect(context, &loginparam).await {
-                warn!(context, "SMTP connection failure: {:?}", err);
-                return Status::RetryLater;
-            }
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            return Status::RetryLater;
         }
 
         let filename = job_try!(job_try!(self
@@ -479,12 +486,9 @@ impl Job {
         let recipients = vec![recipient];
 
         // connect to SMTP server, if not yet done
-        if !smtp.is_connected().await {
-            let loginparam = LoginParam::from_database(context, "configured_").await;
-            if let Err(err) = smtp.connect(context, &loginparam).await {
-                warn!(context, "SMTP connection failure: {:?}", err);
-                return Status::RetryLater;
-            }
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            return Status::RetryLater;
         }
 
         self.smtp_send(context, recipients, body, self.job_id, smtp, || {
@@ -558,6 +562,11 @@ impl Job {
                     context,
                     "The message is deleted from the server when all parts are deleted.",
                 );
+            } else if cnt == 0 {
+                warn!(
+                    context,
+                    "The message {} has no UID on the server to delete", &msg.rfc724_mid
+                );
             } else {
                 /* if this is the last existing part of the message,
                 we delete the message from the server */
@@ -612,19 +621,77 @@ impl Job {
         }
     }
 
-    async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
+    /// Read the recipients from old emails sent by the user and add them as contacts.
+    /// This way, we can already offer them some email addresses they can write to.
+    ///
+    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
+    /// and show them in the chat list.
+    async fn fetch_existing_msgs(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if context.get_config_bool(Config::Bot).await {
+            return Status::Finished(Ok(())); // Bots don't want those messages
+        }
         if let Err(err) = imap.connect_configured(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
 
-        if self.foreign_id & DC_EMPTY_MVBOX > 0 {
-            if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
-                imap.empty_folder(context, &mvbox_folder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredSentboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredMvboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredInboxFolder).await;
+
+        if context.get_config_bool(Config::FetchExistingMsgs).await {
+            for config in &[
+                Config::ConfiguredMvboxFolder,
+                Config::ConfiguredInboxFolder,
+                Config::ConfiguredSentboxFolder,
+            ] {
+                if let Some(folder) = context.get_config(*config).await {
+                    if let Err(e) = imap.fetch_new_messages(context, folder, true).await {
+                        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                        warn!(context, "Could not fetch messages, retrying: {:#}", e);
+                        return Status::RetryLater;
+                    };
+                }
             }
         }
-        if self.foreign_id & DC_EMPTY_INBOX > 0 {
-            imap.empty_folder(context, "INBOX").await;
+        info!(context, "Done fetching existing messages.");
+        Status::Finished(Ok(()))
+    }
+
+    /// Synchronizes UIDs for sentbox, inbox and mvbox, in this order.
+    ///
+    /// If a copy of the message is present in multiple folders, mvbox
+    /// is preferred to inbox, which is in turn preferred to
+    /// sentbox. This is because in the database it is impossible to
+    /// store multiple UIDs for one message, so we prefer to
+    /// automatically delete messages in the folders managed by Delta
+    /// Chat in contrast to the Sent folder, which is normally managed
+    /// by the user via webmail or another email client.
+    async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        if let Some(sentbox_folder) = &context.get_config(Config::ConfiguredSentboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, sentbox_folder.to_string())
+                    .await
+            );
+        }
+
+        if let Some(inbox_folder) = &context.get_config(Config::ConfiguredInboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, inbox_folder.to_string())
+                    .await
+            );
+        }
+
+        if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, mvbox_folder.to_string())
+                    .await
+            );
         }
         Status::Finished(Ok(()))
     }
@@ -638,7 +705,21 @@ impl Job {
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         let folder = msg.server_folder.as_ref().unwrap();
-        match imap.set_seen(context, folder, msg.server_uid).await {
+
+        let result = if msg.server_uid == 0 {
+            // The message is moved or deleted by us.
+            //
+            // Do not call set_seen with zero UID, as it will return
+            // ImapActionResult::RetryLater, but we do not want to
+            // retry. If the message was moved, we will create another
+            // job to mark the message as seen later. If it was
+            // deleted, there is nothing to do.
+            ImapActionResult::Failed
+        } else {
+            imap.set_seen(context, folder, msg.server_uid).await
+        };
+
+        match result {
             ImapActionResult::RetryLater => Status::RetryLater,
             ImapActionResult::AlreadyDone => Status::Finished(Ok(())),
             ImapActionResult::Success | ImapActionResult::Failed => {
@@ -646,7 +727,18 @@ impl Job {
                 // we want to send out an MDN anyway
                 // The job will not be retried so locally
                 // there is no risk of double-sending MDNs.
+                //
+                // Read receipts for system messages are never
+                // sent. These messages have no place to display
+                // received read receipt anyway.  And since their text
+                // is locally generated, quoting them is dangerous as
+                // it may contain contact names. E.g., for original
+                // message "Group left by me", a read receipt will
+                // quote "Group left by <name>", and the name can be a
+                // display name stored in address book rather than
+                // the name sent in the From field by the user.
                 if msg.param.get_bool(Param::WantsMdn).unwrap_or_default()
+                    && !msg.is_system_message()
                     && context.get_config_bool(Config::MdnsEnabled).await
                 {
                     if let Err(err) = send_mdn(context, &msg).await {
@@ -703,7 +795,51 @@ async fn set_delivered(context: &Context, msg_id: MsgId) {
         )
         .await
         .unwrap_or_default();
-    context.emit_event(Event::MsgDelivered { chat_id, msg_id });
+    context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
+}
+
+async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
+    let mailbox = if let Some(m) = context.get_config(folder).await {
+        m
+    } else {
+        return;
+    };
+    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
+        warn!(context, "Could not select {}: {}", mailbox, e);
+        return;
+    }
+    match imap.get_all_recipients(context).await {
+        Ok(contacts) => {
+            let mut any_modified = false;
+            for contact in contacts {
+                let display_name_normalized = contact
+                    .display_name
+                    .as_ref()
+                    .map(normalize_name)
+                    .unwrap_or_default();
+
+                match Contact::add_or_lookup(
+                    context,
+                    display_name_normalized,
+                    contact.addr,
+                    Origin::OutgoingTo,
+                )
+                .await
+                {
+                    Ok((_, modified)) => {
+                        if modified != Modifier::None {
+                            any_modified = true;
+                        }
+                    }
+                    Err(e) => warn!(context, "Could not add recipient: {}", e),
+                }
+            }
+            if any_modified {
+                context.emit_event(EventType::ContactsChanged(None));
+            }
+        }
+        Err(e) => warn!(context, "Could not add recipients: {}", e),
+    };
 }
 
 /// Constructs a job for sending a message.
@@ -828,25 +964,6 @@ pub(crate) enum Connection<'a> {
     Smtp(&'a mut Smtp),
 }
 
-async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
-    if let Some(delete_server_after) = context.get_config_delete_server_after().await {
-        let threshold_timestamp = time() - delete_server_after;
-
-        context
-            .sql
-            .query_row_optional(
-                "SELECT id FROM msgs \
-             WHERE timestamp < ? \
-             AND server_uid != 0",
-                paramsv![threshold_timestamp],
-                |row| row.get::<_, MsgId>(0),
-            )
-            .await
-    } else {
-        Ok(None)
-    }
-}
-
 async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
     let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
         Some(Job::new(
@@ -969,21 +1086,18 @@ async fn perform_job_action(
         Action::MaybeSendLocationsEnded => {
             location::job_maybe_send_locations_ended(context, job).await
         }
-        Action::EmptyServer => job.empty_server(context, connection.inbox()).await,
-        Action::OldDeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
+        Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
+        Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::Housekeeping => {
             sql::housekeeping(context).await;
             Status::Finished(Ok(()))
         }
     };
 
-    info!(
-        context,
-        "Inbox finished immediate try {} of job {}", tries, job
-    );
+    info!(context, "Finished immediate try {} of job {}", tries, job);
 
     try_res
 }
@@ -1008,6 +1122,15 @@ async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     Ok(())
 }
 
+pub(crate) async fn schedule_resync(context: &Context) {
+    kill_action(context, Action::ResyncFolders).await;
+    add(
+        context,
+        Job::new(Action::ResyncFolders, 0, Params::new(), 0),
+    )
+    .await;
+}
+
 /// Creates a job.
 pub fn create(action: Action, foreign_id: i32, param: Params, delay_seconds: i64) -> Result<Job> {
     ensure!(
@@ -1030,10 +1153,10 @@ pub async fn add(context: &Context, job: Job) {
         match action {
             Action::Unknown => unreachable!(),
             Action::Housekeeping
-            | Action::EmptyServer
-            | Action::OldDeleteMsgOnImap
             | Action::DeleteMsgOnImap
+            | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
+            | Action::FetchExistingMsgs
             | Action::MoveMsg => {
                 info!(context, "interrupt: imap");
                 context

src/key.rs

@@ -222,7 +222,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     let addr = context
         .get_config(Config::ConfiguredAddr)
         .await
-        .ok_or_else(|| Error::NoConfiguredAddr)?;
+        .ok_or(Error::NoConfiguredAddr)?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -247,7 +247,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             secret: SignedSecretKey::from_slice(&sec_bytes)?,
         }),
         Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-            let start = std::time::Instant::now();
+            let start = std::time::SystemTime::now();
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
@@ -258,7 +258,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             info!(
                 context,
                 "Keypair generated in {:.3}s.",
-                start.elapsed().as_secs()
+                start.elapsed().unwrap_or_default().as_secs()
             );
             Ok(keypair)
         }
@@ -355,7 +355,7 @@ pub async fn store_self_keypair(
 }
 
 /// A key fingerprint
-#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+#[derive(Clone, Eq, PartialEq, Hash)]
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {
@@ -375,6 +375,14 @@ impl Fingerprint {
     }
 }
 
+impl fmt::Debug for Fingerprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Fingerprint")
+            .field("hex", &self.hex())
+            .finish()
+    }
+}
+
 /// Make a human-readable fingerprint.
 impl fmt::Display for Fingerprint {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -423,11 +431,9 @@ mod tests {
     use std::error::Error;
 
     use async_std::sync::Arc;
-    use lazy_static::lazy_static;
+    use once_cell::sync::Lazy;
 
-    lazy_static! {
-        static ref KEYPAIR: KeyPair = alice_keypair();
-    }
+    static KEYPAIR: Lazy<KeyPair> = Lazy::new(alice_keypair);
 
     #[test]
     fn test_from_armored_string() {
@@ -529,11 +535,11 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[test]
     fn test_from_slice_bad_data() {
         let mut bad_data: [u8; 4096] = [0; 4096];
-        for i in 0..4096 {
-            bad_data[i] = (i & 0xff) as u8;
+        for (i, v) in bad_data.iter_mut().enumerate() {
+            *v = (i & 0xff) as u8;
         }
         for j in 0..(4096 / 40) {
-            let slice = &bad_data[j..j + 4096 / 2 + j];
+            let slice = &bad_data.get(j..j + 4096 / 2 + j).unwrap();
             assert!(SignedPublicKey::from_slice(slice).is_err());
             assert!(SignedSecretKey::from_slice(slice).is_err());
         }
@@ -593,7 +599,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         let ctx0 = ctx.clone();
         let thr0 =
             thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
-        let ctx1 = ctx.clone();
+        let ctx1 = ctx;
         let thr1 =
             thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
         let res0 = thr0.join().unwrap();

src/lib.rs

@@ -1,6 +1,11 @@
 #![forbid(unsafe_code)]
-#![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-#![allow(clippy::match_bool)]
+#![deny(
+    clippy::correctness,
+    missing_debug_implementations,
+    clippy::all,
+    clippy::indexing_slicing
+)]
+#![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
 #[macro_use]
 extern crate num_derive;
@@ -43,11 +48,13 @@ pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
+pub mod ephemeral;
 mod imap;
 pub mod imex;
 mod scheduler;
 #[macro_use]
 pub mod job;
+mod format_flowed;
 pub mod key;
 mod keyring;
 pub mod location;
@@ -73,6 +80,8 @@ mod dehtml;
 pub mod dc_receive_imf;
 pub mod dc_tools;
 
+pub mod accounts;
+
 /// if set imap/incoming and smtp/outgoing MIME messages will be printed
 pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
 

src/location.rs

@@ -9,7 +9,7 @@ use crate::constants::*;
 use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::{ensure, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
@@ -227,7 +227,7 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
                     .await;
                 chat::add_info_msg(context, chat_id, stock_str).await;
             }
-            context.emit_event(Event::ChatModified(chat_id));
+            context.emit_event(EventType::ChatModified(chat_id));
             if 0 != seconds {
                 schedule_maybe_send_locations(context, false).await;
                 job::add(
@@ -301,7 +301,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
             }
         }
         if continue_streaming {
-            context.emit_event(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
         schedule_maybe_send_locations(context, false).await;
     }
@@ -381,7 +381,7 @@ pub async fn delete_all(context: &Context) -> Result<(), Error> {
         .sql
         .execute("DELETE FROM locations;", paramsv![])
         .await?;
-    context.emit_event(Event::LocationChanged(None));
+    context.emit_event(EventType::LocationChanged(None));
     Ok(())
 }
 
@@ -469,7 +469,7 @@ pub fn get_message_kml(timestamp: i64, latitude: f64, longitude: f64) -> String
          <Document>\n\
          <Placemark>\
          <Timestamp><when>{}</when></Timestamp>\
-         <Point><coordinates>{:.2},{:.2}</coordinates></Point>\
+         <Point><coordinates>{},{}</coordinates></Point>\
          </Placemark>\n\
          </Document>\n\
          </kml>",
@@ -715,7 +715,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
                 .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
                 .await;
             chat::add_info_msg(context, chat_id, stock_str).await;
-            context.emit_event(Event::ChatModified(chat_id));
+            context.emit_event(EventType::ChatModified(chat_id));
         }
     }
     job::Status::Finished(Ok(()))
@@ -723,6 +723,8 @@ pub(crate) async fn job_maybe_send_locations_ended(
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use crate::test_utils::TestContext;
 
@@ -757,4 +759,22 @@ mod tests {
         assert!(locations_ref[1].accuracy < 2.6f64);
         assert_eq!(locations_ref[1].timestamp, 1544739072);
     }
+
+    #[async_std::test]
+    async fn test_get_message_kml() {
+        let context = TestContext::new().await;
+        let timestamp = 1598490000;
+
+        let xml = get_message_kml(timestamp, 51.423723f64, 8.552556f64);
+        let kml = Kml::parse(&context.ctx, xml.as_bytes()).expect("parsing failed");
+        let locations_ref = &kml.locations;
+        assert_eq!(locations_ref.len(), 1);
+
+        assert!(locations_ref[0].latitude >= 51.423723f64);
+        assert!(locations_ref[0].latitude < 51.423724f64);
+        assert!(locations_ref[0].longitude >= 8.552556f64);
+        assert!(locations_ref[0].longitude < 8.552557f64);
+        assert!(locations_ref[0].accuracy.abs() < f64::EPSILON);
+        assert_eq!(locations_ref[0].timestamp, timestamp);
+    }
 }

src/log.rs

@@ -11,7 +11,7 @@ macro_rules! info {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::Event::Info(full));
+        emit_event!($ctx, $crate::EventType::Info(full));
     }};
 }
 
@@ -26,7 +26,7 @@ macro_rules! warn {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::Event::Warning(full));
+        emit_event!($ctx, $crate::EventType::Warning(full));
     }};
 }
 
@@ -37,7 +37,18 @@ macro_rules! error {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        emit_event!($ctx, $crate::Event::Error(formatted));
+        emit_event!($ctx, $crate::EventType::Error(formatted));
+    }};
+}
+
+#[macro_export]
+macro_rules! error_network {
+    ($ctx:expr, $msg:expr) => {
+        error_network!($ctx, $msg,)
+    };
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
+        let formatted = format!($msg, $($args),*);
+        emit_event!($ctx, $crate::EventType::ErrorNetwork(formatted));
     }};
 }
 

src/login_param.rs

@@ -3,13 +3,16 @@
 use std::borrow::Cow;
 use std::fmt;
 
-use crate::context::Context;
+use crate::{context::Context, provider::Socket};
 
-#[derive(Copy, Clone, Debug, Display, FromPrimitive)]
+#[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(i32)]
 #[strum(serialize_all = "snake_case")]
 pub enum CertificateChecks {
+    /// Same as AcceptInvalidCertificates unless overridden by
+    /// `strict_tls` setting in provider database.
     Automatic = 0,
+
     Strict = 1,
 
     /// Same as AcceptInvalidCertificates
@@ -25,30 +28,29 @@ impl Default for CertificateChecks {
     }
 }
 
-#[derive(Default, Debug)]
+/// Login parameters for a single server, either IMAP or SMTP
+#[derive(Default, Debug, Clone)]
+pub struct ServerLoginParam {
+    pub server: String,
+    pub user: String,
+    pub password: String,
+    pub port: u16,
+    pub security: Socket,
+
+    /// TLS options: whether to allow invalid certificates and/or
+    /// invalid hostnames
+    pub certificate_checks: CertificateChecks,
+}
+
+#[derive(Default, Debug, Clone)]
 pub struct LoginParam {
     pub addr: String,
-    pub mail_server: String,
-    pub mail_user: String,
-    pub mail_pw: String,
-    pub mail_port: i32,
-    /// IMAP TLS options: whether to allow invalid certificates and/or invalid hostnames
-    pub imap_certificate_checks: CertificateChecks,
-    pub send_server: String,
-    pub send_user: String,
-    pub send_pw: String,
-    pub send_port: i32,
-    /// SMTP TLS options: whether to allow invalid certificates and/or invalid hostnames
-    pub smtp_certificate_checks: CertificateChecks,
+    pub imap: ServerLoginParam,
+    pub smtp: ServerLoginParam,
     pub server_flags: i32,
 }
 
 impl LoginParam {
-    /// Create a new `LoginParam` with default values.
-    pub fn new() -> Self {
-        Default::default()
-    }
-
     /// Read the login parameters from the database.
     pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Self {
         let prefix = prefix.as_ref();
@@ -77,6 +79,13 @@ impl LoginParam {
         let key = format!("{}mail_pw", prefix);
         let mail_pw = sql.get_raw_config(context, key).await.unwrap_or_default();
 
+        let key = format!("{}mail_security", prefix);
+        let mail_security = sql
+            .get_raw_config_int(context, key)
+            .await
+            .and_then(num_traits::FromPrimitive::from_i32)
+            .unwrap_or_default();
+
         let key = format!("{}imap_certificate_checks", prefix);
         let imap_certificate_checks =
             if let Some(certificate_checks) = sql.get_raw_config_int(context, key).await {
@@ -100,6 +109,13 @@ impl LoginParam {
         let key = format!("{}send_pw", prefix);
         let send_pw = sql.get_raw_config(context, key).await.unwrap_or_default();
 
+        let key = format!("{}send_security", prefix);
+        let send_security = sql
+            .get_raw_config_int(context, key)
+            .await
+            .and_then(num_traits::FromPrimitive::from_i32)
+            .unwrap_or_default();
+
         let key = format!("{}smtp_certificate_checks", prefix);
         let smtp_certificate_checks =
             if let Some(certificate_checks) = sql.get_raw_config_int(context, key).await {
@@ -116,16 +132,22 @@ impl LoginParam {
 
         LoginParam {
             addr,
-            mail_server,
-            mail_user,
-            mail_pw,
-            mail_port,
-            imap_certificate_checks,
-            send_server,
-            send_user,
-            send_pw,
-            send_port,
-            smtp_certificate_checks,
+            imap: ServerLoginParam {
+                server: mail_server,
+                user: mail_user,
+                password: mail_pw,
+                port: mail_port as u16,
+                security: mail_security,
+                certificate_checks: imap_certificate_checks,
+            },
+            smtp: ServerLoginParam {
+                server: send_server,
+                user: send_user,
+                password: send_pw,
+                port: send_port as u16,
+                security: send_security,
+                certificate_checks: smtp_certificate_checks,
+            },
             server_flags,
         }
     }
@@ -143,41 +165,51 @@ impl LoginParam {
         sql.set_raw_config(context, key, Some(&self.addr)).await?;
 
         let key = format!("{}mail_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_server))
+        sql.set_raw_config(context, key, Some(&self.imap.server))
             .await?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_raw_config_int(context, key, self.mail_port).await?;
+        sql.set_raw_config_int(context, key, self.imap.port as i32)
+            .await?;
 
         let key = format!("{}mail_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_user))
+        sql.set_raw_config(context, key, Some(&self.imap.user))
             .await?;
 
         let key = format!("{}mail_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_pw))
+        sql.set_raw_config(context, key, Some(&self.imap.password))
+            .await?;
+
+        let key = format!("{}mail_security", prefix);
+        sql.set_raw_config_int(context, key, self.imap.security as i32)
             .await?;
 
         let key = format!("{}imap_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)
+        sql.set_raw_config_int(context, key, self.imap.certificate_checks as i32)
             .await?;
 
         let key = format!("{}send_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_server))
+        sql.set_raw_config(context, key, Some(&self.smtp.server))
             .await?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_raw_config_int(context, key, self.send_port).await?;
+        sql.set_raw_config_int(context, key, self.smtp.port as i32)
+            .await?;
 
         let key = format!("{}send_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_user))
+        sql.set_raw_config(context, key, Some(&self.smtp.user))
             .await?;
 
         let key = format!("{}send_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_pw))
+        sql.set_raw_config(context, key, Some(&self.smtp.password))
+            .await?;
+
+        let key = format!("{}send_security", prefix);
+        sql.set_raw_config_int(context, key, self.smtp.security as i32)
             .await?;
 
         let key = format!("{}smtp_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)
+        sql.set_raw_config_int(context, key, self.smtp.certificate_checks as i32)
             .await?;
 
         let key = format!("{}server_flags", prefix);
@@ -199,16 +231,24 @@ impl fmt::Display for LoginParam {
             f,
             "{} imap:{}:{}:{}:{}:cert_{} smtp:{}:{}:{}:{}:cert_{} {}",
             unset_empty(&self.addr),
-            unset_empty(&self.mail_user),
-            if !self.mail_pw.is_empty() { pw } else { unset },
-            unset_empty(&self.mail_server),
-            self.mail_port,
-            self.imap_certificate_checks,
-            unset_empty(&self.send_user),
-            if !self.send_pw.is_empty() { pw } else { unset },
-            unset_empty(&self.send_server),
-            self.send_port,
-            self.smtp_certificate_checks,
+            unset_empty(&self.imap.user),
+            if !self.imap.password.is_empty() {
+                pw
+            } else {
+                unset
+            },
+            unset_empty(&self.imap.server),
+            self.imap.port,
+            self.imap.certificate_checks,
+            unset_empty(&self.smtp.user),
+            if !self.smtp.password.is_empty() {
+                pw
+            } else {
+                unset
+            },
+            unset_empty(&self.smtp.server),
+            self.smtp.port,
+            self.smtp.certificate_checks,
             flags_readable,
         )
     }
@@ -237,30 +277,6 @@ fn get_readable_flags(flags: i32) -> String {
                 res += "AUTH_NORMAL ";
                 flag_added = true;
             }
-            if 1 << bit == 0x100 {
-                res += "IMAP_STARTTLS ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x200 {
-                res += "IMAP_SSL ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x400 {
-                res += "IMAP_PLAIN ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x10000 {
-                res += "SMTP_STARTTLS ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x20000 {
-                res += "SMTP_SSL ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x40000 {
-                res += "SMTP_PLAIN ";
-                flag_added = true;
-            }
             if flag_added {
                 res += &format!("{:#0x}", 1 << bit);
             }

src/lot.rs

@@ -91,6 +91,9 @@ pub enum LotState {
     /// text1=domain
     QrAccount = 250,
 
+    /// text1=domain, text2=instance pattern
+    QrWebrtcInstance = 260,
+
     /// id=contact
     QrAddr = 320,
 

src/message.rs

@@ -2,26 +2,24 @@
 
 use async_std::path::{Path, PathBuf};
 use deltachat_derive::{FromSql, ToSql};
-use lazy_static::lazy_static;
+use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
+use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::{ensure, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::job::{self, Action};
 use crate::lot::{Lot, LotState, Meaning};
 use crate::mimeparser::{FailureReport, SystemMessage};
 use crate::param::*;
 use crate::pgp::*;
 use crate::stock::StockMessage;
-
-lazy_static! {
-    static ref UNWRAP_RE: regex::Regex = regex::Regex::new(r"\s+").unwrap();
-}
+use std::collections::BTreeMap;
 
 // In practice, the user additionally cuts the string themselves
 // pixel-accurate.
@@ -68,18 +66,38 @@ impl MsgId {
         self.0 == 0
     }
 
-    /// Whether the message ID is the special marker1 marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_marker1(self) -> bool {
-        self.0 == DC_MSG_ID_MARKER1
+    /// Returns message state.
+    pub async fn get_state(self, context: &Context) -> crate::sql::Result<MessageState> {
+        let result = context
+            .sql
+            .query_get_value_result("SELECT state FROM msgs WHERE id=?", paramsv![self])
+            .await?
+            .unwrap_or_default();
+        Ok(result)
     }
 
-    /// Whether the message ID is the special day marker.
-    ///
-    /// See the docs of the `dc_get_chat_msgs` C API for details.
-    pub fn is_daymarker(self) -> bool {
-        self.0 == DC_MSG_ID_DAYMARKER
+    /// Returns true if the message needs to be moved from `folder`.
+    pub async fn needs_move(self, context: &Context, folder: &str) -> Result<bool, Error> {
+        if !context.get_config_bool(Config::MvboxMove).await {
+            return Ok(false);
+        }
+
+        if context.is_mvbox(folder).await {
+            return Ok(false);
+        }
+
+        let msg = Message::load_from_db(context, self).await?;
+
+        if msg.is_setupmessage() {
+            // do not move setup messages;
+            // there may be a non-delta device that wants to handle it
+            return Ok(false);
+        }
+
+        match msg.is_dc_message {
+            MessengerMessage::No => Ok(false),
+            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
+        }
     }
 
     /// Put message into trash chat and delete message text.
@@ -143,16 +161,7 @@ impl MsgId {
 
 impl std::fmt::Display for MsgId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        // Would be nice if we could use match here, but no computed values in ranges.
-        if self.0 == DC_MSG_ID_MARKER1 {
-            write!(f, "Msg#Marker1")
-        } else if self.0 == DC_MSG_ID_DAYMARKER {
-            write!(f, "Msg#DayMarker")
-        } else if self.0 <= DC_MSG_ID_LAST_SPECIAL {
-            write!(f, "Msg#UnknownSpecial")
-        } else {
-            write!(f, "Msg#{}", self.0)
-        }
+        write!(f, "Msg#{}", self.0)
     }
 }
 
@@ -246,16 +255,17 @@ pub struct Message {
     pub(crate) timestamp_sort: i64,
     pub(crate) timestamp_sent: i64,
     pub(crate) timestamp_rcvd: i64,
+    pub(crate) ephemeral_timer: u32,
+    pub(crate) ephemeral_timestamp: i64,
     pub(crate) text: Option<String>,
     pub(crate) rfc724_mid: String,
     pub(crate) in_reply_to: Option<String>,
     pub(crate) server_folder: Option<String>,
     pub(crate) server_uid: u32,
     pub(crate) is_dc_message: MessengerMessage,
-    pub(crate) starred: bool,
     pub(crate) chat_blocked: Blocked,
     pub(crate) location_id: u32,
-    pub(crate) error: String,
+    error: Option<String>,
     pub(crate) param: Params,
 }
 
@@ -288,13 +298,14 @@ impl Message {
                     "    m.timestamp AS timestamp,",
                     "    m.timestamp_sent AS timestamp_sent,",
                     "    m.timestamp_rcvd AS timestamp_rcvd,",
+                    "    m.ephemeral_timer AS ephemeral_timer,",
+                    "    m.ephemeral_timestamp AS ephemeral_timestamp,",
                     "    m.type AS type,",
                     "    m.state AS state,",
                     "    m.error AS error,",
                     "    m.msgrmsg AS msgrmsg,",
                     "    m.txt AS txt,",
                     "    m.param AS param,",
-                    "    m.starred AS starred,",
                     "    m.hidden AS hidden,",
                     "    m.location_id AS location,",
                     "    c.blocked AS blocked",
@@ -316,9 +327,12 @@ impl Message {
                     msg.timestamp_sort = row.get("timestamp")?;
                     msg.timestamp_sent = row.get("timestamp_sent")?;
                     msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
+                    msg.ephemeral_timer = row.get("ephemeral_timer")?;
+                    msg.ephemeral_timestamp = row.get("ephemeral_timestamp")?;
                     msg.viewtype = row.get("type")?;
                     msg.state = row.get("state")?;
-                    msg.error = row.get("error")?;
+                    let error: String = row.get("error")?;
+                    msg.error = Some(error).filter(|error| !error.is_empty());
                     msg.is_dc_message = row.get("msgrmsg")?;
 
                     let text;
@@ -342,7 +356,6 @@ impl Message {
                     msg.text = Some(text);
 
                     msg.param = row.get::<_, String>("param")?.parse().unwrap_or_default();
-                    msg.starred = row.get("starred")?;
                     msg.hidden = row.get("hidden")?;
                     msg.location_id = row.get("location")?;
                     msg.chat_blocked = row
@@ -476,7 +489,7 @@ impl Message {
     pub fn get_text(&self) -> Option<String> {
         self.text
             .as_ref()
-            .map(|text| dc_truncate(text, 30000).to_string())
+            .map(|text| dc_truncate(text, DC_MAX_GET_TEXT_LEN).to_string())
     }
 
     pub fn get_filename(&self) -> Option<String> {
@@ -510,6 +523,14 @@ impl Message {
         self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0
     }
 
+    pub fn get_ephemeral_timer(&self) -> u32 {
+        self.ephemeral_timer
+    }
+
+    pub fn get_ephemeral_timestamp(&self) -> i64 {
+        self.ephemeral_timestamp
+    }
+
     pub async fn get_summary(&mut self, context: &Context, chat: Option<&Chat>) -> Lot {
         let mut ret = Lot::new();
 
@@ -523,9 +544,7 @@ impl Message {
             return ret;
         };
 
-        let contact = if self.from_id != DC_CONTACT_ID_SELF as u32
-            && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
-        {
+        let contact = if self.from_id != DC_CONTACT_ID_SELF as u32 && chat.typ == Chattype::Group {
             Contact::get_by_id(context, self.from_id).await.ok()
         } else {
             None
@@ -559,10 +578,6 @@ impl Message {
         self.state as i32 >= MessageState::OutDelivered as i32
     }
 
-    pub fn is_starred(&self) -> bool {
-        self.starred
-    }
-
     pub fn is_forwarded(&self) -> bool {
         0 != self.param.get_int(Param::Forwarded).unwrap_or_default()
     }
@@ -574,6 +589,15 @@ impl Message {
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
+    pub fn get_info_type(&self) -> SystemMessage {
+        self.param.get_cmd()
+    }
+
+    pub fn is_system_message(&self) -> bool {
+        let cmd = self.param.get_cmd();
+        cmd != SystemMessage::Unknown
+    }
+
     /// Whether the message is still being created.
     ///
     /// Messages with attachments might be created before the
@@ -612,6 +636,76 @@ impl Message {
         None
     }
 
+    // add room to a webrtc_instance as defined by the corresponding config-value;
+    // the result may still be prefixed by the type
+    pub fn create_webrtc_instance(instance: &str, room: &str) -> String {
+        let (videochat_type, mut url) = Message::parse_webrtc_instance(instance);
+
+        // make sure, there is a scheme in the url
+        if !url.contains(':') {
+            url = format!("https://{}", url);
+        }
+
+        // add/replace room
+        let url = if url.contains("$ROOM") {
+            url.replace("$ROOM", &room)
+        } else {
+            // if there nothing that would separate the room, add a slash as a separator;
+            // this way, urls can be given as "https://meet.jit.si" as well as "https://meet.jit.si/"
+            let maybe_slash = if url.ends_with('/')
+                || url.ends_with('?')
+                || url.ends_with('#')
+                || url.ends_with('=')
+            {
+                ""
+            } else {
+                "/"
+            };
+            format!("{}{}{}", url, maybe_slash, room)
+        };
+
+        // re-add and normalize type
+        match videochat_type {
+            VideochatType::BasicWebrtc => format!("basicwebrtc:{}", url),
+            VideochatType::Jitsi => format!("jitsi:{}", url),
+            VideochatType::Unknown => url,
+        }
+    }
+
+    /// split a webrtc_instance as defined by the corresponding config-value into a type and a url
+    pub fn parse_webrtc_instance(instance: &str) -> (VideochatType, String) {
+        let instance: String = instance.split_whitespace().collect();
+        let mut split = instance.splitn(2, ':');
+        let type_str = split.next().unwrap_or_default().to_lowercase();
+        let url = split.next();
+        match type_str.as_str() {
+            "basicwebrtc" => (
+                VideochatType::BasicWebrtc,
+                url.unwrap_or_default().to_string(),
+            ),
+            "jitsi" => (VideochatType::Jitsi, url.unwrap_or_default().to_string()),
+            _ => (VideochatType::Unknown, instance.to_string()),
+        }
+    }
+
+    pub fn get_videochat_url(&self) -> Option<String> {
+        if self.viewtype == Viewtype::VideochatInvitation {
+            if let Some(instance) = self.param.get(Param::WebrtcRoom) {
+                return Some(Message::parse_webrtc_instance(instance).1);
+            }
+        }
+        None
+    }
+
+    pub fn get_videochat_type(&self) -> Option<VideochatType> {
+        if self.viewtype == Viewtype::VideochatInvitation {
+            if let Some(instance) = self.param.get(Param::WebrtcRoom) {
+                return Some(Message::parse_webrtc_instance(instance).0);
+            }
+        }
+        None
+    }
+
     pub fn set_text(&mut self, text: Option<String>) {
         self.text = text;
     }
@@ -649,6 +743,61 @@ impl Message {
         self.update_param(context).await;
     }
 
+    /// Sets message quote.
+    ///
+    /// Message-Id is used to set Reply-To field, message text is used for quote.
+    ///
+    /// Encryption is required if quoted message was encrypted.
+    ///
+    /// The message itself is not required to exist in the database,
+    /// it may even be deleted from the database by the time the message is prepared.
+    pub async fn set_quote(&mut self, context: &Context, quote: &Message) -> Result<(), Error> {
+        ensure!(
+            !quote.rfc724_mid.is_empty(),
+            "Message without Message-Id cannot be quoted"
+        );
+        self.in_reply_to = Some(quote.rfc724_mid.clone());
+
+        if quote
+            .param
+            .get_bool(Param::GuaranteeE2ee)
+            .unwrap_or_default()
+        {
+            self.param.set(Param::GuaranteeE2ee, "1");
+        }
+
+        let text = quote.get_text().unwrap_or_default();
+        self.param.set(
+            Param::Quote,
+            if text.is_empty() {
+                // Use summary, similar to "Image" to avoid sending empty quote.
+                quote.get_summarytext(context, 500).await
+            } else {
+                text
+            },
+        );
+
+        Ok(())
+    }
+
+    pub fn quoted_text(&self) -> Option<String> {
+        self.param.get(Param::Quote).map(|s| s.to_string())
+    }
+
+    pub async fn quoted_message(&self, context: &Context) -> Result<Option<Message>, Error> {
+        if self.param.get(Param::Quote).is_some() {
+            if let Some(in_reply_to) = &self.in_reply_to {
+                let rfc724_mid = in_reply_to.trim_start_matches('<').trim_end_matches('>');
+                if !rfc724_mid.is_empty() {
+                    if let Some((_, _, msg_id)) = rfc724_mid_exists(context, rfc724_mid).await? {
+                        return Ok(Some(Message::load_from_db(context, msg_id).await?));
+                    }
+                }
+            }
+        }
+        Ok(None)
+    }
+
     pub async fn update_param(&mut self, context: &Context) -> bool {
         context
             .sql
@@ -659,6 +808,22 @@ impl Message {
             .await
             .is_ok()
     }
+
+    /// Gets the error status of the message.
+    ///
+    /// A message can have an associated error status if something went wrong when sending or
+    /// receiving message itself.  The error status is free-form text and should not be further parsed,
+    /// rather it's presence is meant to indicate *something* went wrong with the message and the
+    /// text of the error is detailed information on what.
+    ///
+    /// Some common reasons error can be associated with messages are:
+    /// * Lack of valid signature on an e2ee message, usually for received messages.
+    /// * Failure to decrypt an e2ee message, usually for received messages.
+    /// * When a message could not be delivered to one or more recipients the non-delivery
+    ///    notification text can be stored in the error status.
+    pub fn error(&self) -> Option<String> {
+        self.error.clone()
+    }
 }
 
 #[derive(
@@ -765,13 +930,18 @@ impl From<MessageState> for LotState {
 
 impl MessageState {
     pub fn can_fail(self) -> bool {
-        match self {
-            MessageState::OutPreparing
-            | MessageState::OutPending
-            | MessageState::OutDelivered
-            | MessageState::OutMdnRcvd => true, // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
-            _ => false,
-        }
+        use MessageState::*;
+        matches!(
+            self,
+            OutPreparing | OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
+        )
+    }
+    pub fn is_outgoing(self) -> bool {
+        use MessageState::*;
+        matches!(
+            self,
+            OutPreparing | OutDraft | OutPending | OutFailed | OutDelivered | OutMdnRcvd
+        )
     }
 }
 
@@ -808,7 +978,7 @@ impl Lot {
                 );
                 self.text1_meaning = Meaning::Text1Self;
             }
-        } else if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
+        } else if chat.typ == Chattype::Group {
             if msg.is_info() || contact.is_none() {
                 self.text1 = None;
                 self.text1_meaning = Meaning::None;
@@ -828,16 +998,23 @@ impl Lot {
             }
         }
 
-        self.text2 = Some(
-            get_summarytext_by_raw(
-                msg.viewtype,
-                msg.text.as_ref(),
-                &msg.param,
-                SUMMARY_CHARACTERS,
-                context,
-            )
-            .await,
-        );
+        let mut text2 = get_summarytext_by_raw(
+            msg.viewtype,
+            msg.text.as_ref(),
+            &msg.param,
+            SUMMARY_CHARACTERS,
+            context,
+        )
+        .await;
+
+        if text2.is_empty() && msg.quoted_text().is_some() {
+            text2 = context
+                .stock_str(StockMessage::ReplyNoun)
+                .await
+                .into_owned()
+        }
+
+        self.text2 = Some(text2);
 
         self.timestamp = msg.get_timestamp();
         self.state = msg.state.into();
@@ -868,7 +1045,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         return ret;
     }
     let rawtxt = rawtxt.unwrap_or_default();
-    let rawtxt = dc_truncate(rawtxt.trim(), 100_000);
+    let rawtxt = dc_truncate(rawtxt.trim(), DC_MAX_GET_INFO_LEN);
 
     let fts = dc_timestamp_to_str(msg.get_timestamp());
     ret += &format!("Sent: {}", fts);
@@ -891,6 +1068,17 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         ret += "\n";
     }
 
+    if msg.ephemeral_timer != 0 {
+        ret += &format!("Ephemeral timer: {}\n", msg.ephemeral_timer);
+    }
+
+    if msg.ephemeral_timestamp != 0 {
+        ret += &format!(
+            "Expires: {}\n",
+            dc_timestamp_to_str(msg.ephemeral_timestamp)
+        );
+    }
+
     if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return ret;
@@ -942,8 +1130,8 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
 
     ret += "\n";
 
-    if !msg.error.is_empty() {
-        ret += &format!("Error: {}", msg.error);
+    if let Some(error) = msg.error.as_ref() {
+        ret += &format!("Error: {}", error);
     }
 
     if let Some(path) = msg.get_file(context) {
@@ -984,18 +1172,71 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
 pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
     let extension: &str = &path.extension()?.to_str()?.to_lowercase();
     let info = match extension {
-        "mp3" => (Viewtype::Audio, "audio/mpeg"),
+        // before using viewtype other than Viewtype::File,
+        // make sure, all target UIs support that type in the context of the used viewer/player.
+        // if in doubt, it is better to default to Viewtype::File that passes handing to an external app.
+        // (cmp. https://developer.android.com/guide/topics/media/media-formats )
+        "3gp" => (Viewtype::Video, "video/3gpp"),
         "aac" => (Viewtype::Audio, "audio/aac"),
-        "mp4" => (Viewtype::Video, "video/mp4"),
-        "webm" => (Viewtype::Video, "video/webm"),
-        "jpg" => (Viewtype::Image, "image/jpeg"),
+        "avi" => (Viewtype::Video, "video/x-msvideo"),
+        "doc" => (Viewtype::File, "application/msword"),
+        "docx" => (
+            Viewtype::File,
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        ),
+        "epub" => (Viewtype::File, "application/epub+zip"),
+        "flac" => (Viewtype::Audio, "audio/flac"),
+        "gif" => (Viewtype::Gif, "image/gif"),
+        "html" => (Viewtype::File, "text/html"),
+        "htm" => (Viewtype::File, "text/html"),
+        "ico" => (Viewtype::File, "image/vnd.microsoft.icon"),
+        "jar" => (Viewtype::File, "application/java-archive"),
         "jpeg" => (Viewtype::Image, "image/jpeg"),
         "jpe" => (Viewtype::Image, "image/jpeg"),
+        "jpg" => (Viewtype::Image, "image/jpeg"),
+        "json" => (Viewtype::File, "application/json"),
+        "mov" => (Viewtype::Video, "video/quicktime"),
+        "m4a" => (Viewtype::Audio, "audio/m4a"),
+        "mp3" => (Viewtype::Audio, "audio/mpeg"),
+        "mp4" => (Viewtype::Video, "video/mp4"),
+        "odp" => (
+            Viewtype::File,
+            "application/vnd.oasis.opendocument.presentation",
+        ),
+        "ods" => (
+            Viewtype::File,
+            "application/vnd.oasis.opendocument.spreadsheet",
+        ),
+        "odt" => (Viewtype::File, "application/vnd.oasis.opendocument.text"),
+        "oga" => (Viewtype::Audio, "audio/ogg"),
+        "ogg" => (Viewtype::Audio, "audio/ogg"),
+        "ogv" => (Viewtype::File, "video/ogg"),
+        "opus" => (Viewtype::File, "audio/ogg"), // not supported eg. on Android 4
+        "otf" => (Viewtype::File, "font/otf"),
+        "pdf" => (Viewtype::File, "application/pdf"),
         "png" => (Viewtype::Image, "image/png"),
-        "webp" => (Viewtype::Image, "image/webp"),
-        "gif" => (Viewtype::Gif, "image/gif"),
-        "vcf" => (Viewtype::File, "text/vcard"),
+        "rar" => (Viewtype::File, "application/vnd.rar"),
+        "rtf" => (Viewtype::File, "application/rtf"),
+        "spx" => (Viewtype::File, "audio/ogg"), // Ogg Speex Profile
+        "svg" => (Viewtype::File, "image/svg+xml"),
+        "tgs" => (Viewtype::Sticker, "application/x-tgsticker"),
+        "tiff" => (Viewtype::File, "image/tiff"),
+        "tif" => (Viewtype::File, "image/tiff"),
+        "ttf" => (Viewtype::File, "font/ttf"),
         "vcard" => (Viewtype::File, "text/vcard"),
+        "vcf" => (Viewtype::File, "text/vcard"),
+        "wav" => (Viewtype::File, "audio/wav"),
+        "weba" => (Viewtype::File, "audio/webm"),
+        "webm" => (Viewtype::Video, "video/webm"),
+        "webp" => (Viewtype::Image, "image/webp"), // iOS via SDWebImage, Android since 4.0
+        "wmv" => (Viewtype::Video, "video/x-ms-wmv"),
+        "xhtml" => (Viewtype::File, "application/xhtml+xml"),
+        "xlsx" => (
+            Viewtype::File,
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        ),
+        "xml" => (Viewtype::File, "application/vnd.ms-excel"),
+        "zip" => (Viewtype::File, "application/zip"),
         _ => {
             return None;
         }
@@ -1032,7 +1273,7 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
     }
 
     if !msg_ids.is_empty() {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -1066,6 +1307,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
         .with_conn(move |conn| {
             let mut stmt = conn.prepare_cached(concat!(
                 "SELECT",
+                "    m.chat_id AS chat_id,",
                 "    m.state AS state,",
                 "    c.blocked AS blocked",
                 " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
@@ -1076,6 +1318,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
             for id in msg_ids.into_iter() {
                 let query_res = stmt.query_row(paramsv![id], |row| {
                     Ok((
+                        row.get::<_, ChatId>("chat_id")?,
                         row.get::<_, MessageState>("state")?,
                         row.get::<_, Option<Blocked>>("blocked")?
                             .unwrap_or_default(),
@@ -1084,8 +1327,8 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
                 if let Err(rusqlite::Error::QueryReturnedNoRows) = query_res {
                     continue;
                 }
-                let (state, blocked) = query_res.map_err(Into::<anyhow::Error>::into)?;
-                msgs.push((id, state, blocked));
+                let (chat_id, state, blocked) = query_res.map_err(Into::<anyhow::Error>::into)?;
+                msgs.push((id, chat_id, state, blocked));
             }
 
             Ok(msgs)
@@ -1093,9 +1336,17 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
         .await
         .unwrap_or_default();
 
-    let mut send_event = false;
+    let mut updated_chat_ids = BTreeMap::new();
+
+    for (id, curr_chat_id, curr_state, curr_blocked) in msgs.into_iter() {
+        if let Err(err) = id.start_ephemeral_timer(context).await {
+            error!(
+                context,
+                "Failed to start ephemeral timer for message {}: {}", id, err
+            );
+            continue;
+        }
 
-    for (id, curr_state, curr_blocked) in msgs.into_iter() {
         if curr_blocked == Blocked::Not {
             if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
                 update_msg_state(context, id, MessageState::InSeen).await;
@@ -1106,19 +1357,16 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
                     job::Job::new(Action::MarkseenMsgOnImap, id.to_u32(), Params::new(), 0),
                 )
                 .await;
-                send_event = true;
+                updated_chat_ids.insert(curr_chat_id, true);
             }
         } else if curr_state == MessageState::InFresh {
             update_msg_state(context, id, MessageState::InNoticed).await;
-            send_event = true;
+            updated_chat_ids.insert(ChatId::new(DC_CHAT_ID_DEADDROP), true);
         }
     }
 
-    if send_event {
-        context.emit_event(Event::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
+    for updated_chat_id in updated_chat_ids.keys() {
+        context.emit_event(EventType::MsgsNoticed(*updated_chat_id));
     }
 
     true
@@ -1135,24 +1383,7 @@ pub async fn update_msg_state(context: &Context, msg_id: MsgId, state: MessageSt
         .is_ok()
 }
 
-pub async fn star_msgs(context: &Context, msg_ids: Vec<MsgId>, star: bool) -> bool {
-    if msg_ids.is_empty() {
-        return false;
-    }
-    context
-        .sql
-        .with_conn(move |conn| {
-            let mut stmt = conn.prepare("UPDATE msgs SET starred=? WHERE id=?;")?;
-            for msg_id in msg_ids.into_iter() {
-                stmt.execute(paramsv![star as i32, msg_id])?;
-            }
-            Ok(())
-        })
-        .await
-        .is_ok()
-}
-
-/// Returns a summary test.
+/// Returns a summary text.
 pub async fn get_summarytext_by_raw(
     viewtype: Viewtype,
     text: Option<impl AsRef<str>>,
@@ -1196,6 +1427,13 @@ pub async fn get_summarytext_by_raw(
                 format!("{} – {}", label, file_name)
             }
         }
+        Viewtype::VideochatInvitation => {
+            append_text = false;
+            context
+                .stock_str(StockMessage::VideochatInvitation)
+                .await
+                .into_owned()
+        }
         _ => {
             if param.get_cmd() != SystemMessage::LocationOnly {
                 "".to_string()
@@ -1223,7 +1461,7 @@ pub async fn get_summarytext_by_raw(
         prefix
     };
 
-    UNWRAP_RE.replace_all(&summary, " ").to_string()
+    summary.split_whitespace().join(" ")
 }
 
 // as we do not cut inside words, this results in about 32-42 characters.
@@ -1275,7 +1513,7 @@ pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl
             )
             .await
         {
-            Ok(_) => context.emit_event(Event::MsgFailed {
+            Ok(_) => context.emit_event(EventType::MsgFailed {
                 chat_id: msg.chat_id,
                 msg_id,
             }),
@@ -1293,7 +1531,7 @@ pub async fn handle_mdn(
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Option<(ChatId, MsgId)> {
-    if from_id <= DC_MSG_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
+    if from_id <= DC_CONTACT_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
         return None;
     }
 
@@ -1400,14 +1638,16 @@ pub(crate) async fn handle_ndn(
     context: &Context,
     failed: &FailureReport,
     error: Option<impl AsRef<str>>,
-) {
+) -> anyhow::Result<()> {
     if failed.rfc724_mid.is_empty() {
-        return;
+        return Ok(());
     }
 
-    let res = context
+    // The NDN might be for a message-id that had attachments and was sent from a non-Delta Chat client.
+    // In this case we need to mark multiple "msgids" as failed that all refer to the same message-id.
+    let msgs: Vec<_> = context
         .sql
-        .query_row(
+        .query_map(
             concat!(
                 "SELECT",
                 "    m.id AS msg_id,",
@@ -1424,37 +1664,42 @@ pub(crate) async fn handle_ndn(
                     row.get::<_, Chattype>("type")?,
                 ))
             },
+            |rows| Ok(rows.collect::<Vec<_>>()),
         )
-        .await;
-    if let Err(ref err) = res {
-        info!(context, "Failed to select NDN {:?}", err);
-    }
+        .await?;
 
-    if let Ok((msg_id, chat_id, chat_type)) = res {
-        set_msg_failed(context, msg_id, error).await;
-
-        if chat_type == Chattype::Group || chat_type == Chattype::VerifiedGroup {
-            if let Some(failed_recipient) = &failed.failed_recipient {
-                let contact_id =
-                    Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
-                if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-                    // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
-                    chat::add_info_msg(
-                        context,
-                        chat_id,
-                        context
-                            .stock_string_repl_str(
-                                StockMessage::FailedSendingTo,
-                                contact.get_display_name(),
-                            )
-                            .await,
-                    )
-                    .await;
-                    context.emit_event(Event::ChatModified(chat_id));
-                }
-            }
+    for (i, msg) in msgs.into_iter().enumerate() {
+        let (msg_id, chat_id, chat_type) = msg?;
+        set_msg_failed(context, msg_id, error.as_ref()).await;
+        if i == 0 {
+            // Add only one info msg for all failed messages
+            ndn_maybe_add_info_msg(context, failed, chat_id, chat_type).await?;
         }
     }
+
+    Ok(())
+}
+
+async fn ndn_maybe_add_info_msg(
+    context: &Context,
+    failed: &FailureReport,
+    chat_id: ChatId,
+    chat_type: Chattype,
+) -> anyhow::Result<()> {
+    if chat_type == Chattype::Group {
+        if let Some(failed_recipient) = &failed.failed_recipient {
+            let contact_id =
+                Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
+            let contact = Contact::load_from_db(context, contact_id).await?;
+            // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
+            let text = context
+                .stock_string_repl_str(StockMessage::FailedSendingTo, contact.get_display_name())
+                .await;
+            chat::add_info_msg(context, chat_id, text).await;
+            context.emit_event(EventType::ChatModified(chat_id));
+        }
+    }
+    Ok(())
 }
 
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)
@@ -1618,19 +1863,10 @@ pub async fn update_server_uid(
     }
 }
 
-#[allow(dead_code)]
-pub async fn dc_empty_server(context: &Context, flags: u32) {
-    job::kill_action(context, Action::EmptyServer).await;
-    job::add(
-        context,
-        job::Job::new(Action::EmptyServer, flags, Params::new(), 0),
-    )
-    .await;
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chat::ChatItem;
     use crate::test_utils as test;
 
     #[test]
@@ -1667,12 +1903,29 @@ mod tests {
         assert_eq!(_msg2.get_filemime(), None);
     }
 
+    /// Tests that message cannot be prepared if account has no configured address.
+    #[async_std::test]
+    async fn test_prepare_not_configured() {
+        let d = test::TestContext::new().await;
+        let ctx = &d.ctx;
+
+        let contact = Contact::create(ctx, "", "dest@example.com")
+            .await
+            .expect("failed to create contact");
+
+        let chat = chat::create_by_contact_id(ctx, contact).await.unwrap();
+
+        let mut msg = Message::new(Viewtype::Text);
+
+        assert!(chat::prepare_msg(ctx, chat, &mut msg).await.is_err());
+    }
+
     #[async_std::test]
     async fn test_get_summarytext_by_raw() {
         let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
-        let some_text = Some("bla bla".to_string());
+        let some_text = Some(" bla \t\n\tbla\n\t".to_string());
         let empty_text = Some("".to_string());
         let no_text: Option<String> = None;
 
@@ -1713,8 +1966,7 @@ mod tests {
         );
 
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::Voice, no_text.as_ref(), &mut some_file, 50, &ctx)
-                .await,
+            get_summarytext_by_raw(Viewtype::Voice, no_text.as_ref(), &some_file, 50, &ctx).await,
             "Voice message" // file names are not added for voice messages
         );
 
@@ -1724,8 +1976,7 @@ mod tests {
         );
 
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::Audio, no_text.as_ref(), &mut some_file, 50, &ctx)
-                .await,
+            get_summarytext_by_raw(Viewtype::Audio, no_text.as_ref(), &some_file, 50, &ctx).await,
             "Audio \u{2013} foo.bar" // file name is added for audio
         );
 
@@ -1741,8 +1992,7 @@ mod tests {
         );
 
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::File, some_text.as_ref(), &mut some_file, 50, &ctx)
-                .await,
+            get_summarytext_by_raw(Viewtype::File, some_text.as_ref(), &some_file, 50, &ctx).await,
             "File \u{2013} foo.bar \u{2013} bla bla" // file name is added for files
         );
 
@@ -1750,8 +2000,134 @@ mod tests {
         asm_file.set(Param::File, "foo.bar");
         asm_file.set_cmd(SystemMessage::AutocryptSetupMessage);
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), &mut asm_file, 50, &ctx).await,
+            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), &asm_file, 50, &ctx).await,
             "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
         );
     }
+
+    #[async_std::test]
+    async fn test_parse_webrtc_instance() {
+        let (webrtc_type, url) = Message::parse_webrtc_instance("basicwebrtc:https://foo/bar");
+        assert_eq!(webrtc_type, VideochatType::BasicWebrtc);
+        assert_eq!(url, "https://foo/bar");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("bAsIcwEbrTc:url");
+        assert_eq!(webrtc_type, VideochatType::BasicWebrtc);
+        assert_eq!(url, "url");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("https://foo/bar?key=val#key=val");
+        assert_eq!(webrtc_type, VideochatType::Unknown);
+        assert_eq!(url, "https://foo/bar?key=val#key=val");
+
+        let (webrtc_type, url) = Message::parse_webrtc_instance("jitsi:https://j.si/foo");
+        assert_eq!(webrtc_type, VideochatType::Jitsi);
+        assert_eq!(url, "https://j.si/foo");
+    }
+
+    #[async_std::test]
+    async fn test_create_webrtc_instance() {
+        // webrtc_instance may come from an input field of the ui, be pretty tolerant on input
+        let instance = Message::create_webrtc_instance("https://meet.jit.si/", "123");
+        assert_eq!(instance, "https://meet.jit.si/123");
+
+        let instance = Message::create_webrtc_instance("https://meet.jit.si", "456");
+        assert_eq!(instance, "https://meet.jit.si/456");
+
+        let instance = Message::create_webrtc_instance("meet.jit.si", "789");
+        assert_eq!(instance, "https://meet.jit.si/789");
+
+        let instance = Message::create_webrtc_instance("bla.foo?", "123");
+        assert_eq!(instance, "https://bla.foo?123");
+
+        let instance = Message::create_webrtc_instance("jitsi:bla.foo#", "456");
+        assert_eq!(instance, "jitsi:https://bla.foo#456");
+
+        let instance = Message::create_webrtc_instance("bla.foo#room=", "789");
+        assert_eq!(instance, "https://bla.foo#room=789");
+
+        let instance = Message::create_webrtc_instance("https://bla.foo#room", "123");
+        assert_eq!(instance, "https://bla.foo#room/123");
+
+        let instance = Message::create_webrtc_instance("bla.foo#room$ROOM", "123");
+        assert_eq!(instance, "https://bla.foo#room123");
+
+        let instance = Message::create_webrtc_instance("bla.foo#room=$ROOM&after=cont", "234");
+        assert_eq!(instance, "https://bla.foo#room=234&after=cont");
+
+        let instance = Message::create_webrtc_instance("  meet.jit .si ", "789");
+        assert_eq!(instance, "https://meet.jit.si/789");
+
+        let instance = Message::create_webrtc_instance(" basicwebrtc: basic . stuff\n ", "12345ab");
+        assert_eq!(instance, "basicwebrtc:https://basic.stuff/12345ab");
+    }
+
+    #[async_std::test]
+    async fn test_get_width_height() {
+        let t = test::TestContext::new().await;
+
+        // test that get_width() and get_height() are returning some dimensions for images;
+        // (as the device-chat contains a welcome-images, we check that)
+        t.ctx.update_device_chats().await.ok();
+        let (device_chat_id, _) =
+            chat::create_or_lookup_by_contact_id(&t.ctx, DC_CONTACT_ID_DEVICE, Blocked::Not)
+                .await
+                .unwrap();
+
+        let mut has_image = false;
+        let chatitems = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        for chatitem in chatitems {
+            if let ChatItem::Message { msg_id } = chatitem {
+                if let Ok(msg) = Message::load_from_db(&t.ctx, msg_id).await {
+                    if msg.get_viewtype() == Viewtype::Image {
+                        has_image = true;
+                        // just check that width/height are inside some reasonable ranges
+                        assert!(msg.get_width() > 100);
+                        assert!(msg.get_height() > 100);
+                        assert!(msg.get_width() < 4000);
+                        assert!(msg.get_height() < 4000);
+                    }
+                }
+            }
+        }
+        assert!(has_image);
+    }
+
+    #[async_std::test]
+    async fn test_quote() {
+        use crate::config::Config;
+
+        let d = test::TestContext::new().await;
+        let ctx = &d.ctx;
+
+        let contact = Contact::create(ctx, "", "dest@example.com")
+            .await
+            .expect("failed to create contact");
+
+        let res = ctx
+            .set_config(Config::ConfiguredAddr, Some("self@example.com"))
+            .await;
+        assert!(res.is_ok());
+
+        let chat = chat::create_by_contact_id(ctx, contact).await.unwrap();
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("Quoted message".to_string()));
+
+        // Prepare message for sending, so it gets a Message-Id.
+        assert!(msg.rfc724_mid.is_empty());
+        let msg_id = chat::prepare_msg(ctx, chat, &mut msg).await.unwrap();
+        let msg = Message::load_from_db(ctx, msg_id).await.unwrap();
+        assert!(!msg.rfc724_mid.is_empty());
+
+        let mut msg2 = Message::new(Viewtype::Text);
+        msg2.set_quote(ctx, &msg).await.expect("can't set quote");
+        assert!(msg2.quoted_text() == msg.get_text());
+
+        let quoted_msg = msg2
+            .quoted_message(ctx)
+            .await
+            .expect("error while retrieving quoted message")
+            .expect("quoted message not found");
+        assert!(quoted_msg.get_text() == msg2.quoted_text());
+    }
 }

src/mimefactory.rs

@@ -9,7 +9,9 @@ use crate::contact::*;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::*;
 use crate::e2ee::*;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
+use crate::format_flowed::{format_flowed, format_flowed_quote};
 use crate::location;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
@@ -112,12 +114,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 )
                 .await?;
 
-            let command = msg.param.get_cmd();
-
-            if command != SystemMessage::AutocryptSetupMessage
-                && command != SystemMessage::SecurejoinMessage
-                && context.get_config_bool(Config::MdnsEnabled).await
-            {
+            if !msg.is_system_message() && context.get_config_bool(Config::MdnsEnabled).await {
                 req_mdn = true;
             }
         }
@@ -148,7 +145,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             selfstatus: context
                 .get_config(Config::Selfstatus)
                 .await
-                .unwrap_or_else(|| default_str),
+                .unwrap_or(default_str),
             recipients,
             timestamp: msg.timestamp_sort,
             loaded: Loaded::Message { chat },
@@ -186,7 +183,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let selfstatus = context
             .get_config(Config::Selfstatus)
             .await
-            .unwrap_or_else(|| default_str);
+            .unwrap_or(default_str);
         let timestamp = dc_create_smeared_timestamp(context).await;
 
         let res = MimeFactory::<'a, 'b> {
@@ -225,7 +222,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .filter(|(_, addr)| addr != &self_addr)
         {
             res.push((
-                Peerstate::from_addr(self.context, addr).await,
+                Peerstate::from_addr(self.context, addr).await?,
                 addr.as_str(),
             ));
         }
@@ -236,26 +233,20 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     fn is_e2ee_guaranteed(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.typ == Chattype::VerifiedGroup {
+                if chat.is_protected() {
                     return true;
                 }
 
-                let force_plaintext = self
+                !self
                     .msg
                     .param
-                    .get_int(Param::ForcePlaintext)
-                    .unwrap_or_default();
-
-                if force_plaintext == 0 {
-                    return self
+                    .get_bool(Param::ForcePlaintext)
+                    .unwrap_or_default()
+                    && self
                         .msg
                         .param
-                        .get_int(Param::GuaranteeE2ee)
+                        .get_bool(Param::GuaranteeE2ee)
                         .unwrap_or_default()
-                        != 0;
-                }
-
-                false
             }
             Loaded::MDN { .. } => false,
         }
@@ -264,7 +255,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     fn min_verified(&self) -> PeerstateVerifiedStatus {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.typ == Chattype::VerifiedGroup {
+                if chat.is_protected() {
                     PeerstateVerifiedStatus::BidirectVerified
                 } else {
                     PeerstateVerifiedStatus::Unverified
@@ -274,19 +265,30 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
     }
 
-    fn should_force_plaintext(&self) -> i32 {
+    fn should_force_plaintext(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.typ == Chattype::VerifiedGroup {
-                    0
+                if chat.is_protected() {
+                    false
                 } else {
                     self.msg
                         .param
-                        .get_int(Param::ForcePlaintext)
+                        .get_bool(Param::ForcePlaintext)
                         .unwrap_or_default()
                 }
             }
-            Loaded::MDN { .. } => ForcePlaintext::NoAutocryptHeader as i32,
+            Loaded::MDN { .. } => true,
+        }
+    }
+
+    fn should_skip_autocrypt(&self) -> bool {
+        match &self.loaded {
+            Loaded::Message { .. } => self
+                .msg
+                .param
+                .get_bool(Param::SkipAutocrypt)
+                .unwrap_or_default(),
+            Loaded::MDN { .. } => true,
         }
     }
 
@@ -343,7 +345,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                         .stock_str(StockMessage::AcSetupMsgSubject)
                         .await
                         .into_owned()
-                } else if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
+                } else if chat.typ == Chattype::Group {
                     let re = if self.in_reply_to.is_empty() {
                         ""
                     } else {
@@ -457,21 +459,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         unprotected_headers.push(Header::new("Date".into(), date));
 
-        let os_name = &self.context.os_name;
-        let os_part = os_name
-            .as_ref()
-            .map(|s| format!("/{}", s))
-            .unwrap_or_default();
-        let version = get_version_str();
-
-        // Add a X-Mailer header.
-        // This is only informational for debugging and may be removed in the release.
-        // We do not rely on this header as it may be removed by MTAs.
-
-        unprotected_headers.push(Header::new(
-            "X-Mailer".into(),
-            format!("Delta Chat Core {}{}", version, os_part),
-        ));
         unprotected_headers.push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
 
         if let Loaded::MDN { .. } = self.loaded {
@@ -494,11 +481,21 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let min_verified = self.min_verified();
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
+        let skip_autocrypt = self.should_skip_autocrypt();
         let subject_str = self.subject_str().await;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let encrypt_helper = EncryptHelper::new(self.context).await?;
 
-        let subject = encode_words(&subject_str);
+        let subject = if subject_str
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
+        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
+        // but we do not want to encode all subjects just because they contain a space.
+        {
+            subject_str
+        } else {
+            encode_words(&subject_str)
+        };
 
         let mut message = match self.loaded {
             Loaded::Message { .. } => {
@@ -508,7 +505,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::MDN { .. } => self.render_mdn().await?,
         };
 
-        if force_plaintext != ForcePlaintext::NoAutocryptHeader as i32 {
+        if !skip_autocrypt {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
             unprotected_headers.push(Header::new("Autocrypt".into(), aheader));
@@ -519,13 +516,21 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let peerstates = self.peerstates_for_recipients().await?;
         let should_encrypt =
             encrypt_helper.should_encrypt(self.context, e2ee_guaranteed, &peerstates)?;
-        let is_encrypted = should_encrypt && force_plaintext == 0;
+        let is_encrypted = should_encrypt && !force_plaintext;
 
         let rfc724_mid = match self.loaded {
             Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
             Loaded::MDN { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
         };
 
+        let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(self.context).await?;
+        if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
+            protected_headers.push(Header::new(
+                "Ephemeral-Timer".to_string(),
+                duration.to_string(),
+            ));
+        }
+
         // we could also store the message-id in the protected headers
         // which would probably help to survive providers like
         // Outlook.com or hotmail which mangle the Message-ID.
@@ -703,11 +708,11 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let mut placeholdertext = None;
         let mut meta_part = None;
 
-        if chat.typ == Chattype::VerifiedGroup {
+        if chat.is_protected() {
             protected_headers.push(Header::new("Chat-Verified".to_string(), "1".to_string()));
         }
 
-        if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
+        if chat.typ == Chattype::Group {
             protected_headers.push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
             let encoded = encode_words(&chat.name);
@@ -776,6 +781,26 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     "location-streaming-enabled".into(),
                 ));
             }
+            SystemMessage::EphemeralTimerChanged => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "ephemeral-timer-changed".to_string(),
+                ));
+            }
+            SystemMessage::LocationOnly => {
+                // This should prevent automatic replies,
+                // such as non-delivery reports.
+                //
+                // See https://tools.ietf.org/html/rfc3834
+                //
+                // Adding this header without encryption leaks some
+                // information about the message contents, but it can
+                // already be easily guessed from message timing and size.
+                unprotected_headers.push(Header::new(
+                    "Auto-Submitted".to_string(),
+                    "auto-generated".to_string(),
+                ));
+            }
             SystemMessage::AutocryptSetupMessage => {
                 unprotected_headers
                     .push(Header::new("Autocrypt-Setup-Message".into(), "v1".into()));
@@ -821,6 +846,18 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                 }
             }
+            SystemMessage::ChatProtectionEnabled => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "protection-enabled".to_string(),
+                ));
+            }
+            SystemMessage::ChatProtectionDisabled => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "protection-disabled".to_string(),
+                ));
+            }
             _ => {}
         }
 
@@ -837,6 +874,19 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         if self.msg.viewtype == Viewtype::Sticker {
             protected_headers.push(Header::new("Chat-Content".into(), "sticker".into()));
+        } else if self.msg.viewtype == Viewtype::VideochatInvitation {
+            protected_headers.push(Header::new(
+                "Chat-Content".into(),
+                "videochat-invitation".into(),
+            ));
+            protected_headers.push(Header::new(
+                "Chat-Webrtc-Room".into(),
+                self.msg
+                    .param
+                    .get(Param::WebrtcRoom)
+                    .unwrap_or_default()
+                    .into(),
+            ));
         }
 
         if self.msg.viewtype == Viewtype::Voice
@@ -879,11 +929,18 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         };
 
+        let quoted_text = self
+            .msg
+            .quoted_text()
+            .map(|quote| format_flowed_quote(&quote) + "\r\n\r\n");
+        let flowed_text = format_flowed(final_text);
+
         let footer = &self.selfstatus;
         let message_text = format!(
-            "{}{}{}{}{}",
+            "{}{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
-            escape_message_footer_marks(final_text),
+            quoted_text.unwrap_or_default(),
+            escape_message_footer_marks(&flowed_text),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -895,7 +952,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         // Message is sent as text/plain, with charset = utf-8
         let main_part = PartBuilder::new()
-            .content_type(&mime::TEXT_PLAIN_UTF_8)
+            .header((
+                "Content-Type".to_string(),
+                "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
+            ))
             .body(message_text);
         let mut parts = Vec::new();
 
@@ -1081,13 +1141,23 @@ async fn build_body_file(
         Viewtype::Image | Viewtype::Gif => format!(
             "{}.{}",
             if base_name.is_empty() {
-                "image"
+                chrono::Utc
+                    .timestamp(msg.timestamp_sort as i64, 0)
+                    .format("image_%Y-%m-%d_%H-%M-%S")
+                    .to_string()
             } else {
-                base_name
+                base_name.to_string()
             },
             &suffix,
         ),
-        Viewtype::Video => format!("video.{}", &suffix),
+        Viewtype::Video => format!(
+            "video_{}.{}",
+            chrono::Utc
+                .timestamp(msg.timestamp_sort as i64, 0)
+                .format("%Y-%m-%d_%H-%M-%S")
+                .to_string(),
+            &suffix
+        ),
         _ => blob.as_file_name().to_string(),
     };
 
@@ -1379,6 +1449,54 @@ mod tests {
                 .as_bytes(),
         )
         .await;
+
+        // 5. Receive an mdn (read receipt) and make sure the mdn's subject is not used
+        let t = TestContext::new_alice().await;
+        dc_receive_imf(
+            &t.ctx,
+            b"From: alice@example.com\n\
+            To: Charlie <charlie@example.com>\n\
+            Subject: Hello, Charlie\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+        let new_msg = incoming_msg_to_reply_msg(b"From: charlie@example.com\n\
+                 To: alice@example.com\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: <Mr.12345678902@example.com>\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Reporting-UA: Delta Chat 1.28.0\n\
+                 Original-Recipient: rfc822;charlie@example.com\n\
+                 Final-Recipient: rfc822;charlie@example.com\n\
+                 Original-Message-ID: <2893@example.com>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n", &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        // The subject string should not be "Re: message opened"
+        assert_eq!("Re: Hello, Charlie", mf.subject_str().await);
     }
 
     async fn first_subject_str(t: TestContext) -> String {
@@ -1415,7 +1533,7 @@ mod tests {
         mf.subject_str().await
     }
 
-    // Creates a mimefactory for a message that replies "Hi" to the incoming message in `imf_raw`.
+    // Creates a `Message` that replies "Hi" to the incoming email in `imf_raw`.
     async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
         context
             .set_config(Config::ShowEmails, Some("2"))

src/mimeparser.rs

@@ -3,9 +3,9 @@ use std::future::Future;
 use std::pin::Pin;
 
 use deltachat_derive::{FromSql, ToSql};
-use lazy_static::lazy_static;
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
+use once_cell::sync::Lazy;
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
@@ -16,14 +16,14 @@ use crate::dc_tools::*;
 use crate::dehtml::dehtml;
 use crate::e2ee;
 use crate::error::{bail, Result};
-use crate::events::Event;
+use crate::events::EventType;
+use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
 use crate::location;
 use crate::message;
 use crate::param::*;
 use crate::peerstate::Peerstate;
-use crate::securejoin::handle_degrade_event;
 use crate::simplify::*;
 use crate::stock::StockMessage;
 
@@ -46,7 +46,14 @@ pub struct MimeMessage {
     pub from: Vec<SingleInfo>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
+
+    /// Set of valid signature fingerprints if a message is an
+    /// Autocrypt encrypted and signed message.
+    ///
+    /// If a message is not encrypted or the signature is not valid,
+    /// this set is empty.
     pub signatures: HashSet<Fingerprint>,
+
     pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
@@ -76,6 +83,13 @@ pub enum SystemMessage {
     SecurejoinMessage = 7,
     LocationStreamingEnabled = 8,
     LocationOnly = 9,
+
+    /// Chat ephemeral message timer is changed.
+    EphemeralTimerChanged = 10,
+
+    // Chat protection state changed
+    ChatProtectionEnabled = 11,
+    ChatProtectionDisabled = 12,
 }
 
 impl Default for SystemMessage {
@@ -113,58 +127,80 @@ impl MimeMessage {
 
         // remove headers that are allowed _only_ in the encrypted part
         headers.remove("secure-join-fingerprint");
+        headers.remove("chat-verified");
 
         // Memory location for a possible decrypted message.
         let mail_raw;
         let mut gossipped_addr = Default::default();
 
-        let (mail, signatures) = match e2ee::try_decrypt(context, &mail, message_time).await {
-            Ok((raw, signatures)) => {
-                if let Some(raw) = raw {
-                    // Valid autocrypt message, encrypted
-                    mail_raw = raw;
-                    let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
-                    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
-                        info!(context, "decrypted message mime-body:");
-                        println!("{}", String::from_utf8_lossy(&mail_raw));
-                    }
+        let (mail, signatures, warn_empty_signature) =
+            match e2ee::try_decrypt(context, &mail, message_time).await {
+                Ok((raw, signatures)) => {
+                    if let Some(raw) = raw {
+                        // Encrypted, but maybe unsigned message. Only if
+                        // `signatures` set is non-empty, it is a valid
+                        // autocrypt message.
 
-                    // Handle any gossip headers if the mail was encrypted.  See section
-                    // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
-                    let gossip_headers = decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
-                    gossipped_addr =
-                        update_gossip_peerstates(context, message_time, &mail, gossip_headers)
+                        mail_raw = raw;
+                        let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
+                        if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
+                            info!(context, "decrypted message mime-body:");
+                            println!("{}", String::from_utf8_lossy(&mail_raw));
+                        }
+
+                        // Handle any gossip headers if the mail was encrypted.  See section
+                        // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
+                        // but only if the mail was correctly signed:
+                        if !signatures.is_empty() {
+                            let gossip_headers =
+                                decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
+                            gossipped_addr = update_gossip_peerstates(
+                                context,
+                                message_time,
+                                &mail,
+                                gossip_headers,
+                            )
                             .await?;
+                        }
 
-                    // let known protected headers from the decrypted
-                    // part override the unencrypted top-level
-                    MimeMessage::merge_headers(
-                        context,
-                        &mut headers,
-                        &mut recipients,
-                        &mut from,
-                        &mut chat_disposition_notification_to,
-                        &decrypted_mail.headers,
-                    );
+                        // let known protected headers from the decrypted
+                        // part override the unencrypted top-level
 
-                    (decrypted_mail, signatures)
-                } else {
-                    // Message was not encrypted
-                    (mail, signatures)
+                        // Signature was checked for original From, so we
+                        // do not allow overriding it.
+                        let mut throwaway_from = from.clone();
+
+                        // We do not want to allow unencrypted subject in encrypted emails because the user might falsely think that the subject is safe.
+                        // See https://github.com/deltachat/deltachat-core-rust/issues/1790.
+                        headers.remove("subject");
+
+                        MimeMessage::merge_headers(
+                            context,
+                            &mut headers,
+                            &mut recipients,
+                            &mut throwaway_from,
+                            &mut chat_disposition_notification_to,
+                            &decrypted_mail.headers,
+                        );
+
+                        (decrypted_mail, signatures, true)
+                    } else {
+                        // Message was not encrypted
+                        (mail, signatures, false)
+                    }
                 }
-            }
-            Err(err) => {
-                // continue with the current, still encrypted, mime tree.
-                // unencrypted parts will be replaced by an error message
-                // that is added as "the message" to the chat then.
-                //
-                // if we just return here, the header is missing
-                // and the caller cannot display the message
-                // and try to assign the message to a chat
-                warn!(context, "decryption failed: {}", err);
-                (mail, Default::default())
-            }
-        };
+                Err(err) => {
+                    // continue with the current, still encrypted, mime tree.
+                    // unencrypted parts will be replaced by an error message
+                    // that is added as "the message" to the chat then.
+                    //
+                    // if we just return here, the header is missing
+                    // and the caller cannot display the message
+                    // and try to assign the message to a chat
+                    warn!(context, "decryption failed: {}", err);
+                    (mail, Default::default(), true)
+                }
+            };
 
         let mut parser = MimeMessage {
             parts: Vec::new(),
@@ -187,9 +223,16 @@ impl MimeMessage {
             failure_report: None,
         };
         parser.parse_mime_recursive(context, &mail).await?;
+        parser.maybe_remove_bad_parts().await;
         parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context)?;
 
+        if warn_empty_signature && parser.signatures.is_empty() {
+            for part in parser.parts.iter_mut() {
+                part.error = Some("No valid signature".to_string());
+            }
+        }
+
         Ok(parser)
     }
 
@@ -214,6 +257,12 @@ impl MimeMessage {
         } else if let Some(value) = self.get(HeaderDef::ChatContent) {
             if value == "location-streaming-enabled" {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
+            } else if value == "ephemeral-timer-changed" {
+                self.is_system_message = SystemMessage::EphemeralTimerChanged;
+            } else if value == "protection-enabled" {
+                self.is_system_message = SystemMessage::ChatProtectionEnabled;
+            } else if value == "protection-disabled" {
+                self.is_system_message = SystemMessage::ChatProtectionDisabled;
             }
         }
         Ok(())
@@ -230,10 +279,25 @@ impl MimeMessage {
         }
     }
 
+    fn parse_videochat_headers(&mut self) {
+        if let Some(value) = self.get(HeaderDef::ChatContent).cloned() {
+            if value == "videochat-invitation" {
+                let instance = self.get(HeaderDef::ChatWebrtcRoom).cloned();
+                if let Some(part) = self.parts.first_mut() {
+                    part.typ = Viewtype::VideochatInvitation;
+                    part.param
+                        .set(Param::WebrtcRoom, instance.unwrap_or_default());
+                }
+            }
+        }
+    }
+
     /// Squashes mutlipart chat messages with attachment into single-part messages.
     ///
     /// Delta Chat sends attachments, such as images, in two-part messages, with the first message
-    /// containing an explanation. If such a message is detected, first part can be safely dropped.
+    /// containing a description. If such a message is detected, text from the first part can be
+    /// moved to the second part, and the first part dropped.
+    #[allow(clippy::indexing_slicing)]
     fn squash_attachment_parts(&mut self) {
         if let [textpart, filepart] = &self.parts[..] {
             let need_drop = {
@@ -252,6 +316,9 @@ impl MimeMessage {
 
                 // insert new one
                 filepart.msg = self.parts[0].msg.clone();
+                if let Some(quote) = self.parts[0].param.get(Param::Quote) {
+                    filepart.param.set(Param::Quote, quote);
+                }
 
                 // forget the one we use now
                 self.parts[0].msg = "".to_string();
@@ -267,22 +334,21 @@ impl MimeMessage {
     fn parse_attachments(&mut self) {
         // Attachment messages should be squashed into a single part
         // before calling this function.
-        if self.parts.len() == 1 {
-            if self.parts[0].typ == Viewtype::Audio
-                && self.get(HeaderDef::ChatVoiceMessage).is_some()
-            {
-                let part_mut = &mut self.parts[0];
-                part_mut.typ = Viewtype::Voice;
+        if self.parts.len() != 1 {
+            return;
+        }
+
+        if let Some(mut part) = self.parts.pop() {
+            if part.typ == Viewtype::Audio && self.get(HeaderDef::ChatVoiceMessage).is_some() {
+                part.typ = Viewtype::Voice;
             }
-            if self.parts[0].typ == Viewtype::Image {
+            if part.typ == Viewtype::Image {
                 if let Some(value) = self.get(HeaderDef::ChatContent) {
                     if value == "sticker" {
-                        let part_mut = &mut self.parts[0];
-                        part_mut.typ = Viewtype::Sticker;
+                        part.typ = Viewtype::Sticker;
                     }
                 }
             }
-            let part = &self.parts[0];
             if part.typ == Viewtype::Audio
                 || part.typ == Viewtype::Voice
                 || part.typ == Viewtype::Video
@@ -290,17 +356,19 @@ impl MimeMessage {
                 if let Some(field_0) = self.get(HeaderDef::ChatDuration) {
                     let duration_ms = field_0.parse().unwrap_or_default();
                     if duration_ms > 0 && duration_ms < 24 * 60 * 60 * 1000 {
-                        let part_mut = &mut self.parts[0];
-                        part_mut.param.set_int(Param::Duration, duration_ms);
+                        part.param.set_int(Param::Duration, duration_ms);
                     }
                 }
             }
+
+            self.parts.push(part);
         }
     }
 
     fn parse_headers(&mut self, context: &Context) -> Result<()> {
         self.parse_system_message_headers(context)?;
         self.parse_avatar_headers();
+        self.parse_videochat_headers();
         self.squash_attachment_parts();
 
         if let Some(ref subject) = self.get_subject() {
@@ -316,12 +384,11 @@ impl MimeMessage {
                 }
             }
             if prepend_subject {
-                let subj = if let Some(n) = subject.find('[') {
-                    &subject[0..n]
-                } else {
-                    subject
-                }
-                .trim();
+                let subj = subject
+                    .find('[')
+                    .and_then(|n| subject.get(..n))
+                    .unwrap_or(subject)
+                    .trim();
 
                 if !subj.is_empty() {
                     for part in self.parts.iter_mut() {
@@ -379,8 +446,7 @@ impl MimeMessage {
             Some(AvatarAction::Delete)
         } else {
             let mut i = 0;
-            while i != self.parts.len() {
-                let part = &mut self.parts[i];
+            while let Some(part) = self.parts.get_mut(i) {
                 if let Some(part_filename) = &part.org_filename {
                     if part_filename == &header_value {
                         if let Some(blob) = part.param.get(Param::File) {
@@ -397,6 +463,11 @@ impl MimeMessage {
         }
     }
 
+    /// Returns true if the message was encrypted as defined in
+    /// Autocrypt standard.
+    ///
+    /// This means the message was both encrypted and signed with a
+    /// valid signature.
     pub fn was_encrypted(&self) -> bool {
         !self.signatures.is_empty()
     }
@@ -532,7 +603,7 @@ impl MimeMessage {
                 part.typ = Viewtype::Text;
                 part.msg_raw = Some(txt.clone());
                 part.msg = txt;
-                part.error = "Decryption failed".to_string();
+                part.error = Some("Decryption failed".to_string());
 
                 self.parts.push(part);
 
@@ -544,11 +615,11 @@ impl MimeMessage {
                 contains exactly two body parts.  The first body
                 part is the body part over which the digital signature was created [...]
                 The second body part contains the control information necessary to
-                verify the digital signature." We simpliy take the first body part and
+                verify the digital signature." We simply take the first body part and
                 skip the rest.  (see
                 https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html
                 for background information why we use encrypted+signed) */
-                if let Some(first) = mail.subparts.iter().next() {
+                if let Some(first) = mail.subparts.get(0) {
                     any_part_added = self.parse_mime_recursive(context, first).await?;
                 }
             }
@@ -585,7 +656,7 @@ impl MimeMessage {
                             }
                         }
                         Some(_) => {
-                            if let Some(first) = mail.subparts.iter().next() {
+                            if let Some(first) = mail.subparts.get(0) {
                                 any_part_added = self.parse_mime_recursive(context, first).await?;
                             }
                         }
@@ -647,23 +718,55 @@ impl MimeMessage {
                             }
                         };
 
-                        let (simplified_txt, is_forwarded) = if decoded_data.is_empty() {
-                            ("".into(), false)
+                        let mut dehtml_failed = false;
+
+                        let (simplified_txt, is_forwarded, top_quote) = if decoded_data.is_empty() {
+                            ("".to_string(), false, None)
                         } else {
                             let is_html = mime_type == mime::TEXT_HTML;
                             let out = if is_html {
-                                dehtml(&decoded_data)
+                                dehtml(&decoded_data).unwrap_or_else(|| {
+                                    dehtml_failed = true;
+                                    decoded_data.clone()
+                                })
                             } else {
                                 decoded_data.clone()
                             };
                             simplify(out, self.has_chat_version())
                         };
 
-                        if !simplified_txt.is_empty() {
+                        let is_format_flowed = if let Some(format) = mail.ctype.params.get("format")
+                        {
+                            format.as_str().to_ascii_lowercase() == "flowed"
+                        } else {
+                            false
+                        };
+
+                        let (simplified_txt, simplified_quote) = if mime_type.type_() == mime::TEXT
+                            && mime_type.subtype() == mime::PLAIN
+                            && is_format_flowed
+                        {
+                            let delsp = if let Some(delsp) = mail.ctype.params.get("delsp") {
+                                delsp.as_str().to_ascii_lowercase() == "yes"
+                            } else {
+                                false
+                            };
+                            let unflowed_text = unformat_flowed(&simplified_txt, delsp);
+                            let unflowed_quote = top_quote.map(|q| unformat_flowed(&q, delsp));
+                            (unflowed_text, unflowed_quote)
+                        } else {
+                            (simplified_txt, top_quote)
+                        };
+
+                        if !simplified_txt.is_empty() || simplified_quote.is_some() {
                             let mut part = Part::default();
+                            part.dehtlm_failed = dehtml_failed;
                             part.typ = Viewtype::Text;
                             part.mimetype = Some(mime_type);
                             part.msg = simplified_txt;
+                            if let Some(quote) = simplified_quote {
+                                part.param.set(Param::Quote, quote);
+                            }
                             part.msg_raw = Some(decoded_data);
                             self.do_add_single_part(part);
                         }
@@ -765,16 +868,12 @@ impl MimeMessage {
     }
 
     pub fn repl_msg_by_error(&mut self, error_msg: impl AsRef<str>) {
-        if self.parts.is_empty() {
-            return;
+        self.is_system_message = SystemMessage::Unknown;
+        if let Some(part) = self.parts.first_mut() {
+            part.typ = Viewtype::Text;
+            part.msg = format!("[{}]", error_msg.as_ref());
+            self.parts.truncate(1);
         }
-
-        let part = &mut self.parts[0];
-        part.typ = Viewtype::Text;
-        part.msg = format!("[{}]", error_msg.as_ref());
-        self.parts.truncate(1);
-
-        assert_eq!(self.parts.len(), 1);
     }
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
@@ -825,7 +924,11 @@ impl MimeMessage {
         report: &mailparse::ParsedMail<'_>,
     ) -> Result<Option<Report>> {
         // parse as mailheaders
-        let report_body = report.subparts[1].get_body_raw()?;
+        let report_body = if let Some(subpart) = report.subparts.get(1) {
+            subpart.get_body_raw()?
+        } else {
+            bail!("Report does not have second MIME part");
+        };
         let (report_fields, _) = mailparse::parse_headers(&report_body)?;
 
         // must be present
@@ -900,10 +1003,21 @@ impl MimeMessage {
         Ok(None)
     }
 
+    async fn maybe_remove_bad_parts(&mut self) {
+        let good_parts = self.parts.iter().filter(|p| !p.dehtlm_failed).count();
+        if good_parts == 0 {
+            // We have no good part but show at least one bad part in order to show anything at all
+            self.parts.truncate(1);
+        } else if good_parts < self.parts.len() {
+            self.parts.retain(|p| !p.dehtlm_failed);
+        }
+    }
+
     /// Some providers like GMX and Yahoo do not send standard NDNs (Non Delivery notifications).
     /// If you improve heuristics here you might also have to change prefetch_should_download() in imap/mod.rs.
     /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
-    async fn heuristically_parse_ndn(&mut self, context: &Context) -> Option<()> {
+    #[allow(clippy::indexing_slicing)]
+    async fn heuristically_parse_ndn(&mut self, context: &Context) {
         let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
             let from = from.to_ascii_lowercase();
             from.contains("mailer-daemon") || from.contains("mail-daemon")
@@ -911,9 +1025,8 @@ impl MimeMessage {
             false
         };
         if maybe_ndn && self.failure_report.is_none() {
-            lazy_static! {
-                static ref RE: regex::Regex = regex::Regex::new(r"Message-ID:(.*)").unwrap();
-            }
+            static RE: Lazy<regex::Regex> =
+                Lazy::new(|| regex::Regex::new(r"Message-ID:(.*)").unwrap());
             for captures in self
                 .parts
                 .iter()
@@ -933,7 +1046,6 @@ impl MimeMessage {
                 }
             }
         }
-        None // Always return None, we just return anything so that we can use the '?' operator.
     }
 
     /// Handle reports
@@ -953,23 +1065,43 @@ impl MimeMessage {
                 if let Some((chat_id, msg_id)) =
                     message::handle_mdn(context, from_id, original_message_id, sent_timestamp).await
                 {
-                    context.emit_event(Event::MsgRead { chat_id, msg_id });
+                    context.emit_event(EventType::MsgRead { chat_id, msg_id });
                 }
             }
         }
 
         if let Some(failure_report) = &self.failure_report {
-            let error = parts.iter().find(|p| p.typ == Viewtype::Text).map(|p| {
-                let msg = &p.msg;
-                match msg.find("\n--- ") {
-                    Some(footer_start) => &msg[..footer_start],
-                    None => msg,
-                }
-                .trim()
-            });
-            message::handle_ndn(context, failure_report, error).await
+            let error = parts
+                .iter()
+                .find(|p| p.typ == Viewtype::Text)
+                .map(|p| p.msg.clone());
+            if let Err(e) = message::handle_ndn(context, failure_report, error).await {
+                warn!(context, "Could not handle ndn: {}", e);
+            }
         }
     }
+
+    /// Returns timestamp of the parent message.
+    ///
+    /// If there is no parent message or it is not found in the
+    /// database, returns None.
+    pub async fn get_parent_timestamp(&self, context: &Context) -> Result<Option<i64>> {
+        let parent_timestamp = if let Some(field) = self
+            .get(HeaderDef::InReplyTo)
+            .and_then(|msgid| parse_message_id(msgid).ok())
+        {
+            context
+                .sql
+                .query_get_value_result(
+                    "SELECT timestamp FROM msgs WHERE rfc724_mid=?",
+                    paramsv![field],
+                )
+                .await?
+        } else {
+            None
+        };
+        Ok(parent_timestamp)
+    }
 }
 
 async fn update_gossip_peerstates(
@@ -989,7 +1121,7 @@ async fn update_gossip_peerstates(
                 .iter()
                 .any(|info| info.addr == header.addr.to_lowercase())
             {
-                let mut peerstate = Peerstate::from_addr(context, &header.addr).await;
+                let mut peerstate = Peerstate::from_addr(context, &header.addr).await?;
                 if let Some(ref mut peerstate) = peerstate {
                     peerstate.apply_gossip(header, message_time);
                     peerstate.save_to_db(&context.sql, false).await?;
@@ -999,9 +1131,7 @@ async fn update_gossip_peerstates(
                     peerstate = Some(p);
                 }
                 if let Some(peerstate) = peerstate {
-                    if peerstate.degrade_event.is_some() {
-                        handle_degrade_event(context, &peerstate).await?;
-                    }
+                    peerstate.handle_fingerprint_change(context).await?;
                 }
 
                 gossipped_addr.insert(header.addr.clone());
@@ -1031,6 +1161,7 @@ pub(crate) struct FailureReport {
     pub failed_recipient: Option<String>,
 }
 
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn parse_message_ids(ids: &str) -> Result<Vec<String>> {
     // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>`
     let mut msgids = Vec::new();
@@ -1058,11 +1189,21 @@ pub(crate) fn parse_message_id(ids: &str) -> Result<String> {
 }
 
 fn is_known(key: &str) -> bool {
-    match key {
-        "return-path" | "date" | "from" | "sender" | "reply-to" | "to" | "cc" | "bcc"
-        | "message-id" | "in-reply-to" | "references" | "subject" => true,
-        _ => false,
-    }
+    matches!(
+        key,
+        "return-path"
+            | "date"
+            | "from"
+            | "sender"
+            | "reply-to"
+            | "to"
+            | "cc"
+            | "bcc"
+            | "message-id"
+            | "in-reply-to"
+            | "references"
+            | "subject"
+    )
 }
 
 #[derive(Debug, Default, Clone)]
@@ -1074,7 +1215,8 @@ pub struct Part {
     pub bytes: usize,
     pub param: Params,
     org_filename: Option<String>,
-    pub error: String,
+    pub error: Option<String>,
+    dehtlm_failed: bool,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1176,9 +1318,9 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String
 }
 
 /// Returned addresses are normalized and lowercased.
-fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
+pub(crate) fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
     get_all_addresses_from_header(headers, |header_key| {
-        header_key == "to" || header_key == "cc"
+        header_key == "to" || header_key == "cc" || header_key == "bcc"
     })
 }
 
@@ -1223,6 +1365,8 @@ where
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use crate::test_utils::*;
 
@@ -1340,6 +1484,39 @@ mod tests {
         assert!(mimeparser.chat_disposition_notification_to.is_none());
     }
 
+    #[async_std::test]
+    async fn test_get_parent_timestamp() {
+        let context = TestContext::new().await;
+        let raw = b"From: foo@example.org\n\
+                    Content-Type: text/plain\n\
+                    Chat-Version: 1.0\n\
+                    In-Reply-To: <Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org>\n\
+                    \n\
+                    Some reply\n\
+                    ";
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(
+            mimeparser.get_parent_timestamp(&context.ctx).await.unwrap(),
+            None
+        );
+        let timestamp = 1570435529;
+        context
+            .ctx
+            .sql
+            .execute(
+                "INSERT INTO msgs (rfc724_mid, timestamp) VALUES(?,?)",
+                paramsv!["Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org", timestamp],
+            )
+            .await
+            .expect("Failed to write to the database");
+        assert_eq!(
+            mimeparser.get_parent_timestamp(&context.ctx).await.unwrap(),
+            Some(timestamp)
+        );
+    }
+
     #[async_std::test]
     async fn test_mimeparser_with_context() {
         let context = TestContext::new().await;
@@ -1433,6 +1610,28 @@ mod tests {
         assert!(mimeparser.group_avatar.unwrap().is_change());
     }
 
+    #[async_std::test]
+    async fn test_mimeparser_with_videochat() {
+        let t = TestContext::new().await;
+
+        let raw = include_bytes!("../test-data/message/videochat_invitation.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+        assert_eq!(mimeparser.parts.len(), 1);
+        assert_eq!(mimeparser.parts[0].typ, Viewtype::VideochatInvitation);
+        assert_eq!(
+            mimeparser.parts[0]
+                .param
+                .get(Param::WebrtcRoom)
+                .unwrap_or_default(),
+            "https://example.org/p2p/?roomname=6HiduoAn4xN"
+        );
+        assert!(mimeparser.parts[0]
+            .msg
+            .contains("https://example.org/p2p/?roomname=6HiduoAn4xN"));
+        assert_eq!(mimeparser.user_avatar, None);
+        assert_eq!(mimeparser.group_avatar, None);
+    }
+
     #[async_std::test]
     async fn test_mimeparser_message_kml() {
         let context = TestContext::new().await;
@@ -1699,6 +1898,55 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
         assert_eq!(message.parts[0].msg, "Hello!");
     }
 
+    #[async_std::test]
+    async fn test_hide_html_without_content() {
+        let t = TestContext::new().await;
+        let raw = br#"Date: Thu, 13 Feb 2020 22:41:20 +0000 (UTC)
+From: sender@example.com
+To: receiver@example.com
+Subject: Mail with inline attachment
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+	boundary="----=_Part_25_46172632.1581201680436"
+
+------=_Part_25_46172632.1581201680436
+Content-Type: text/html; charset=utf-8
+
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=Windows-1252">
+<meta name="GENERATOR" content="MSHTML 11.00.10570.1001"></head>
+<body><img align="baseline" alt="" src="cid:1712254131-1" border="0" hspace="0">
+</body>
+
+------=_Part_25_46172632.1581201680436
+Content-Type: application/pdf; name="some_pdf.pdf"
+Content-Transfer-Encoding: base64
+Content-Disposition: inline; filename="some_pdf.pdf"
+
+JVBERi0xLjUKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURl
+Y29kZT4+CnN0cmVhbQp4nGVOuwoCMRDs8xVbC8aZvC4Hx4Hno7ATAhZi56MTtPH33YtXiLKQ3ZnM
+MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
+------=_Part_25_46172632.1581201680436--
+"#;
+
+        let message = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::File);
+        assert_eq!(message.parts[0].msg, "");
+
+        // Make sure the file is there even though the html is wrong:
+        let param = &message.parts[0].param;
+        let blob: BlobObject = param
+            .get_blob(Param::File, &t.ctx, false)
+            .await
+            .unwrap()
+            .unwrap();
+        let f = async_std::fs::File::open(blob.to_abs_path()).await.unwrap();
+        let size = f.metadata().await.unwrap().len();
+        assert_eq!(size, 154);
+    }
+
     #[async_std::test]
     async fn parse_inline_image() {
         let context = TestContext::new().await;
@@ -1956,4 +2204,121 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
         let test = parse_message_ids("  < ").unwrap();
         assert!(test.is_empty());
     }
+
+    #[async_std::test]
+    async fn parse_format_flowed_quote() {
+        let context = TestContext::new().await;
+        let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+Subject: Re: swipe-to-reply
+MIME-Version: 1.0
+In-Reply-To: <bar@example.org>
+Date: Tue, 06 Oct 2020 00:00:00 +0000
+Chat-Version: 1.0
+Message-ID: <foo@example.org>
+To: bob <bob@example.org>
+From: alice <alice@example.org>
+
+> Long 
+> quote.
+
+Reply
+"##;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(
+            message.get_subject(),
+            Some("Re: swipe-to-reply".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Text);
+        assert_eq!(
+            message.parts[0].param.get(Param::Quote).unwrap(),
+            "Long quote."
+        );
+        assert_eq!(message.parts[0].msg, "Reply");
+    }
+
+    #[async_std::test]
+    async fn parse_quote_without_reply() {
+        let context = TestContext::new().await;
+        let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+Subject: Re: swipe-to-reply
+MIME-Version: 1.0
+In-Reply-To: <bar@example.org>
+Date: Tue, 06 Oct 2020 00:00:00 +0000
+Message-ID: <foo@example.org>
+To: bob <bob@example.org>
+From: alice <alice@example.org>
+
+> Just a quote.
+"##;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(
+            message.get_subject(),
+            Some("Re: swipe-to-reply".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Text);
+        assert_eq!(
+            message.parts[0].param.get(Param::Quote).unwrap(),
+            "Just a quote."
+        );
+        assert_eq!(message.parts[0].msg, "");
+    }
+
+    #[async_std::test]
+    async fn parse_quote_top_posting() {
+        let context = TestContext::new().await;
+        let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+Subject: Re: top posting
+MIME-Version: 1.0
+In-Reply-To: <bar@example.org>
+Message-ID: <foo@example.org>
+To: bob <bob@example.org>
+From: alice <alice@example.org>
+
+A reply.
+
+On 2020-10-25, Bob wrote:
+> A quote.
+"##;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(message.get_subject(), Some("Re: top posting".to_string()));
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Text);
+        assert_eq!(
+            message.parts[0].param.get(Param::Quote).unwrap(),
+            "A quote."
+        );
+        assert_eq!(message.parts[0].msg, "A reply.");
+    }
+
+    #[async_std::test]
+    async fn test_attachment_quote() {
+        let context = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/quote_attach.eml");
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+
+        assert_eq!(mimeparser.get_subject().unwrap(), "Message from Alice");
+        assert_eq!(mimeparser.parts.len(), 1);
+        assert_eq!(mimeparser.parts[0].msg, "Reply");
+        assert_eq!(
+            mimeparser.parts[0].param.get(Param::Quote).unwrap(),
+            "Quote"
+        );
+        assert_eq!(mimeparser.parts[0].typ, Viewtype::File);
+    }
 }

src/oauth2.rs

@@ -170,16 +170,14 @@ pub async fn dc_get_oauth2_access_token(
         }
 
         // ... and POST
-        let response = surf::post(post_url).body_form(&post_param);
-        if response.is_err() {
-            warn!(
-                context,
-                "Error calling OAuth2 at {}: {:?}", token_url, response
-            );
+        let mut req = surf::post(post_url).build();
+        if let Err(err) = req.body_form(&post_param) {
+            warn!(context, "Error calling OAuth2 at {}: {:?}", token_url, err);
             return None;
         }
 
-        let parsed: Result<Response, _> = response.unwrap().recv_json().await;
+        let client = surf::Client::new();
+        let parsed: Result<Response, _> = client.recv_json(req).await;
         if parsed.is_err() {
             warn!(
                 context,
@@ -305,7 +303,7 @@ impl Oauth2 {
 
                     let mut fqdn: String = String::from(domain.as_ref());
                     if !fqdn.ends_with('.') {
-                        fqdn.push_str(".");
+                        fqdn.push('.');
                     }
 
                     if let Ok(res) = resolver.mx_lookup(fqdn).await {
@@ -323,7 +321,7 @@ impl Oauth2 {
     }
 
     async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
-        let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
+        let userinfo_url = self.get_userinfo.unwrap_or("");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
 
         // should returns sth. as

src/param.rs

@@ -3,12 +3,13 @@ use std::fmt;
 use std::str;
 
 use async_std::path::PathBuf;
+use itertools::Itertools;
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
-use crate::error::{self, bail, ensure};
+use crate::error::{self, bail};
 use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 
@@ -40,16 +41,21 @@ pub enum Param {
     /// 'c' nor 'e' are preset, the messages is only transport encrypted.
     ErroneousE2ee = b'e',
 
-    /// For Messages: force unencrypted message, either `ForcePlaintext::AddAutocryptHeader` (1),
-    /// `ForcePlaintext::NoAutocryptHeader` (2) or 0.
+    /// For Messages: force unencrypted message, a value from `ForcePlaintext` enum.
     ForcePlaintext = b'u',
 
+    /// For Messages: do not include Autocrypt header.
+    SkipAutocrypt = b'o',
+
     /// For Messages
     WantsMdn = b'r',
 
     /// For Messages
     Forwarded = b'a',
 
+    /// For Messages: quoted text.
+    Quote = b'q',
+
     /// For Messages
     Cmd = b'S',
 
@@ -68,6 +74,9 @@ pub enum Param {
     /// For Messages
     AttachGroupImage = b'A',
 
+    /// For Messages
+    WebrtcRoom = b'V',
+
     /// For Messages: space-separated list of messaged IDs of forwarded copies.
     ///
     /// This is used when a [crate::message::Message] is in the
@@ -92,6 +101,12 @@ pub enum Param {
     Recipients = b'R',
 
     /// For Groups
+    ///
+    /// An unpromoted group has not had any messages sent to it and thus only exists on the
+    /// creator's device.  Any changes made to an unpromoted group do not need to send
+    /// system messages to the group members to update them of the changes.  Once a message
+    /// has been sent to a group it is promoted and group changes require sending system
+    /// messages to all members.
     Unpromoted = b'U',
 
     /// For Groups and Contacts
@@ -119,14 +134,6 @@ pub enum Param {
     MsgId = b'I',
 }
 
-/// Possible values for `Param::ForcePlaintext`.
-#[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive)]
-#[repr(u8)]
-pub enum ForcePlaintext {
-    AddAutocryptHeader = 1,
-    NoAutocryptHeader = 2,
-}
-
 /// An object for handling key=value parameter lists.
 ///
 /// The structure is serialized by calling `to_string()` on it.
@@ -143,7 +150,12 @@ impl fmt::Display for Params {
             if i > 0 {
                 writeln!(f)?;
             }
-            write!(f, "{}={}", *key as u8 as char, value)?;
+            write!(
+                f,
+                "{}={}",
+                *key as u8 as char,
+                value.split('\n').join("\n\n")
+            )?;
         }
         Ok(())
     }
@@ -154,27 +166,28 @@ impl str::FromStr for Params {
 
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let mut inner = BTreeMap::new();
-        for pair in s.trim().lines() {
-            let pair = pair.trim();
-            if pair.is_empty() {
-                continue;
-            }
-            // TODO: probably nicer using a regex
-            ensure!(pair.len() > 1, "Invalid key pair: '{}'", pair);
-            let mut split = pair.splitn(2, '=');
-            let key = split.next();
-            let value = split.next();
+        let mut lines = s.lines().peekable();
 
-            ensure!(key.is_some(), "Missing key");
-            ensure!(value.is_some(), "Missing value");
+        while let Some(line) = lines.next() {
+            if let [key, value] = line.splitn(2, '=').collect::<Vec<_>>()[..] {
+                let key = key.to_string();
+                let mut value = value.to_string();
+                while let Some(s) = lines.peek() {
+                    if !s.is_empty() {
+                        break;
+                    }
+                    lines.next();
+                    value.push('\n');
+                    value += lines.next().unwrap_or_default();
+                }
 
-            let key = key.unwrap_or_default().trim();
-            let value = value.unwrap_or_default().trim();
-
-            if let Some(key) = Param::from_u8(key.as_bytes()[0]) {
-                inner.insert(key, value.to_string());
+                if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
+                    inner.insert(key, value);
+                } else {
+                    bail!("Unknown key: {}", key);
+                }
             } else {
-                bail!("Unknown key: {}", key);
+                bail!("Not a key-value pair: {:?}", line);
             }
         }
 
@@ -370,7 +383,7 @@ mod tests {
 
     #[test]
     fn test_dc_param() {
-        let mut p1: Params = "\r\n\r\na=1\nf=2\n\nc = 3 ".parse().unwrap();
+        let mut p1: Params = "a=1\nf=2\nc=3".parse().unwrap();
 
         assert_eq!(p1.get_int(Param::Forwarded), Some(1));
         assert_eq!(p1.get_int(Param::File), Some(2));
@@ -404,6 +417,14 @@ mod tests {
         assert_eq!(p1.len(), 0)
     }
 
+    #[test]
+    fn test_roundtrip() {
+        let mut params = Params::new();
+        params.set(Param::Height, "foo\nbar=baz\nquux");
+        params.set(Param::Width, "\n\n\na=\n=");
+        assert_eq!(params.to_string().parse::<Params>().unwrap(), params);
+    }
+
     #[test]
     fn test_regression() {
         let p1: Params = "a=cli%40deltachat.de\nn=\ni=TbnwJ6lSvD5\ns=0ejvbdFSQxB"
@@ -466,8 +487,8 @@ mod tests {
         );
 
         // Blob in blobdir, expect blob.
-        let bar = t.ctx.get_blobdir().join("bar");
-        p.set(Param::File, bar.to_str().unwrap());
+        let bar_path = t.ctx.get_blobdir().join("bar");
+        p.set(Param::File, bar_path.to_str().unwrap());
         let blob = p
             .get_blob(Param::File, &t.ctx, false)
             .await

src/peerstate.rs

@@ -5,9 +5,14 @@ use std::fmt;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
+use crate::chat;
+use crate::constants::Blocked;
 use crate::context::Context;
+use crate::error::{bail, Result};
+use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::sql::Sql;
+use crate::stock::StockMessage;
 
 #[derive(Debug)]
 pub enum PeerstateKeyType {
@@ -38,7 +43,7 @@ pub struct Peerstate<'a> {
     pub verified_key: Option<SignedPublicKey>,
     pub verified_key_fingerprint: Option<Fingerprint>,
     pub to_save: Option<ToSave>,
-    pub degrade_event: Option<DegradeEvent>,
+    pub fingerprint_changed: bool,
 }
 
 impl<'a> PartialEq for Peerstate<'a> {
@@ -55,7 +60,7 @@ impl<'a> PartialEq for Peerstate<'a> {
             && self.verified_key == other.verified_key
             && self.verified_key_fingerprint == other.verified_key_fingerprint
             && self.to_save == other.to_save
-            && self.degrade_event == other.degrade_event
+            && self.fingerprint_changed == other.fingerprint_changed
     }
 }
 
@@ -76,7 +81,7 @@ impl<'a> fmt::Debug for Peerstate<'a> {
             .field("verified_key", &self.verified_key)
             .field("verified_key_fingerprint", &self.verified_key_fingerprint)
             .field("to_save", &self.to_save)
-            .field("degrade_event", &self.degrade_event)
+            .field("fingerprint_changed", &self.fingerprint_changed)
             .finish()
     }
 }
@@ -88,62 +93,51 @@ pub enum ToSave {
     All = 0x02,
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
-#[repr(u8)]
-pub enum DegradeEvent {
-    /// Recoverable by an incoming encrypted mail.
-    EncryptionPaused = 0x01,
-
-    /// Recoverable by a new verify.
-    FingerprintChanged = 0x02,
-}
-
 impl<'a> Peerstate<'a> {
-    pub fn new(context: &'a Context, addr: String) -> Self {
+    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
         Peerstate {
             context,
-            addr,
-            last_seen: 0,
-            last_seen_autocrypt: 0,
-            prefer_encrypt: Default::default(),
-            public_key: None,
-            public_key_fingerprint: None,
+            addr: header.addr.clone(),
+            last_seen: message_time,
+            last_seen_autocrypt: message_time,
+            prefer_encrypt: header.prefer_encrypt,
+            public_key: Some(header.public_key.clone()),
+            public_key_fingerprint: Some(header.public_key.fingerprint()),
             gossip_key: None,
             gossip_key_fingerprint: None,
             gossip_timestamp: 0,
             verified_key: None,
             verified_key_fingerprint: None,
-            to_save: None,
-            degrade_event: None,
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
         }
     }
 
-    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, header.addr.clone());
-
-        res.last_seen = message_time;
-        res.last_seen_autocrypt = message_time;
-        res.to_save = Some(ToSave::All);
-        res.prefer_encrypt = header.prefer_encrypt;
-        res.public_key = Some(header.public_key.clone());
-        res.recalc_fingerprint();
-
-        res
-    }
-
     pub fn from_gossip(context: &'a Context, gossip_header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, gossip_header.addr.clone());
-
-        res.gossip_timestamp = message_time;
-        res.to_save = Some(ToSave::All);
-        res.gossip_key = Some(gossip_header.public_key.clone());
-        res.recalc_fingerprint();
-
-        res
+        Peerstate {
+            context,
+            addr: gossip_header.addr.clone(),
+            last_seen: 0,
+            last_seen_autocrypt: 0,
+            prefer_encrypt: Default::default(),
+            public_key: None,
+            public_key_fingerprint: None,
+            gossip_key: Some(gossip_header.public_key.clone()),
+            gossip_key_fingerprint: Some(gossip_header.public_key.fingerprint()),
+            gossip_timestamp: message_time,
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        }
     }
 
-    pub async fn from_addr(context: &'a Context, addr: &str) -> Option<Peerstate<'a>> {
-        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, verified_key, verified_key_fingerprint FROM acpeerstates  WHERE addr=? COLLATE NOCASE;";
+    pub async fn from_addr(context: &'a Context, addr: &str) -> Result<Option<Peerstate<'a>>> {
+        let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
+                     gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
+                     verified_key, verified_key_fingerprint \
+                     FROM acpeerstates \
+                     WHERE addr=? COLLATE NOCASE;";
         Self::from_stmt(context, query, paramsv![addr]).await
     }
 
@@ -151,7 +145,7 @@ impl<'a> Peerstate<'a> {
         context: &'a Context,
         _sql: &Sql,
         fingerprint: &Fingerprint,
-    ) -> Option<Peerstate<'a>> {
+    ) -> Result<Option<Peerstate<'a>>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint \
@@ -167,51 +161,58 @@ impl<'a> Peerstate<'a> {
         context: &'a Context,
         query: &str,
         params: Vec<&dyn crate::ToSql>,
-    ) -> Option<Peerstate<'a>> {
-        context
+    ) -> Result<Option<Peerstate<'a>>> {
+        let peerstate = context
             .sql
-            .query_row(query, params, |row| {
+            .query_row_optional(query, params, |row| {
                 /* all the above queries start with this: SELECT
                 addr, last_seen, last_seen_autocrypt, prefer_encrypted,
                 public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
                 gossip_key_fingerprint, verified_key, verified_key_fingerprint
                 */
-                let mut res = Self::new(context, row.get(0)?);
 
-                res.last_seen = row.get(1)?;
-                res.last_seen_autocrypt = row.get(2)?;
-                res.prefer_encrypt = EncryptPreference::from_i32(row.get(3)?).unwrap_or_default();
-                res.gossip_timestamp = row.get(5)?;
-
-                res.public_key_fingerprint = row
-                    .get::<_, Option<String>>(7)?
-                    .map(|s| s.parse::<Fingerprint>())
-                    .transpose()?;
-                res.gossip_key_fingerprint = row
-                    .get::<_, Option<String>>(8)?
-                    .map(|s| s.parse::<Fingerprint>())
-                    .transpose()?;
-                res.verified_key_fingerprint = row
-                    .get::<_, Option<String>>(10)?
-                    .map(|s| s.parse::<Fingerprint>())
-                    .transpose()?;
-                res.public_key = row
-                    .get(4)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
-                res.gossip_key = row
-                    .get(6)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
-                res.verified_key = row
-                    .get(9)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
+                let res = Peerstate {
+                    context,
+                    addr: row.get(0)?,
+                    last_seen: row.get(1)?,
+                    last_seen_autocrypt: row.get(2)?,
+                    prefer_encrypt: EncryptPreference::from_i32(row.get(3)?).unwrap_or_default(),
+                    public_key: row
+                        .get(4)
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    public_key_fingerprint: row
+                        .get::<_, Option<String>>(7)?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    gossip_key: row
+                        .get(6)
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    gossip_key_fingerprint: row
+                        .get::<_, Option<String>>(8)?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    gossip_timestamp: row.get(5)?,
+                    verified_key: row
+                        .get(9)
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    verified_key_fingerprint: row
+                        .get::<_, Option<String>>(10)?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    to_save: None,
+                    fingerprint_changed: false,
+                };
 
                 Ok(res)
             })
-            .await
-            .ok()
+            .await?;
+        Ok(peerstate)
     }
 
     pub fn recalc_fingerprint(&mut self) {
@@ -225,7 +226,7 @@ impl<'a> Peerstate<'a> {
             {
                 self.to_save = Some(ToSave::All);
                 if old_public_fingerprint.is_some() {
-                    self.degrade_event = Some(DegradeEvent::FingerprintChanged);
+                    self.fingerprint_changed = true;
                 }
             }
         }
@@ -239,29 +240,57 @@ impl<'a> Peerstate<'a> {
                 || old_gossip_fingerprint != self.gossip_key_fingerprint
             {
                 self.to_save = Some(ToSave::All);
-                if old_gossip_fingerprint.is_some() {
-                    self.degrade_event = Some(DegradeEvent::FingerprintChanged);
+
+                // Warn about gossip key change only if there is no public key obtained from
+                // Autocrypt header, which overrides gossip key.
+                if old_gossip_fingerprint.is_some() && self.public_key_fingerprint.is_none() {
+                    self.fingerprint_changed = true;
                 }
             }
         }
     }
 
     pub fn degrade_encryption(&mut self, message_time: i64) {
-        if self.prefer_encrypt == EncryptPreference::Mutual {
-            self.degrade_event = Some(DegradeEvent::EncryptionPaused);
-        }
-
         self.prefer_encrypt = EncryptPreference::Reset;
         self.last_seen = message_time;
         self.to_save = Some(ToSave::All);
     }
 
+    /// Adds a warning to the chat corresponding to peerstate if fingerprint has changed.
+    pub(crate) async fn handle_fingerprint_change(&self, context: &Context) -> Result<()> {
+        if self.fingerprint_changed {
+            if let Some(contact_id) = context
+                .sql
+                .query_get_value_result(
+                    "SELECT id FROM contacts WHERE addr=?;",
+                    paramsv![self.addr],
+                )
+                .await?
+            {
+                let (contact_chat_id, _) =
+                    chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Deaddrop)
+                        .await
+                        .unwrap_or_default();
+
+                let msg = context
+                    .stock_string_repl_str(StockMessage::ContactSetupChanged, self.addr.clone())
+                    .await;
+
+                chat::add_info_msg(context, contact_chat_id, msg).await;
+                emit_event!(context, EventType::ChatModified(contact_chat_id));
+            } else {
+                bail!("contact with peerstate.addr {:?} not found", &self.addr);
+            }
+        }
+        Ok(())
+    }
+
     pub fn apply_header(&mut self, header: &Aheader, message_time: i64) {
         if self.addr.to_lowercase() != header.addr.to_lowercase() {
             return;
         }
 
-        if message_time > self.last_seen_autocrypt {
+        if message_time > self.last_seen {
             self.last_seen = message_time;
             self.last_seen_autocrypt = message_time;
             self.to_save = Some(ToSave::Timestamps);
@@ -269,11 +298,6 @@ impl<'a> Peerstate<'a> {
                 || header.prefer_encrypt == EncryptPreference::NoPreference)
                 && header.prefer_encrypt != self.prefer_encrypt
             {
-                if self.prefer_encrypt == EncryptPreference::Mutual
-                    && header.prefer_encrypt != EncryptPreference::Mutual
-                {
-                    self.degrade_event = Some(DegradeEvent::EncryptionPaused);
-                }
                 self.prefer_encrypt = header.prefer_encrypt;
                 self.to_save = Some(ToSave::All)
             }
@@ -400,21 +424,20 @@ impl<'a> Peerstate<'a> {
     }
 
     pub async fn save_to_db(&self, sql: &Sql, create: bool) -> crate::sql::Result<()> {
-        if create {
-            sql.execute(
-                "INSERT INTO acpeerstates (addr) VALUES(?);",
-                paramsv![self.addr],
-            )
-            .await?;
-        }
-
         if self.to_save == Some(ToSave::All) || create {
             sql.execute(
+                if create {
+                "INSERT INTO acpeerstates (last_seen, last_seen_autocrypt, prefer_encrypted, \
+                 public_key, gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
+                 verified_key, verified_key_fingerprint, addr \
+                ) VALUES(?,?,?,?,?,?,?,?,?,?,?)"
+                } else {
                 "UPDATE acpeerstates \
                  SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
                  public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
                  verified_key=?, verified_key_fingerprint=? \
-                 WHERE addr=?;",
+                 WHERE addr=?"
+                },
                 paramsv![
                     self.last_seen,
                     self.last_seen_autocrypt,
@@ -466,7 +489,6 @@ mod tests {
     use super::*;
     use crate::test_utils::*;
     use pretty_assertions::assert_eq;
-    use tempfile::TempDir;
 
     #[async_std::test]
     async fn test_peerstate_save_to_db() {
@@ -489,7 +511,7 @@ mod tests {
             verified_key: Some(pub_key.clone()),
             verified_key_fingerprint: Some(pub_key.fingerprint()),
             to_save: Some(ToSave::All),
-            degrade_event: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -499,7 +521,8 @@ mod tests {
 
         let peerstate_new = Peerstate::from_addr(&ctx.ctx, addr)
             .await
-            .expect("failed to load peerstate from db");
+            .expect("failed to load peerstate from db")
+            .expect("no peerstate found in the database");
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
@@ -507,7 +530,8 @@ mod tests {
         let peerstate_new2 =
             Peerstate::from_fingerprint(&ctx.ctx, &ctx.ctx.sql, &pub_key.fingerprint())
                 .await
-                .expect("failed to load peerstate from db");
+                .expect("failed to load peerstate from db")
+                .expect("no peerstate found in the database");
         assert_eq!(peerstate, peerstate_new2);
     }
 
@@ -531,7 +555,7 @@ mod tests {
             verified_key: None,
             verified_key_fingerprint: None,
             to_save: Some(ToSave::All),
-            degrade_event: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -565,7 +589,7 @@ mod tests {
             verified_key: None,
             verified_key_fingerprint: None,
             to_save: Some(ToSave::All),
-            degrade_event: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -579,13 +603,76 @@ mod tests {
 
         // clear to_save, as that is not persissted
         peerstate.to_save = None;
-        assert_eq!(peerstate, peerstate_new);
+        assert_eq!(Some(peerstate), peerstate_new);
     }
 
-    // TODO: don't copy this from stress.rs
-    #[allow(dead_code)]
-    struct TestContext {
-        ctx: Context,
-        dir: TempDir,
+    #[async_std::test]
+    async fn test_peerstate_load_db_defaults() {
+        let ctx = crate::test_utils::TestContext::new().await;
+        let addr = "hello@mail.com";
+
+        // Old code created peerstates with this code and updated
+        // other values later.  If UPDATE failed, other columns had
+        // default values, in particular fingerprints were set to
+        // empty strings instead of NULL. This should not be the case
+        // anymore, but the regression test still checks that defaults
+        // can be loaded without errors.
+        ctx.ctx
+            .sql
+            .execute("INSERT INTO acpeerstates (addr) VALUES(?)", paramsv![addr])
+            .await
+            .expect("Failed to write to the database");
+
+        let peerstate = Peerstate::from_addr(&ctx.ctx, addr)
+            .await
+            .expect("Failed to load peerstate from db")
+            .expect("Loaded peerstate is empty");
+
+        // Check that default values for fingerprints are treated like
+        // NULL.
+        assert_eq!(peerstate.public_key_fingerprint, None);
+        assert_eq!(peerstate.gossip_key_fingerprint, None);
+        assert_eq!(peerstate.verified_key_fingerprint, None);
+    }
+
+    #[async_std::test]
+    async fn test_peerstate_degrade_reordering() {
+        let context = crate::test_utils::TestContext::new().await.ctx;
+        let addr = "example@example.org";
+        let pub_key = alice_keypair().public;
+        let header = Aheader::new(addr.to_string(), pub_key, EncryptPreference::Mutual);
+
+        let mut peerstate = Peerstate {
+            context: &context,
+            addr: addr.to_string(),
+            last_seen: 0,
+            last_seen_autocrypt: 0,
+            prefer_encrypt: EncryptPreference::NoPreference,
+            public_key: None,
+            public_key_fingerprint: None,
+            gossip_key: None,
+            gossip_timestamp: 0,
+            gossip_key_fingerprint: None,
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: None,
+            fingerprint_changed: false,
+        };
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::NoPreference);
+
+        peerstate.apply_header(&header, 100);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
+
+        peerstate.degrade_encryption(300);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Reset);
+
+        // This has message time 200, while encryption was degraded at timestamp 300.
+        // Because of reordering, header should not be applied.
+        peerstate.apply_header(&header, 200);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Reset);
+
+        // Same header will be applied in the future.
+        peerstate.apply_header(&header, 400);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
     }
 }

src/pgp.rs

@@ -365,11 +365,13 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
         let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
 
         let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-        ensure!(!msgs.is_empty(), "No valid messages found");
-
-        match msgs[0].get_content()? {
-            Some(content) => Ok(content),
-            None => bail!("Decrypted message is empty"),
+        if let Some(msg) = msgs.first() {
+            match msg.get_content()? {
+                Some(content) => Ok(content),
+                None => bail!("Decrypted message is empty"),
+            }
+        } else {
+            bail!("No valid messages found")
         }
     })
     .await
@@ -379,7 +381,7 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
 mod tests {
     use super::*;
     use crate::test_utils::*;
-    use lazy_static::lazy_static;
+    use once_cell::sync::Lazy;
 
     #[test]
     fn test_split_armored_data_1() {
@@ -437,9 +439,9 @@ mod tests {
             let bob = bob_keypair();
             TestKeys {
                 alice_secret: alice.secret.clone(),
-                alice_public: alice.public.clone(),
+                alice_public: alice.public,
                 bob_secret: bob.secret.clone(),
-                bob_public: bob.public.clone(),
+                bob_public: bob.public,
             }
         }
     }
@@ -447,26 +449,29 @@ mod tests {
     /// The original text of [CTEXT_SIGNED]
     static CLEARTEXT: &[u8] = b"This is a test";
 
-    lazy_static! {
-        /// Initialised [TestKeys] for tests.
-        static ref KEYS: TestKeys = TestKeys::new();
+    /// Initialised [TestKeys] for tests.
+    static KEYS: Lazy<TestKeys> = Lazy::new(TestKeys::new);
 
-        /// A cyphertext encrypted to Alice & Bob, signed by Alice.
-        static ref CTEXT_SIGNED: String = {
-            let mut keyring = Keyring::new();
-            keyring.add(KEYS.alice_public.clone());
-            keyring.add(KEYS.bob_public.clone());
-            smol::block_on(pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))).unwrap()
-        };
+    /// A cyphertext encrypted to Alice & Bob, signed by Alice.
+    static CTEXT_SIGNED: Lazy<String> = Lazy::new(|| {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_public.clone());
+        keyring.add(KEYS.bob_public.clone());
+        futures_lite::future::block_on(pk_encrypt(
+            CLEARTEXT,
+            keyring,
+            Some(KEYS.alice_secret.clone()),
+        ))
+        .unwrap()
+    });
 
-        /// A cyphertext encrypted to Alice & Bob, not signed.
-        static ref CTEXT_UNSIGNED: String = {
-            let mut keyring = Keyring::new();
-            keyring.add(KEYS.alice_public.clone());
-            keyring.add(KEYS.bob_public.clone());
-            smol::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
-        };
-    }
+    /// A cyphertext encrypted to Alice & Bob, not signed.
+    static CTEXT_UNSIGNED: Lazy<String> = Lazy::new(|| {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_public.clone());
+        keyring.add(KEYS.bob_public.clone());
+        futures_lite::future::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
+    });
 
     #[test]
     fn test_encrypt_signed() {

src/provider/mod.rs

@@ -4,9 +4,10 @@ mod data;
 
 use crate::config::Config;
 use crate::dc_tools::EmailAddress;
-use crate::provider::data::PROVIDER_DATA;
+use crate::provider::data::{PROVIDER_DATA, PROVIDER_UPDATED};
+use chrono::{NaiveDateTime, NaiveTime};
 
-#[derive(Debug, Copy, Clone, PartialEq, ToPrimitive)]
+#[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum Status {
     OK = 1,
@@ -14,21 +15,29 @@ pub enum Status {
     BROKEN = 3,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, Display, PartialEq, Copy, Clone, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum Protocol {
     SMTP = 1,
     IMAP = 2,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, Display, PartialEq, Copy, Clone, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum Socket {
-    STARTTLS = 1,
-    SSL = 2,
+    Automatic = 0,
+    SSL = 1,
+    STARTTLS = 2,
+    Plain = 3,
 }
 
-#[derive(Debug, PartialEq)]
+impl Default for Socket {
+    fn default() -> Self {
+        Socket::Automatic
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
 #[repr(u8)]
 pub enum UsernamePattern {
     EMAIL = 1,
@@ -42,7 +51,7 @@ pub enum Oauth2Authorizer {
     Gmail = 2,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Server {
     pub protocol: Protocol,
     pub socket: Socket,
@@ -51,20 +60,6 @@ pub struct Server {
     pub username_pattern: UsernamePattern,
 }
 
-impl Server {
-    pub fn apply_username_pattern(&self, addr: String) -> String {
-        match self.username_pattern {
-            UsernamePattern::EMAIL => addr,
-            UsernamePattern::EMAILLOCALPART => {
-                if let Some(at) = addr.find('@') {
-                    return addr.split_at(at).0.to_string();
-                }
-                addr
-            }
-        }
-    }
-}
-
 #[derive(Debug)]
 pub struct ConfigDefault {
     pub key: Config,
@@ -80,28 +75,10 @@ pub struct Provider {
     pub server: Vec<Server>,
     pub config_defaults: Option<Vec<ConfigDefault>>,
     pub strict_tls: bool,
+    pub max_smtp_rcpt_to: Option<u16>,
     pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
-impl Provider {
-    pub fn get_server(&self, protocol: Protocol) -> Option<&Server> {
-        for record in self.server.iter() {
-            if record.protocol == protocol {
-                return Some(record);
-            }
-        }
-        None
-    }
-
-    pub fn get_imap_server(&self) -> Option<&Server> {
-        self.get_server(Protocol::IMAP)
-    }
-
-    pub fn get_smtp_server(&self) -> Option<&Server> {
-        self.get_server(Protocol::SMTP)
-    }
-}
-
 pub fn get_provider_info(addr: &str) -> Option<&Provider> {
     let domain = match addr.parse::<EmailAddress>() {
         Ok(addr) => addr.domain,
@@ -116,9 +93,18 @@ pub fn get_provider_info(addr: &str) -> Option<&Provider> {
     None
 }
 
+// returns update timestamp in seconds, UTC, compatible for comparison with time() and database times
+pub fn get_provider_update_timestamp() -> i64 {
+    NaiveDateTime::new(*PROVIDER_UPDATED, NaiveTime::from_hms(0, 0, 0)).timestamp_millis() / 1_000
+}
+
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
+    use crate::dc_tools::time;
+    use chrono::NaiveDate;
 
     #[test]
     fn test_get_provider_info_unexistant() {
@@ -137,15 +123,16 @@ mod tests {
         let provider = get_provider_info("nauta.cu"); // this is no email address
         assert!(provider.is_none());
 
-        let provider = get_provider_info("user@nauta.cu").unwrap();
+        let addr = "user@nauta.cu";
+        let provider = get_provider_info(addr).unwrap();
         assert!(provider.status == Status::OK);
-        let server = provider.get_imap_server().unwrap();
+        let server = &provider.server[0];
         assert_eq!(server.protocol, Protocol::IMAP);
         assert_eq!(server.socket, Socket::STARTTLS);
         assert_eq!(server.hostname, "imap.nauta.cu");
         assert_eq!(server.port, 143);
         assert_eq!(server.username_pattern, UsernamePattern::EMAIL);
-        let server = provider.get_smtp_server().unwrap();
+        let server = &provider.server[1];
         assert_eq!(server.protocol, Protocol::SMTP);
         assert_eq!(server.socket, Socket::STARTTLS);
         assert_eq!(server.hostname, "smtp.nauta.cu");
@@ -160,4 +147,16 @@ mod tests {
         let provider = get_provider_info("user@googlemail.com").unwrap();
         assert!(provider.status == Status::PREPARATION);
     }
+
+    #[test]
+    fn test_get_provider_update_timestamp() {
+        let timestamp_past = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 9, 9),
+            NaiveTime::from_hms(0, 0, 0),
+        )
+        .timestamp_millis()
+            / 1_000;
+        assert!(get_provider_update_timestamp() <= time());
+        assert!(get_provider_update_timestamp() > timestamp_past);
+    }
 }

src/provider/update.py

@@ -4,6 +4,7 @@
 import sys
 import os
 import yaml
+import datetime
 
 out_all = ""
 out_domains = ""
@@ -41,8 +42,8 @@ def process_config_defaults(data):
     config_defaults = data.get("config_defaults", "")
     for key in config_defaults:
         value = str(config_defaults[key])
-        defaults += "            ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
-    defaults += "        ])"
+        defaults += "        ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
+    defaults += "    ])"
     return defaults
 
 
@@ -65,7 +66,7 @@ def process_data(data, file):
             raise TypeError("domain used twice: " + domain)
         domains_dict[domain] = True
 
-        domains += "        (\"" + domain + "\", &*" + file2varname(file) + "),\n"
+        domains += "    (\"" + domain + "\", &*" + file2varname(file) + "),\n"
         comment += domain + ", "
 
 
@@ -95,7 +96,7 @@ def process_data(data, file):
             if username_pattern != "EMAIL" and username_pattern != "EMAILLOCALPART":
                 raise TypeError("bad username pattern")
 
-            server += ("            Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
+            server += ("        Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
             + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern + " },\n")
 
     config_defaults = process_config_defaults(data)
@@ -103,6 +104,9 @@ def process_data(data, file):
     strict_tls = data.get("strict_tls", False)
     strict_tls = "true" if strict_tls else "false"
 
+    max_smtp_rcpt_to = data.get("max_smtp_rcpt_to", 0)
+    max_smtp_rcpt_to = "Some(" + str(max_smtp_rcpt_to) + ")" if max_smtp_rcpt_to != 0 else "None"
+
     oauth2 = data.get("oauth2", "")
     oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
 
@@ -110,16 +114,17 @@ def process_data(data, file):
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
     if (not has_imap and not has_smtp) or (has_imap and has_smtp):
-        provider += "    static ref " + file2varname(file) + ": Provider = Provider {\n"
-        provider += "        status: Status::" + status + ",\n"
-        provider += "        before_login_hint: \"" + before_login_hint + "\",\n"
-        provider += "        after_login_hint: \"" + after_login_hint + "\",\n"
-        provider += "        overview_page: \"" + file2url(file) + "\",\n"
-        provider += "        server: vec![\n" + server + "        ],\n"
-        provider += "        config_defaults: " + config_defaults + ",\n"
-        provider += "        strict_tls: " + strict_tls + ",\n"
-        provider += "        oauth2_authorizer: " + oauth2 + ",\n"
-        provider += "    };\n\n"
+        provider += "static " + file2varname(file) + ": Lazy<Provider> = Lazy::new(|| Provider {\n"
+        provider += "    status: Status::" + status + ",\n"
+        provider += "    before_login_hint: \"" + before_login_hint + "\",\n"
+        provider += "    after_login_hint: \"" + after_login_hint + "\",\n"
+        provider += "    overview_page: \"" + file2url(file) + "\",\n"
+        provider += "    server: vec![\n" + server + "    ],\n"
+        provider += "    config_defaults: " + config_defaults + ",\n"
+        provider += "    strict_tls: " + strict_tls + ",\n"
+        provider += "    max_smtp_rcpt_to: " + max_smtp_rcpt_to + ",\n"
+        provider += "    oauth2_authorizer: " + oauth2 + ",\n"
+        provider += "});\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
 
@@ -128,7 +133,7 @@ def process_data(data, file):
 
     # finally, add the provider
     global out_all, out_domains
-    out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
+    out_all += "// " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
 
     # also add provider with no special things to do -
     # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
@@ -163,12 +168,16 @@ if __name__ == "__main__":
     "use crate::provider::UsernamePattern::*;\n"
     "use crate::provider::*;\n"
     "use std::collections::HashMap;\n\n"
-    "lazy_static::lazy_static! {\n\n")
+    "use once_cell::sync::Lazy;\n\n")
 
     process_dir(sys.argv[1])
 
-    out_all += "    pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [\n"
+    out_all += "pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
     out_all += out_domains;
-    out_all += "    ].iter().copied().collect();\n}"
+    out_all += "].iter().copied().collect());\n\n"
+
+    now = datetime.datetime.utcnow()
+    out_all += "pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> = "\
+               "Lazy::new(|| chrono::NaiveDate::from_ymd("+str(now.year)+", "+str(now.month)+", "+str(now.day)+"));\n"
 
     print(out_all)

src/qr.rs

@@ -1,6 +1,6 @@
 //! # QR code module
 
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use percent_encoding::percent_decode_str;
 use serde::Deserialize;
 
@@ -12,11 +12,13 @@ use crate::context::Context;
 use crate::error::{bail, ensure, format_err, Error};
 use crate::key::Fingerprint;
 use crate::lot::{Lot, LotState};
+use crate::message::Message;
 use crate::param::*;
 use crate::peerstate::*;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
+const DCWEBRTC_SCHEME: &str = "DCWEBRTC:";
 const MAILTO_SCHEME: &str = "mailto:";
 const MATMSG_SCHEME: &str = "MATMSG:";
 const VCARD_SCHEME: &str = "BEGIN:VCARD";
@@ -51,6 +53,8 @@ pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
         decode_openpgp(context, qr).await
     } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
         decode_account(context, qr)
+    } else if starts_with_ignore_case(qr, DCWEBRTC_SCHEME) {
+        decode_webrtc_instance(context, qr)
     } else if qr.starts_with(MAILTO_SCHEME) {
         decode_mailto(context, qr).await
     } else if qr.starts_with(SMTP_SCHEME) {
@@ -68,6 +72,7 @@ pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
 
 /// scheme: `OPENPGP4FPR:FINGERPRINT#a=ADDR&n=NAME&i=INVITENUMBER&s=AUTH`
 ///     or: `OPENPGP4FPR:FINGERPRINT#a=ADDR&g=GROUPNAME&x=GROUPID&i=INVITENUMBER&s=AUTH`
+#[allow(clippy::indexing_slicing)]
 async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[OPENPGP4FPR_SCHEME.len()..];
 
@@ -138,7 +143,10 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     let mut lot = Lot::new();
 
     // retrieve known state for this fingerprint
-    let peerstate = Peerstate::from_fingerprint(context, &context.sql, &fingerprint).await;
+    let peerstate = match Peerstate::from_fingerprint(context, &context.sql, &fingerprint).await {
+        Ok(peerstate) => peerstate,
+        Err(err) => return format_err!("Can't load peerstate: {}", err).into(),
+    };
 
     if invitenumber.is_none() || auth.is_none() {
         if let Some(peerstate) = peerstate {
@@ -187,13 +195,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 }
 
 /// scheme: `DCACCOUNT:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3`
+#[allow(clippy::indexing_slicing)]
 fn decode_account(_context: &Context, qr: &str) -> Lot {
     let payload = &qr[DCACCOUNT_SCHEME.len()..];
 
     let mut lot = Lot::new();
 
     if let Ok(url) = url::Url::parse(payload) {
-        if url.scheme() == "https" {
+        if url.scheme() == "http" || url.scheme() == "https" {
             lot.state = LotState::QrAccount;
             lot.text1 = url.host_str().map(|x| x.to_string());
         } else {
@@ -208,6 +217,31 @@ fn decode_account(_context: &Context, qr: &str) -> Lot {
     lot
 }
 
+/// scheme: `DCWEBRTC:https://meet.jit.si/$ROOM`
+#[allow(clippy::indexing_slicing)]
+fn decode_webrtc_instance(_context: &Context, qr: &str) -> Lot {
+    let payload = &qr[DCWEBRTC_SCHEME.len()..];
+
+    let mut lot = Lot::new();
+
+    let (_type, url) = Message::parse_webrtc_instance(payload);
+    if let Ok(url) = url::Url::parse(&url) {
+        if url.scheme() == "http" || url.scheme() == "https" {
+            lot.state = LotState::QrWebrtcInstance;
+            lot.text1 = url.host_str().map(|x| x.to_string());
+            lot.text2 = Some(payload.to_string())
+        } else {
+            lot.state = LotState::QrError;
+            lot.text1 = Some(format!("Bad scheme for webrtc instance: {}", payload));
+        }
+    } else {
+        lot.state = LotState::QrError;
+        lot.text1 = Some(format!("Invalid webrtc instance: {}", payload));
+    }
+
+    lot
+}
+
 #[derive(Debug, Deserialize)]
 struct CreateAccountResponse {
     email: String,
@@ -217,7 +251,8 @@ struct CreateAccountResponse {
 /// take a qr of the type DC_QR_ACCOUNT, parse it's parameters,
 /// download additional information from the contained url and set the parameters.
 /// on success, a configure::configure() should be able to log in to the account
-pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
+#[allow(clippy::indexing_slicing)]
+async fn set_account_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
     let url_str = &qr[DCACCOUNT_SCHEME.len()..];
 
     let response: Result<CreateAccountResponse, surf::Error> =
@@ -237,9 +272,24 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error
     Ok(())
 }
 
+pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
+    match check_qr(context, &qr).await.state {
+        LotState::QrAccount => set_account_from_qr(context, qr).await,
+        LotState::QrWebrtcInstance => {
+            let val = decode_webrtc_instance(context, qr).text2;
+            context
+                .set_config(Config::WebrtcInstance, val.as_deref())
+                .await?;
+            Ok(())
+        }
+        _ => bail!("qr code does not contain config: {}", qr),
+    }
+}
+
 /// Extract address for the mailto scheme.
 ///
 /// Scheme: `mailto:addr...?subject=...&body=..`
+#[allow(clippy::indexing_slicing)]
 async fn decode_mailto(context: &Context, qr: &str) -> Lot {
     let payload = &qr[MAILTO_SCHEME.len()..];
 
@@ -261,6 +311,7 @@ async fn decode_mailto(context: &Context, qr: &str) -> Lot {
 /// Extract address for the smtp scheme.
 ///
 /// Scheme: `SMTP:addr...:subject...:body...`
+#[allow(clippy::indexing_slicing)]
 async fn decode_smtp(context: &Context, qr: &str) -> Lot {
     let payload = &qr[SMTP_SCHEME.len()..];
 
@@ -283,6 +334,7 @@ async fn decode_smtp(context: &Context, qr: &str) -> Lot {
 /// Scheme: `MATMSG:TO:addr...;SUB:subject...;BODY:body...;`
 ///
 /// There may or may not be linebreaks after the fields.
+#[allow(clippy::indexing_slicing)]
 async fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     // Does not work when the text `TO:` is used in subject/body _and_ TO: is not the first field.
     // we ignore this case.
@@ -306,24 +358,23 @@ async fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     Lot::from_address(context, name, addr).await
 }
 
-lazy_static! {
-    static ref VCARD_NAME_RE: regex::Regex =
-        regex::Regex::new(r"(?m)^N:([^;]*);([^;\n]*)").unwrap();
-    static ref VCARD_EMAIL_RE: regex::Regex =
-        regex::Regex::new(r"(?m)^EMAIL([^:\n]*):([^;\n]*)").unwrap();
-}
+static VCARD_NAME_RE: Lazy<regex::Regex> =
+    Lazy::new(|| regex::Regex::new(r"(?m)^N:([^;]*);([^;\n]*)").unwrap());
+static VCARD_EMAIL_RE: Lazy<regex::Regex> =
+    Lazy::new(|| regex::Regex::new(r"(?m)^EMAIL([^:\n]*):([^;\n]*)").unwrap());
 
 /// Extract address for the matmsg scheme.
 ///
 /// Scheme: `VCARD:BEGIN\nN:last name;first name;...;\nEMAIL;<type>:addr...;
+#[allow(clippy::indexing_slicing)]
 async fn decode_vcard(context: &Context, qr: &str) -> Lot {
     let name = VCARD_NAME_RE
         .captures(qr)
-        .map(|caps| {
-            let last_name = &caps[1];
-            let first_name = &caps[2];
+        .and_then(|caps| {
+            let last_name = caps.get(1)?.as_str().trim();
+            let first_name = caps.get(2)?.as_str().trim();
 
-            format!("{} {}", first_name.trim(), last_name.trim())
+            Some(format!("{} {}", first_name, last_name))
         })
         .unwrap_or_default();
 
@@ -611,12 +662,31 @@ mod tests {
         assert_eq!(res.get_text1().unwrap(), "example.org");
     }
 
+    #[async_std::test]
+    async fn test_decode_webrtc_instance() {
+        let ctx = TestContext::new().await;
+
+        let res = check_qr(&ctx.ctx, "DCWEBRTC:basicwebrtc:https://basicurl.com/$ROOM").await;
+        assert_eq!(res.get_state(), LotState::QrWebrtcInstance);
+        assert_eq!(res.get_text1().unwrap(), "basicurl.com");
+        assert_eq!(
+            res.get_text2().unwrap(),
+            "basicwebrtc:https://basicurl.com/$ROOM"
+        );
+
+        // Test it again with mixcased "dcWebRTC:" uri scheme
+        let res = check_qr(&ctx.ctx, "dcWebRTC:https://example.org/").await;
+        assert_eq!(res.get_state(), LotState::QrWebrtcInstance);
+        assert_eq!(res.get_text1().unwrap(), "example.org");
+        assert_eq!(res.get_text2().unwrap(), "https://example.org/");
+    }
+
     #[async_std::test]
     async fn test_decode_account_bad_scheme() {
         let ctx = TestContext::new().await;
         let res = check_qr(
             &ctx.ctx,
-            "DCACCOUNT:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+            "DCACCOUNT:ftp://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
         )
         .await;
         assert_eq!(res.get_state(), LotState::QrError);
@@ -625,10 +695,40 @@ mod tests {
         // Test it again with lowercased "dcaccount:" uri scheme
         let res = check_qr(
             &ctx.ctx,
-            "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+            "dcaccount:ftp://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
         )
         .await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
     }
+
+    #[async_std::test]
+    async fn test_set_config_from_qr() {
+        let ctx = TestContext::new().await;
+
+        assert!(ctx.ctx.get_config(Config::WebrtcInstance).await.is_none());
+
+        let res = set_config_from_qr(&ctx.ctx, "badqr:https://example.org/").await;
+        assert!(!res.is_ok());
+        assert!(ctx.ctx.get_config(Config::WebrtcInstance).await.is_none());
+
+        let res = set_config_from_qr(&ctx.ctx, "https://no.qr").await;
+        assert!(!res.is_ok());
+        assert!(ctx.ctx.get_config(Config::WebrtcInstance).await.is_none());
+
+        let res = set_config_from_qr(&ctx.ctx, "dcwebrtc:https://example.org/").await;
+        assert!(res.is_ok());
+        assert_eq!(
+            ctx.ctx.get_config(Config::WebrtcInstance).await.unwrap(),
+            "https://example.org/"
+        );
+
+        let res =
+            set_config_from_qr(&ctx.ctx, "DCWEBRTC:basicwebrtc:https://foo.bar/?$ROOM&test").await;
+        assert!(res.is_ok());
+        assert_eq!(
+            ctx.ctx.get_config(Config::WebrtcInstance).await.unwrap(),
+            "basicwebrtc:https://foo.bar/?$ROOM&test"
+        );
+    }
 }

src/scheduler.rs

@@ -1,10 +1,9 @@
-#![warn(clippy::indexing_slicing)]
-
 use async_std::prelude::*;
 use async_std::sync::{channel, Receiver, Sender};
 use async_std::task;
 
 use crate::context::Context;
+use crate::dc_tools::maybe_add_time_based_warnings;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
 use crate::{config::Config, message::MsgId, smtp::Smtp};
@@ -76,6 +75,15 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                 }
                 None => {
                     jobs_loaded = 0;
+
+                    // Expunge folder if needed, e.g. if some jobs have
+                    // deleted messages on the server.
+                    if let Err(err) = connection.maybe_close_folder(&ctx).await {
+                        warn!(ctx, "failed to close folder: {:?}", err);
+                    }
+
+                    maybe_add_time_based_warnings(&ctx).await;
+
                     info = if ctx.get_config_bool(Config::InboxWatch).await {
                         fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
                     } else {
@@ -100,7 +108,7 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
     match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
             if let Err(err) = connection.connect_configured(&ctx).await {
-                error!(ctx, "{}", err);
+                error_network!(ctx, "{}", err);
                 return;
             }
 
@@ -122,8 +130,8 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
         Some(watch_folder) => {
             // connect and fake idle if unable to connect
             if let Err(err) = connection.connect_configured(&ctx).await {
-                error!(ctx, "imap connection failed: {}", err);
-                return connection.fake_idle(&ctx, None).await;
+                warn!(ctx, "imap connection failed: {}", err);
+                return connection.fake_idle(&ctx, Some(watch_folder)).await;
             }
 
             // fetch
@@ -407,10 +415,7 @@ impl Scheduler {
 
     /// Check if the scheduler is running.
     pub fn is_running(&self) -> bool {
-        match self {
-            Scheduler::Running { .. } => true,
-            _ => false,
-        }
+        matches!(self, Scheduler::Running { .. })
     }
 }
 

src/securejoin.rs

@@ -1,7 +1,8 @@
 //! Verified contact protocol implementation as [specified by countermitm project](https://countermitm.readthedocs.io/en/stable/new.html#setup-contact-protocol)
 
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
+use anyhow::{bail, Error};
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 
 use crate::aheader::EncryptPreference;
@@ -11,16 +12,16 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::e2ee::*;
-use crate::error::{bail, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
-use crate::lot::LotState;
+use crate::lot::{Lot, LotState};
 use crate::message::Message;
 use crate::mimeparser::*;
 use crate::param::*;
 use crate::peerstate::*;
 use crate::qr::check_qr;
+use crate::sql;
 use crate::stock::StockMessage;
 use crate::token;
 
@@ -32,7 +33,7 @@ macro_rules! joiner_progress {
             $progress >= 0 && $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::Event::SecurejoinJoinerProgress {
+        $context.emit_event($crate::events::EventType::SecurejoinJoinerProgress {
             contact_id: $contact_id,
             progress: $progress,
         });
@@ -45,7 +46,7 @@ macro_rules! inviter_progress {
             $progress >= 0 && $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::Event::SecurejoinInviterProgress {
+        $context.emit_event($crate::events::EventType::SecurejoinInviterProgress {
             contact_id: $contact_id,
             progress: $progress,
         });
@@ -67,6 +68,41 @@ macro_rules! get_qr_attr {
     };
 }
 
+/// State for setup-contact/secure-join protocol joiner's side.
+///
+/// The setup-contact protocol needs to carry state for both the inviter (Alice) and the
+/// joiner/invitee (Bob).  For Alice this state is minimal and in the `tokens` table in the
+/// database.  For Bob this state is only carried live on the [Context] in this struct.
+#[derive(Debug, Default)]
+pub(crate) struct Bob {
+    /// The next message expected by the protocol.
+    expects: SecureJoinStep,
+    /// The QR-scanned information of the currently running protocol.
+    pub qr_scan: Option<Lot>,
+}
+
+/// The next message expected by [Bob] in the setup-contact/secure-join protocol.
+#[derive(Debug, PartialEq)]
+enum SecureJoinStep {
+    /// No setup-contact protocol running.
+    NotActive,
+    /// Expecting the auth-required message.
+    ///
+    /// This corresponds to the `vc-auth-required` or `vg-auth-required` message of step 3d.
+    AuthRequired,
+    /// Expecting the contact-confirm message.
+    ///
+    /// This corresponds to the `vc-contact-confirm` or `vg-member-added` message of step
+    /// 6b.
+    ContactConfirm,
+}
+
+impl Default for SecureJoinStep {
+    fn default() -> Self {
+        Self::NotActive
+    }
+}
+
 pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<String> {
     /*=======================================================
     ====             Alice - the inviter side            ====
@@ -152,78 +188,79 @@ async fn get_self_fingerprint(context: &Context) -> Option<Fingerprint> {
     }
 }
 
-async fn cleanup(
-    context: &Context,
-    contact_chat_id: ChatId,
-    ongoing_allocated: bool,
-    join_vg: bool,
-) -> ChatId {
+async fn cleanup(context: &Context, ongoing_allocated: bool) {
     let mut bob = context.bob.write().await;
-    bob.expects = 0;
-    let ret_chat_id: ChatId = if bob.status == DC_BOB_SUCCESS {
-        if join_vg {
-            chat::get_chat_id_by_grpid(
-                context,
-                bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap(),
-            )
-            .await
-            .unwrap_or((ChatId::new(0), false, Blocked::Not))
-            .0
-        } else {
-            contact_chat_id
-        }
-    } else {
-        ChatId::new(0)
-    };
+    bob.expects = SecureJoinStep::NotActive;
     bob.qr_scan = None;
-
     if ongoing_allocated {
         context.free_ongoing().await;
     }
-    ret_chat_id
 }
 
-/// Take a scanned QR-code and do the setup-contact/join-group handshake.
-/// See the ffi-documentation for more details.
-pub async fn dc_join_securejoin(context: &Context, qr: &str) -> ChatId {
+#[derive(Debug, thiserror::Error)]
+pub enum JoinError {
+    #[error("Unknown QR-code")]
+    QrCode,
+    #[error("Aborted by user")]
+    Aborted,
+    #[error("Failed to send handshake message")]
+    SendMessage(#[from] SendMsgError),
+    // Note that this can currently only occur if there is a bug in the QR/Lot code as this
+    // is supposed to create a contact for us.
+    #[error("Unknown contact (this is a bug)")]
+    UnknownContact,
+    // Note that this can only occur if we failed to create the chat correctly.
+    #[error("No Chat found for group (this is a bug)")]
+    MissingChat(#[source] sql::Error),
+}
+
+/// Take a scanned QR-code and do the setup-contact/join-group/invite handshake.
+///
+/// This is the start of the process for the joiner.  See the module and ffi documentation
+/// for more details.
+///
+/// When joining a group this will start an "ongoing" process and will block until the
+/// process is completed, the [ChatId] for the new group is not known any sooner.  When
+/// verifying a contact this returns immediately.
+pub async fn dc_join_securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
     if context.alloc_ongoing().await.is_err() {
-        return cleanup(&context, ChatId::new(0), false, false).await;
+        cleanup(&context, false).await;
+        return Err(JoinError::Aborted);
     }
 
     securejoin(context, qr).await
 }
 
-async fn securejoin(context: &Context, qr: &str) -> ChatId {
+async fn securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
     /*========================================================
     ====             Bob - the joiner's side             =====
     ====   Step 2 in "Setup verified contact" protocol   =====
     ========================================================*/
 
-    let mut contact_chat_id = ChatId::new(0);
-    let mut join_vg: bool = false;
-
     info!(context, "Requesting secure-join ...",);
     ensure_secret_key_exists(context).await.ok();
     let qr_scan = check_qr(context, &qr).await;
     if qr_scan.state != LotState::QrAskVerifyContact && qr_scan.state != LotState::QrAskVerifyGroup
     {
         error!(context, "Unknown QR code.",);
-        return cleanup(&context, contact_chat_id, true, join_vg).await;
+        cleanup(&context, true).await;
+        return Err(JoinError::QrCode);
     }
-    contact_chat_id = match chat::create_by_contact_id(context, qr_scan.id).await {
+    let contact_chat_id = match chat::create_by_contact_id(context, qr_scan.id).await {
         Ok(chat_id) => chat_id,
         Err(_) => {
             error!(context, "Unknown contact.");
-            return cleanup(&context, contact_chat_id, true, join_vg).await;
+            cleanup(&context, true).await;
+            return Err(JoinError::UnknownContact);
         }
     };
     if context.shall_stop_ongoing().await {
-        return cleanup(&context, contact_chat_id, true, join_vg).await;
+        cleanup(&context, true).await;
+        return Err(JoinError::Aborted);
     }
-    join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
+    let join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
     {
         let mut bob = context.bob.write().await;
-        bob.status = 0;
         bob.qr_scan = Some(qr_scan);
     }
     if fingerprint_equals_sender(
@@ -245,7 +282,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         // the scanned fingerprint matches Alice's key,
         // we can proceed to step 4b) directly and save two mails
         info!(context, "Taking protocol shortcut.");
-        context.bob.write().await.expects = DC_VC_CONTACT_CONFIRM;
+        context.bob.write().await.expects = SecureJoinStep::ContactConfirm;
         joiner_progress!(
             context,
             chat_id_2_contact_id(context, contact_chat_id).await,
@@ -273,10 +310,11 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         .await
         {
             error!(context, "failed to send handshake message: {}", err);
-            return cleanup(&context, contact_chat_id, true, join_vg).await;
+            cleanup(&context, true).await;
+            return Err(JoinError::SendMessage(err));
         }
     } else {
-        context.bob.write().await.expects = DC_VC_AUTH_REQUIRED;
+        context.bob.write().await.expects = SecureJoinStep::AuthRequired;
 
         // Bob -> Alice
         if let Err(err) = send_handshake_msg(
@@ -290,7 +328,8 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         .await
         {
             error!(context, "failed to send handshake message: {}", err);
-            return cleanup(&context, contact_chat_id, true, join_vg).await;
+            cleanup(&context, true).await;
+            return Err(JoinError::SendMessage(err));
         }
     }
 
@@ -299,15 +338,45 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         while !context.shall_stop_ongoing().await {
             async_std::task::sleep(Duration::from_millis(50)).await;
         }
-        cleanup(&context, contact_chat_id, true, join_vg).await
+
+        // handle_securejoin_handshake() calls Context::stop_ongoing before the group chat
+        // is created (it is created after handle_securejoin_handshake() returns by
+        // dc_receive_imf()).  As a hack we just wait a bit for it to appear.
+        let start = Instant::now();
+        let chatid = loop {
+            {
+                let bob = context.bob.read().await;
+                let grpid = bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap();
+                match chat::get_chat_id_by_grpid(context, grpid).await {
+                    Ok((chatid, _is_protected, _blocked)) => break chatid,
+                    Err(err) => {
+                        if start.elapsed() > Duration::from_secs(7) {
+                            return Err(JoinError::MissingChat(err));
+                        }
+                    }
+                }
+            }
+            async_std::task::sleep(Duration::from_millis(50)).await;
+        };
+
+        cleanup(&context, true).await;
+        Ok(chatid)
     } else {
         // for a one-to-one-chat, the chat is already known, return the chat-id,
         // the verification runs in background
         context.free_ongoing().await;
-        contact_chat_id
+        Ok(contact_chat_id)
     }
 }
 
+/// Error for [send_handshake_msg].
+///
+/// Wrapping the [anyhow::Error] means we can "impl From" more easily on errors from this
+/// function.
+#[derive(Debug, thiserror::Error)]
+#[error("Failed sending handshake message")]
+pub struct SendMsgError(#[from] anyhow::Error);
+
 async fn send_handshake_msg(
     context: &Context,
     contact_chat_id: ChatId,
@@ -315,7 +384,7 @@ async fn send_handshake_msg(
     param2: impl AsRef<str>,
     fingerprint: Option<Fingerprint>,
     grpid: impl AsRef<str>,
-) -> Result<(), HandshakeError> {
+) -> Result<(), SendMsgError> {
     let mut msg = Message::default();
     msg.viewtype = Viewtype::Text;
     msg.text = Some(format!("Secure-Join: {}", step));
@@ -336,25 +405,18 @@ async fn send_handshake_msg(
         msg.param.set(Param::Arg4, grpid.as_ref());
     }
     if step == "vg-request" || step == "vc-request" {
-        msg.param.set_int(
-            Param::ForcePlaintext,
-            ForcePlaintext::AddAutocryptHeader as i32,
-        );
+        msg.param.set_int(Param::ForcePlaintext, 1);
     } else {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
     }
 
-    chat::send_msg(context, contact_chat_id, &mut msg)
-        .await
-        .map_err(HandshakeError::MsgSendFailed)?;
-
+    chat::send_msg(context, contact_chat_id, &mut msg).await?;
     Ok(())
 }
 
 async fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32 {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
-    if contacts.len() == 1 {
-        contacts[0]
+    if let [contact_id] = chat::get_chat_contacts(context, contact_chat_id).await[..] {
+        contact_id
     } else {
         0
     }
@@ -365,11 +427,22 @@ async fn fingerprint_equals_sender(
     fingerprint: &Fingerprint,
     contact_chat_id: ChatId,
 ) -> bool {
-    let contacts = chat::get_chat_contacts(context, contact_chat_id).await;
+    if let [contact_id] = chat::get_chat_contacts(context, contact_chat_id).await[..] {
+        if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
+            let peerstate = match Peerstate::from_addr(context, contact.get_addr()).await {
+                Ok(peerstate) => peerstate,
+                Err(err) => {
+                    warn!(
+                        context,
+                        "Failed to sender peerstate for {}: {}",
+                        contact.get_addr(),
+                        err
+                    );
+                    return false;
+                }
+            };
 
-    if contacts.len() == 1 {
-        if let Ok(contact) = Contact::load_from_db(context, contacts[0]).await {
-            if let Some(peerstate) = Peerstate::from_addr(context, contact.get_addr()).await {
+            if let Some(peerstate) = peerstate {
                 if peerstate.public_key_fingerprint.is_some()
                     && fingerprint == peerstate.public_key_fingerprint.as_ref().unwrap()
                 {
@@ -397,7 +470,7 @@ pub(crate) enum HandshakeError {
     #[error("No configured self address found")]
     NoSelfAddr,
     #[error("Failed to send message")]
-    MsgSendFailed(#[source] Error),
+    MsgSendFailed(#[from] SendMsgError),
     #[error("Failed to parse fingerprint")]
     BadFingerprint(#[from] crate::key::FingerprintError),
 }
@@ -426,6 +499,7 @@ pub(crate) enum HandshakeMessage {
 /// When handle_securejoin_handshake() is called,
 /// the message is not yet filed in the database;
 /// this is done by receive_imf() later on as needed.
+#[allow(clippy::indexing_slicing)]
 pub(crate) async fn handle_securejoin_handshake(
     context: &Context,
     mime_message: &MimeMessage,
@@ -510,7 +584,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 let bob = context.bob.read().await;
                 let scan = bob.qr_scan.as_ref();
                 scan.is_none()
-                    || bob.expects != DC_VC_AUTH_REQUIRED
+                    || bob.expects != SecureJoinStep::AuthRequired
                     || join_vg && scan.unwrap().state != LotState::QrAskVerifyGroup
             };
 
@@ -534,7 +608,6 @@ pub(crate) async fn handle_securejoin_handshake(
                     },
                 )
                 .await;
-                context.bob.write().await.status = 0; // secure-join failed
                 context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
@@ -547,14 +620,13 @@ pub(crate) async fn handle_securejoin_handshake(
                     "Fingerprint mismatch on joiner-side.",
                 )
                 .await;
-                context.bob.write().await.status = 0; // secure-join failed
                 context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
             let own_fingerprint = get_self_fingerprint(context).await.unwrap();
             joiner_progress!(context, contact_id, 400);
-            context.bob.write().await.expects = DC_VC_CONTACT_CONFIRM;
+            context.bob.write().await.expects = SecureJoinStep::ContactConfirm;
 
             // Bob -> Alice
             send_handshake_msg(
@@ -642,7 +714,7 @@ pub(crate) async fn handle_securejoin_handshake(
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited).await;
             info!(context, "Auth verified.",);
             secure_connection_established(context, contact_chat_id).await;
-            emit_event!(context, Event::ContactsChanged(Some(contact_id)));
+            emit_event!(context, EventType::ContactsChanged(Some(contact_id)));
             inviter_progress!(context, contact_id, 600);
             if join_vg {
                 // the vg-member-added message is special:
@@ -698,7 +770,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 HandshakeMessage::Ignore
             };
 
-            if context.bob.read().await.expects != DC_VC_CONTACT_CONFIRM {
+            if context.bob.read().await.expects != SecureJoinStep::ContactConfirm {
                 info!(context, "Message belongs to a different handshake.",);
                 return Ok(abort_retval);
             }
@@ -719,19 +791,19 @@ pub(crate) async fn handle_securejoin_handshake(
 
             let vg_expect_encrypted = if join_vg {
                 let group_id = get_qr_attr!(context, text2).to_string();
-                // This is buggy, is_verified_group will always be
+                // This is buggy, is_protected_group will always be
                 // false since the group is created by receive_imf by
                 // the very handshake message we're handling now.  But
                 // only after we have returned.  It does not impact
                 // the security invariants of secure-join however.
-                let (_, is_verified_group, _) = chat::get_chat_id_by_grpid(context, &group_id)
+                let (_, is_protected_group, _) = chat::get_chat_id_by_grpid(context, &group_id)
                     .await
                     .unwrap_or((ChatId::new(0), false, Blocked::Not));
                 // when joining a non-verified group
                 // the vg-member-added message may be unencrypted
                 // when not all group members have keys or prefer encryption.
                 // So only expect encryption if this is a verified group
-                is_verified_group
+                is_protected_group
             } else {
                 // setup contact is always encrypted
                 true
@@ -745,7 +817,6 @@ pub(crate) async fn handle_securejoin_handshake(
                     "Contact confirm message not encrypted.",
                 )
                 .await;
-                context.bob.write().await.status = 0;
                 return Ok(abort_retval);
             }
 
@@ -762,7 +833,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 return Ok(abort_retval);
             }
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinJoined).await;
-            emit_event!(context, Event::ContactsChanged(None));
+            emit_event!(context, EventType::ContactsChanged(None));
             let cg_member_added = mime_message
                 .get(HeaderDef::ChatGroupMemberAdded)
                 .map(|s| s.as_str())
@@ -777,7 +848,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 return Ok(abort_retval);
             }
             secure_connection_established(context, contact_chat_id).await;
-            context.bob.write().await.expects = 0;
+            context.bob.write().await.expects = SecureJoinStep::NotActive;
 
             // Bob -> Alice
             send_handshake_msg(
@@ -794,7 +865,6 @@ pub(crate) async fn handle_securejoin_handshake(
             )
             .await?;
 
-            context.bob.write().await.status = 1;
             context.stop_ongoing().await;
             Ok(if join_vg {
                 HandshakeMessage::Propagate
@@ -814,6 +884,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     return Ok(HandshakeMessage::Ignore);
                 }
                 if join_vg {
+                    // Responsible for showing "$Bob securely joined $group" message
                     inviter_progress!(context, contact_id, 800);
                     inviter_progress!(context, contact_id, 1000);
                     let field_grpid = mime_message
@@ -949,7 +1020,7 @@ async fn secure_connection_established(context: &Context, contact_chat_id: ChatI
         .stock_string_repl_str(StockMessage::ContactVerified, addr)
         .await;
     chat::add_info_msg(context, contact_chat_id, msg).await;
-    emit_event!(context, Event::ChatModified(contact_chat_id));
+    emit_event!(context, EventType::ChatModified(contact_chat_id));
 }
 
 async fn could_not_establish_secure_connection(
@@ -976,7 +1047,7 @@ async fn could_not_establish_secure_connection(
 
 async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> Result<(), Error> {
     if let Some(ref mut peerstate) =
-        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await
+        Peerstate::from_fingerprint(context, &context.sql, fingerprint).await?
     {
         if peerstate.set_verified(
             PeerstateKeyType::PublicKey,
@@ -1010,65 +1081,353 @@ fn encrypted_and_signed(
     if !mimeparser.was_encrypted() {
         warn!(context, "Message not encrypted.",);
         false
-    } else if mimeparser.signatures.is_empty() {
-        warn!(context, "Message not signed.",);
-        false
-    } else if expected_fingerprint.is_none() {
+    } else if let Some(expected_fingerprint) = expected_fingerprint {
+        if !mimeparser.signatures.contains(expected_fingerprint) {
+            warn!(
+                context,
+                "Message does not match expected fingerprint {}.", expected_fingerprint,
+            );
+            false
+        } else {
+            true
+        }
+    } else {
         warn!(context, "Fingerprint for comparison missing.");
         false
-    } else if !mimeparser
-        .signatures
-        .contains(expected_fingerprint.unwrap())
-    {
-        warn!(
-            context,
-            "Message does not match expected fingerprint {}.",
-            expected_fingerprint.unwrap(),
-        );
-        false
-    } else {
-        true
     }
 }
 
-pub async fn handle_degrade_event(
-    context: &Context,
-    peerstate: &Peerstate<'_>,
-) -> Result<(), Error> {
-    // - we do not issue an warning for DC_DE_ENCRYPTION_PAUSED as this is quite normal
-    // - currently, we do not issue an extra warning for DC_DE_VERIFICATION_LOST - this always comes
-    //   together with DC_DE_FINGERPRINT_CHANGED which is logged, the idea is not to bother
-    //   with things they cannot fix, so the user is just kicked from the verified group
-    //   (and he will know this and can fix this)
-    if Some(DegradeEvent::FingerprintChanged) == peerstate.degrade_event {
-        let contact_id: i32 = match context
-            .sql
-            .query_get_value(
-                context,
-                "SELECT id FROM contacts WHERE addr=?;",
-                paramsv![peerstate.addr],
-            )
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::chat;
+    use crate::chat::ProtectionStatus;
+    use crate::peerstate::Peerstate;
+    use crate::test_utils::TestContext;
+
+    #[async_std::test]
+    async fn test_setup_contact() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // Generate QR-code, ChatId(0) indicates setup-contact
+        let qr = dc_get_securejoin_qr(&alice.ctx, ChatId::new(0))
             .await
-        {
-            None => bail!(
-                "contact with peerstate.addr {:?} not found",
-                &peerstate.addr
-            ),
-            Some(contact_id) => contact_id,
-        };
-        if contact_id > 0 {
-            let (contact_chat_id, _) =
-                chat::create_or_lookup_by_contact_id(context, contact_id as u32, Blocked::Deaddrop)
-                    .await
-                    .unwrap_or_default();
+            .unwrap();
 
-            let msg = context
-                .stock_string_repl_str(StockMessage::ContactSetupChanged, peerstate.addr.clone())
-                .await;
+        // Bob scans QR-code, sends vc-request
+        let bob_chatid = dc_join_securejoin(&bob.ctx, &qr).await.unwrap();
 
-            chat::add_info_msg(context, contact_chat_id, msg).await;
-            emit_event!(context, Event::ChatModified(contact_chat_id));
-        }
+        let sent = bob.pop_sent_msg().await;
+        assert_eq!(sent.id(), bob_chatid);
+        assert_eq!(sent.recipient(), "alice@example.com".parse().unwrap());
+        let msg = alice.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vc-request");
+        assert!(msg.get(HeaderDef::SecureJoinInvitenumber).is_some());
+
+        // Alice receives vc-request, sends vc-auth-required
+        alice.recv_msg(&sent).await;
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vc-auth-required");
+
+        // Bob receives vc-auth-required, sends vc-request-with-auth
+        bob.recv_msg(&sent).await;
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-request-with-auth"
+        );
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
+        let bob_fp = SignedPublicKey::load_self(&bob.ctx)
+            .await
+            .unwrap()
+            .fingerprint();
+        assert_eq!(
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
+            bob_fp.hex()
+        );
+
+        // Alice should not yet have Bob verified
+        let contact_bob_id =
+            Contact::lookup_id_by_addr(&alice.ctx, "bob@example.net", Origin::Unknown).await;
+        let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Alice receives vc-request-with-auth, sends vc-contact-confirm
+        alice.recv_msg(&sent).await;
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm"
+        );
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob.ctx, "alice@example.com", Origin::Unknown).await;
+        let contact_alice = Contact::load_from_db(&bob.ctx, contact_alice_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&bob.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Bob receives vc-contact-confirm, sends vc-contact-confirm-received
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm-received"
+        );
+    }
+
+    #[async_std::test]
+    async fn test_setup_contact_bad_qr() {
+        let bob = TestContext::new_bob().await;
+        let ret = dc_join_securejoin(&bob.ctx, "not a qr code").await;
+        assert!(matches!(ret, Err(JoinError::QrCode)));
+    }
+
+    #[async_std::test]
+    async fn test_setup_contact_bob_knows_alice() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // Ensure Bob knows Alice_FP
+        let alice_pubkey = SignedPublicKey::load_self(&alice.ctx).await.unwrap();
+        let peerstate = Peerstate {
+            context: &bob.ctx,
+            addr: "alice@example.com".into(),
+            last_seen: 10,
+            last_seen_autocrypt: 10,
+            prefer_encrypt: EncryptPreference::Mutual,
+            public_key: Some(alice_pubkey.clone()),
+            public_key_fingerprint: Some(alice_pubkey.fingerprint()),
+            gossip_key: Some(alice_pubkey.clone()),
+            gossip_timestamp: 10,
+            gossip_key_fingerprint: Some(alice_pubkey.fingerprint()),
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        };
+        peerstate.save_to_db(&bob.ctx.sql, true).await.unwrap();
+
+        // Generate QR-code, ChatId(0) indicates setup-contact
+        let qr = dc_get_securejoin_qr(&alice.ctx, ChatId::new(0))
+            .await
+            .unwrap();
+
+        // Bob scans QR-code, sends vc-request-with-auth, skipping vc-request
+        dc_join_securejoin(&bob.ctx, &qr).await.unwrap();
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-request-with-auth"
+        );
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
+        let bob_fp = SignedPublicKey::load_self(&bob.ctx)
+            .await
+            .unwrap()
+            .fingerprint();
+        assert_eq!(
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
+            bob_fp.hex()
+        );
+
+        // Alice should not yet have Bob verified
+        let (contact_bob_id, _modified) = Contact::add_or_lookup(
+            &alice.ctx,
+            "Bob",
+            "bob@example.net",
+            Origin::ManuallyCreated,
+        )
+        .await
+        .unwrap();
+        let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Alice receives vc-request-with-auth, sends vc-contact-confirm
+        alice.recv_msg(&sent).await;
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm"
+        );
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob.ctx, "alice@example.com", Origin::Unknown).await;
+        let contact_alice = Contact::load_from_db(&bob.ctx, contact_alice_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&bob.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Bob receives vc-contact-confirm, sends vc-contact-confirm-received
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm-received"
+        );
+    }
+
+    #[async_std::test]
+    async fn test_secure_join() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chatid = chat::create_group_chat(&alice.ctx, ProtectionStatus::Protected, "the chat")
+            .await
+            .unwrap();
+
+        // Generate QR-code, secure-join implied by chatid
+        let qr = dc_get_securejoin_qr(&alice.ctx, chatid).await.unwrap();
+
+        // Bob scans QR-code, sends vg-request; blocks on ongoing process
+        let joiner = {
+            let qr = qr.clone();
+            let ctx = bob.ctx.clone();
+            async_std::task::spawn(async move { dc_join_securejoin(&ctx, &qr).await.unwrap() })
+        };
+
+        let sent = bob.pop_sent_msg().await;
+        assert_eq!(sent.recipient(), "alice@example.com".parse().unwrap());
+        let msg = alice.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-request");
+        assert!(msg.get(HeaderDef::SecureJoinInvitenumber).is_some());
+
+        // Alice receives vg-request, sends vg-auth-required
+        alice.recv_msg(&sent).await;
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-auth-required");
+
+        // Bob receives vg-auth-required, sends vg-request-with-auth
+        bob.recv_msg(&sent).await;
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vg-request-with-auth"
+        );
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
+        let bob_fp = SignedPublicKey::load_self(&bob.ctx)
+            .await
+            .unwrap()
+            .fingerprint();
+        assert_eq!(
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
+            bob_fp.hex()
+        );
+
+        // Alice should not yet have Bob verified
+        let contact_bob_id =
+            Contact::lookup_id_by_addr(&alice.ctx, "bob@example.net", Origin::Unknown).await;
+        let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Alice receives vg-request-with-auth, sends vg-member-added
+        alice.recv_msg(&sent).await;
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-member-added");
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob.ctx, "alice@example.com", Origin::Unknown).await;
+        let contact_alice = Contact::load_from_db(&bob.ctx, contact_alice_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&bob.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Bob receives vg-member-added, sends vg-member-added-received
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vg-member-added-received"
+        );
+
+        let bob_chatid = joiner.await;
+        let bob_chat = Chat::load_from_db(&bob.ctx, bob_chatid).await.unwrap();
+        assert!(bob_chat.is_protected());
     }
-    Ok(())
 }

src/simplify.rs

@@ -1,3 +1,5 @@
+use itertools::Itertools;
+
 // protect lines starting with `--` against being treated as a footer.
 // for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B);
 // this should be invisible on most systems and there is no need to unescape it again
@@ -7,14 +9,15 @@
 // but for non-delta-compatibility, that seems to be better.
 // (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced)
 pub fn escape_message_footer_marks(text: &str) -> String {
-    if text.starts_with("--") {
-        "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-")
+    if let Some(text) = text.strip_prefix("--") {
+        "-\u{200B}-".to_string() + &text.replace("\n--", "\n-\u{200B}-")
     } else {
         text.replace("\n--", "\n-\u{200B}-")
     }
 }
 
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
+#[allow(clippy::indexing_slicing)]
 fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
     let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
@@ -41,12 +44,11 @@ fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
 
 /// Remove nonstandard footer and a boolean indicating whether such
 /// footer was removed.
+#[allow(clippy::indexing_slicing)]
 fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     for (ix, &line) in lines.iter().enumerate() {
         if line == "--"
-            || line == "---"
-            || line == "----"
-            || line.starts_with("-----")
+            || line.starts_with("---")
             || line.starts_with("_____")
             || line.starts_with("=====")
             || line.starts_with("*****")
@@ -64,33 +66,32 @@ fn split_lines(buf: &str) -> Vec<&str> {
 
 /// Simplify message text for chat display.
 /// Remove quotes, signatures, trailing empty lines etc.
-pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
+pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool, Option<String>) {
     input.retain(|c| c != '\r');
     let lines = split_lines(&input);
     let (lines, is_forwarded) = skip_forward_header(&lines);
 
+    let (lines, mut top_quote) = remove_top_quote(lines);
     let original_lines = &lines;
-
     let lines = remove_message_footer(lines);
 
     let text = if is_chat_message {
-        render_message(lines, false, false)
+        render_message(lines, false)
     } else {
         let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
-        let (lines, has_bottom_quote) = remove_bottom_quote(lines);
-        let (lines, has_top_quote) = remove_top_quote(lines);
+        let (lines, mut bottom_quote) = remove_bottom_quote(lines);
+
+        if top_quote.is_none() && bottom_quote.is_some() {
+            std::mem::swap(&mut top_quote, &mut bottom_quote);
+        }
 
         if lines.iter().all(|it| it.trim().is_empty()) {
-            render_message(original_lines, false, false)
+            render_message(original_lines, false)
         } else {
-            render_message(
-                lines,
-                has_top_quote,
-                has_nonstandard_footer || has_bottom_quote,
-            )
+            render_message(lines, has_nonstandard_footer || bottom_quote.is_some())
         }
     };
-    (text, is_forwarded)
+    (text, is_forwarded, top_quote)
 }
 
 /// Skips "forwarded message" header.
@@ -107,16 +108,28 @@ fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
     }
 }
 
-fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+#[allow(clippy::indexing_slicing)]
+fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<String>) {
+    let mut first_quoted_line = lines.len();
     let mut last_quoted_line = None;
     for (l, line) in lines.iter().enumerate().rev() {
         if is_plain_quote(line) {
+            if last_quoted_line.is_none() {
+                first_quoted_line = l + 1;
+            }
             last_quoted_line = Some(l)
         } else if !is_empty_line(line) {
             break;
         }
     }
     if let Some(mut l_last) = last_quoted_line {
+        let quoted_text = lines[l_last..first_quoted_line]
+            .iter()
+            .map(|s| {
+                s.strip_prefix(">")
+                    .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u))
+            })
+            .join("\n");
         if l_last > 1 && is_empty_line(lines[l_last - 1]) {
             l_last -= 1
         }
@@ -126,17 +139,22 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
                 l_last -= 1
             }
         }
-        (&lines[..l_last], true)
+        (&lines[..l_last], Some(quoted_text))
     } else {
-        (lines, false)
+        (lines, None)
     }
 }
 
-fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+#[allow(clippy::indexing_slicing)]
+fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<String>) {
+    let mut first_quoted_line = 0;
     let mut last_quoted_line = None;
     let mut has_quoted_headline = false;
     for (l, line) in lines.iter().enumerate() {
         if is_plain_quote(line) {
+            if last_quoted_line.is_none() {
+                first_quoted_line = l;
+            }
             last_quoted_line = Some(l)
         } else if !is_empty_line(line) {
             if is_quoted_headline(line) && !has_quoted_headline && last_quoted_line.is_none() {
@@ -148,17 +166,25 @@ fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
         }
     }
     if let Some(last_quoted_line) = last_quoted_line {
-        (&lines[last_quoted_line + 1..], true)
+        (
+            &lines[last_quoted_line + 1..],
+            Some(
+                lines[first_quoted_line..last_quoted_line + 1]
+                    .iter()
+                    .map(|s| {
+                        s.strip_prefix(">")
+                            .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u))
+                    })
+                    .join("\n"),
+            ),
+        )
     } else {
-        (lines, false)
+        (lines, None)
     }
 }
 
-fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) -> String {
+fn render_message(lines: &[&str], is_cut_at_end: bool) -> String {
     let mut ret = String::new();
-    if is_cut_at_begin {
-        ret += "[...]";
-    }
     /* we write empty lines only in case and non-empty line follows */
     let mut pending_linebreaks = 0;
     let mut empty_body = true;
@@ -181,7 +207,7 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) ->
             pending_linebreaks = 1
         }
     }
-    if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
+    if is_cut_at_end && !empty_body {
         ret += " [...]";
     }
     // redo escaping done by escape_message_footer_marks()
@@ -229,7 +255,7 @@ mod tests {
         #[test]
         // proptest does not support [[:graphical:][:space:]] regex.
         fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let (output, _is_forwarded) = simplify(input, true);
+            let (output, _is_forwarded, _) = simplify(input, true);
             assert!(output.split('\n').all(|s| s != "-- "));
         }
     }
@@ -237,7 +263,7 @@ mod tests {
     #[test]
     fn test_dont_remove_whole_message() {
         let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, _) = simplify(input, false);
         assert_eq!(
             plain,
             "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
@@ -248,7 +274,7 @@ mod tests {
     #[test]
     fn test_chat_message() {
         let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
-        let (plain, is_forwarded) = simplify(input, true);
+        let (plain, is_forwarded, _) = simplify(input, true);
         assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
         assert!(!is_forwarded);
     }
@@ -256,7 +282,7 @@ mod tests {
     #[test]
     fn test_simplify_trim() {
         let input = "line1\n\r\r\rline2".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, _) = simplify(input, false);
 
         assert_eq!(plain, "line1\nline2");
         assert!(!is_forwarded);
@@ -265,7 +291,7 @@ mod tests {
     #[test]
     fn test_simplify_forwarded_message() {
         let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, _) = simplify(input, false);
 
         assert_eq!(plain, "Forwarded message");
         assert!(is_forwarded);
@@ -285,17 +311,17 @@ mod tests {
 
     #[test]
     fn test_remove_top_quote() {
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second"]);
+        let (lines, top_quote) = remove_top_quote(&["> first", "> second"]);
         assert!(lines.is_empty());
-        assert!(has_top_quote);
+        assert_eq!(top_quote.unwrap(), "first\nsecond");
 
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
+        let (lines, top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
         assert_eq!(lines, &["not a quote"]);
-        assert!(has_top_quote);
+        assert_eq!(top_quote.unwrap(), "first\nsecond");
 
-        let (lines, has_top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
+        let (lines, top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
         assert_eq!(lines, &["not a quote", "> first", "> second"]);
-        assert!(!has_top_quote);
+        assert!(top_quote.is_none());
     }
 
     #[test]
@@ -310,33 +336,41 @@ mod tests {
     #[test]
     fn test_remove_message_footer() {
         let input = "text\n--\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n--\nno footer");
 
         let input = "text\n\n--\n\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\n\nno footer");
 
         let input = "text\n\n-- no footer\n\n".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n\n-- no footer");
 
         let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\nno footer");
 
         let input = "text\n\n--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, _) = simplify(input.clone(), true);
         assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
+        let (plain, _, _) = simplify(escaped, true);
         assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
 
+        // Nonstandard footer sent by https://siju.es/
+        let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
+        let (plain, _, _) = simplify(input.clone(), false);
+        assert_eq!(plain, "Message text here [...]");
+        let (plain, _, _) = simplify(input.clone(), true);
+        assert_eq!(plain, input);
+
         let input = "--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, _) = simplify(input.clone(), true);
         assert_eq!(plain, ""); // see remove_message_footer() for some explanations
+
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
+        let (plain, _, _) = simplify(escaped, true);
         assert_eq!(plain, "--\ntreated as footer when unescaped");
     }
 }

src/smtp/mod.rs

@@ -1,20 +1,18 @@
 //! # SMTP transport module
 
-#![forbid(clippy::indexing_slicing)]
-
 pub mod send;
 
-use std::time::{Duration, Instant};
+use std::time::{Duration, SystemTime};
 
 use async_smtp::smtp::client::net::*;
 use async_smtp::*;
 
 use crate::constants::*;
 use crate::context::Context;
-use crate::events::Event;
-use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
+use crate::events::EventType;
+use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam};
 use crate::oauth2::*;
-use crate::provider::get_provider_info;
+use crate::provider::{get_provider_info, Socket};
 use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
@@ -32,7 +30,7 @@ pub enum Error {
         error: error::Error,
     },
 
-    #[error("SMTP: failed to connect: {0:?}")]
+    #[error("SMTP: failed to connect: {0}")]
     ConnectionFailure(#[source] smtp::error::Error),
 
     #[error("SMTP: failed to setup connection {0:?}")]
@@ -57,7 +55,7 @@ pub(crate) struct Smtp {
     /// Timestamp of last successful send/receive network interaction
     /// (eg connect or send succeeded). On initialization and disconnect
     /// it is set to None.
-    last_success: Option<Instant>,
+    last_success: Option<SystemTime>,
 }
 
 impl Smtp {
@@ -78,7 +76,11 @@ impl Smtp {
     /// have been successfully used the last 60 seconds
     pub async fn has_maybe_stale_connection(&self) -> bool {
         if let Some(last_success) = self.last_success {
-            Instant::now().duration_since(last_success).as_secs() > 60
+            SystemTime::now()
+                .duration_since(last_success)
+                .unwrap_or_default()
+                .as_secs()
+                > 60
         } else {
             false
         }
@@ -92,31 +94,65 @@ impl Smtp {
             .unwrap_or_default()
     }
 
+    /// Connect using configured parameters.
+    pub async fn connect_configured(&mut self, context: &Context) -> Result<()> {
+        if self.is_connected().await {
+            return Ok(());
+        }
+
+        let lp = LoginParam::from_database(context, "configured_").await;
+        let res = self
+            .connect(
+                context,
+                &lp.smtp,
+                &lp.addr,
+                lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+            )
+            .await;
+        if let Err(ref err) = res {
+            let message = context
+                .stock_string_repl_str2(
+                    StockMessage::ServerResponse,
+                    format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
+                    err.to_string(),
+                )
+                .await;
+
+            context.emit_event(EventType::ErrorNetwork(message));
+        };
+        res
+    }
+
     /// Connect using the provided login params.
-    pub async fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
+    pub async fn connect(
+        &mut self,
+        context: &Context,
+        lp: &ServerLoginParam,
+        addr: &str,
+        oauth2: bool,
+    ) -> Result<()> {
         if self.is_connected().await {
             warn!(context, "SMTP already connected.");
             return Ok(());
         }
 
-        if lp.send_server.is_empty() || lp.send_port == 0 {
-            context.emit_event(Event::ErrorNetwork("SMTP bad parameters.".into()));
+        if lp.server.is_empty() || lp.port == 0 {
             return Err(Error::BadParameters);
         }
 
         let from =
-            EmailAddress::new(lp.addr.clone()).map_err(|err| Error::InvalidLoginAddress {
-                address: lp.addr.clone(),
+            EmailAddress::new(addr.to_string()).map_err(|err| Error::InvalidLoginAddress {
+                address: addr.to_string(),
                 error: err,
             })?;
 
         self.from = Some(from);
 
-        let domain = &lp.send_server;
-        let port = lp.send_port as u16;
+        let domain = &lp.server;
+        let port = lp.port;
 
-        let provider = get_provider_info(&lp.addr);
-        let strict_tls = match lp.smtp_certificate_checks {
+        let provider = get_provider_info(addr);
+        let strict_tls = match lp.certificate_checks {
             CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
             CertificateChecks::Strict => true,
             CertificateChecks::AcceptInvalidCertificates
@@ -125,17 +161,16 @@ impl Smtp {
         let tls_config = dc_build_tls(strict_tls);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
-        let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
+        let (creds, mechanism) = if oauth2 {
             // oauth2
-            let addr = &lp.addr;
-            let send_pw = &lp.send_pw;
+            let send_pw = &lp.password;
             let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false).await;
             if access_token.is_none() {
                 return Err(Error::Oauth2Error {
                     address: addr.to_string(),
                 });
             }
-            let user = &lp.send_user;
+            let user = &lp.user;
             (
                 smtp::authentication::Credentials::new(
                     user.to_string(),
@@ -145,8 +180,8 @@ impl Smtp {
             )
         } else {
             // plain
-            let user = lp.send_user.clone();
-            let pw = lp.send_pw.clone();
+            let user = lp.user.clone();
+            let pw = lp.password.clone();
             (
                 smtp::authentication::Credentials::new(user, pw),
                 vec![
@@ -156,12 +191,9 @@ impl Smtp {
             )
         };
 
-        let security = if 0
-            != lp.server_flags & (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_PLAIN) as i32
-        {
-            smtp::ClientSecurity::Opportunistic(tls_parameters)
-        } else {
-            smtp::ClientSecurity::Wrapper(tls_parameters)
+        let security = match lp.security {
+            Socket::STARTTLS | Socket::Plain => smtp::ClientSecurity::Opportunistic(tls_parameters),
+            _ => smtp::ClientSecurity::Wrapper(tls_parameters),
         };
 
         let client = smtp::SmtpClient::with_security((domain.as_str(), port), security)
@@ -177,24 +209,15 @@ impl Smtp {
 
         let mut trans = client.into_transport();
         if let Err(err) = trans.connect().await {
-            let message = context
-                .stock_string_repl_str2(
-                    StockMessage::ServerResponse,
-                    format!("SMTP {}:{}", domain, port),
-                    format!("{}, ({:?})", err.to_string(), err),
-                )
-                .await;
-
-            emit_event!(context, Event::ErrorNetwork(message));
             return Err(Error::ConnectionFailure(err));
         }
 
         self.transport = Some(trans);
-        self.last_success = Some(Instant::now());
+        self.last_success = Some(SystemTime::now());
 
-        context.emit_event(Event::SmtpConnected(format!(
+        context.emit_event(EventType::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
-            lp.send_user,
+            lp.user,
         )));
 
         Ok(())