use newest crates

Top quotes are displayed as quotes for non-chat mails, so [...] used to
indicate there was a quote is not needed.

.circleci/config.yml

@@ -189,8 +189,6 @@ workflows:
               only: /.*/
 
       - remote_python_packaging:
-          requires: 
-             - remote_tests_python
           filters:
             branches:
               only: master

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all
+          args: --workspace --tests --examples
 
 
   build_and_test:
@@ -47,7 +47,9 @@ jobs:
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        # macOS disabled due to random failures related to caching
+        #os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest]
         rust: [1.45.0]
         experimental: [false]
 #        include:

CHANGELOG.md

@@ -1,5 +1,46 @@
 # Changelog 
 
+## 1.46.0
+
+- breaking change: `dc_configure()` report errors in
+  `DC_EVENT_CONFIGURE_PROGRESS`: capturing error events is no longer working
+  #1886 #1905
+
+- breaking change: removed `DC_LP_{IMAP|SMTP}_SOCKET*` from `server_flags`;
+  added `mail_security` and `send_security` using `DC_SOCKET` enum #1835
+
+- parse multiple servers in Mozilla autoconfig #1860
+
+- try multiple servers for each protocol #1871
+
+- do IMAP and SMTP configuration in parallel #1891
+
+- configuration cleanup and speedup #1858 #1875 #1889 #1904 #1906
+
+- secure-join cleanup, testing, fixing #1876 #1877 #1887 #1888 #1896 #1899 #1900
+
+- do not reset peerstate on encrypted messages,
+  ignore reordered autocrypt headers #1885 #1890
+
+- always sort message replies after parent message #1852
+
+- add an index to significantly speed up `get_fresh_msg_cnt()` #1881
+
+- improve mimetype guessing for PDF and many other formats #1857 #1861
+
+- improve accepting invalid html #1851
+
+- improve tests, cleanup and ci #1850 #1856 #1859 #1861 #1884 #1894 #1895
+
+- tweak HELO command #1908
+
+- make `dc_accounts_get_all()` return accounts sorted #1909
+
+- fix KML coordinates precision used for location streaming #1872
+
+- fix cancelling import/export #1855
+
+
 ## 1.45.0
 
 - add `dc_accounts_t` account manager object and related api functions #1784

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.45.0"
+version = "1.46.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,7 +12,7 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.6.0", default-features = false } 
+pgp = { version = "0.7.0", default-features = false } 
 hex = "0.4.0"
 sha2 = "0.9.0"
 rand = "0.7.0"
@@ -20,12 +20,12 @@ smallvec = "1.0.0"
 surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = "0.3"
+async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.4.0"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.1", features = ["unstable"] }
+async-std = { version = "1.6.4", features = ["unstable"] }
 base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -36,11 +36,11 @@ indexmap = "1.3.0"
 kamadak-exif = "0.5"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.23", features = ["bundled"] }
-r2d2_sqlite = "0.16.0"
+rusqlite = { version = "0.24", features = ["bundled"] }
+r2d2_sqlite = "0.17.0"
 r2d2 = "0.8.5"
-strum = "0.18.0"
-strum_macros = "0.18.0"
+strum = "0.19.0"
+strum_macros = "0.19.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.9.0"
@@ -49,7 +49,7 @@ escaper = "0.1.0"
 bitflags = "1.1.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.1"
+mailparse = "0.13.0"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
@@ -75,8 +75,9 @@ tempfile = "3.0"
 pretty_assertions = "0.6.1"
 pretty_env_logger = "0.4.0"
 proptest = "0.10"
-async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
-smol = "0.1.10"
+async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
+futures-lite = "1.7.0"
+criterion = "0.3"
 
 [workspace]
 members = [
@@ -95,6 +96,10 @@ path = "examples/repl/main.rs"
 required-features = ["repl"]
 
 
+[[bench]]
+name = "create_account"
+harness = false
+
 [features]
 default = []
 internals = []

benches/create_account.rs

@@ -0,0 +1,26 @@
+use async_std::path::PathBuf;
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::accounts::Accounts;
+use tempfile::tempdir;
+
+async fn create_accounts(n: u32) {
+    let dir = tempdir().unwrap();
+    let p: PathBuf = dir.path().join("accounts").into();
+
+    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+    for expected_id in 2..n {
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, expected_id);
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("create 1 account", |b| {
+        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

ci_scripts/remote_python_packaging.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"

ci_scripts/remote_tests_python.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
@@ -30,9 +28,6 @@ ssh $SSHTARGET <<_HERE
 
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL

ci_scripts/remote_tests_rust.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 set -e
@@ -24,9 +22,6 @@ ssh $SSHTARGET <<_HERE
     shopt -s huponexit
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=x86_64-unknown-linux-gnu
     export RUSTC_WRAPPER=sccache
 

ci_scripts/run-python-test.sh

@@ -4,6 +4,7 @@
 # and tox/pytest. 
 
 set -e -x
+shopt -s huponexit
 
 # for core-building and python install step
 export DCC_RS_TARGET=debug

ci_scripts/run-rust-test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -ex
+shopt -s huponexit
 
 #export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.45.0"
+version = "1.46.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/Doxyfile

@@ -236,12 +236,6 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
-# This tag can be used to specify a number of word-keyword mappings (TCL only).
-# A mapping has the form "name=value". For example adding "class=itcl::class"
-# will allow you to use the command class in the itcl::class meaning.
-
-TCL_SUBST              =
-
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all

deltachat-ffi/deltachat.h

@@ -218,7 +218,6 @@ dc_context_t*   dc_context_new               (const char* os_name, const char* d
  * @param context The context object as created by dc_context_new(),
  *     dc_accounts_get_account() or dc_accounts_get_selected_account().
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_context_unref             (dc_context_t* context);
 
@@ -432,7 +431,7 @@ int             dc_set_config_from_qr   (dc_context_t* context, const char* qr);
  * @param context The context object.
  * @return String which must be released using dc_str_unref() after usage.  Never returns NULL.
  */
-char*           dc_get_info                  (dc_context_t* context);
+char*           dc_get_info                  (const dc_context_t* context);
 
 
 /**
@@ -506,7 +505,6 @@ char*           dc_get_oauth2_url            (dc_context_t* context, const char*
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @return None.
  *
  * There is no need to call dc_configure() on every program start,
  * the configuration result is saved in the database
@@ -546,7 +544,6 @@ int             dc_is_configured   (const dc_context_t* context);
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
- * @return None
  */
 void            dc_start_io     (dc_context_t* context);
 
@@ -570,7 +567,6 @@ int             dc_is_io_running(const dc_context_t* context);
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
- * @return None
  */
 void            dc_stop_io(dc_context_t* context);
 
@@ -595,7 +591,6 @@ void            dc_stop_io(dc_context_t* context);
  *
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
- * @return None.
  */
 void            dc_maybe_network             (dc_context_t* context);
 
@@ -936,7 +931,6 @@ uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
  *     NULL deletes the existing draft, if any, without sending it.
  *     Currently, also non-text-messages
  *     will delete the existing drafts.
- * @return None.
  */
 void            dc_set_draft                 (dc_context_t* context, uint32_t chat_id, dc_msg_t* msg);
 
@@ -1004,7 +998,6 @@ uint32_t        dc_add_device_msg            (dc_context_t* context, const char*
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @return None.
  */
 void            dc_update_device_chats       (dc_context_t* context);
 
@@ -1090,7 +1083,8 @@ int             dc_get_fresh_msg_cnt         (dc_context_t* context, uint32_t ch
 /**
  * Estimate the number of messages that will be deleted
  * by the dc_set_config()-options `delete_device_after` or `delete_server_after`.
- * This is typically used to show the estimated impact to the user before actually enabling ephemeral messages.
+ * This is typically used to show the estimated impact to the user
+ * before actually enabling deletion of old messages.
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
@@ -1121,27 +1115,16 @@ dc_array_t*     dc_get_fresh_msgs            (dc_context_t* context);
  * but are still waiting for being marked as "seen" using dc_markseen_msgs()
  * (IMAP/MDNs is not done for noticed messages).
  *
- * Calling this function usually results in the event #DC_EVENT_MSGS_CHANGED.
- * See also dc_marknoticed_all_chats(), dc_marknoticed_contact() and dc_markseen_msgs().
+ * Calling this function usually results in the event #DC_EVENT_MSGS_NOTICED.
+ * See also dc_marknoticed_contact() and dc_markseen_msgs().
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID of which all messages should be marked as being noticed.
- * @return None.
  */
 void            dc_marknoticed_chat          (dc_context_t* context, uint32_t chat_id);
 
 
-/**
- * Same as dc_marknoticed_chat() but for _all_ chats.
- *
- * @memberof dc_context_t
- * @param context The context object as returned from dc_context_new().
- * @return None.
- */
-void            dc_marknoticed_all_chats     (dc_context_t* context);
-
-
 /**
  * Returns all message IDs of the given types in a chat.
  * Typically used to show a gallery.
@@ -1197,7 +1180,6 @@ uint32_t        dc_get_next_media            (dc_context_t* context, uint32_t ms
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The ID of the chat to change the visibility for.
  * @param visibility one of @ref DC_CHAT_VISIBILITY
- * @return None.
  */
 void            dc_set_chat_visibility       (dc_context_t* context, uint32_t chat_id, int visibility);
 
@@ -1226,7 +1208,6 @@ void            dc_set_chat_visibility       (dc_context_t* context, uint32_t ch
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The ID of the chat to delete.
- * @return None.
  */
 void            dc_delete_chat               (dc_context_t* context, uint32_t chat_id);
 
@@ -1252,6 +1233,8 @@ dc_array_t*     dc_get_chat_contacts         (dc_context_t* context, uint32_t ch
 
 /**
  * Get the chat's ephemeral message timer.
+ * The ephemeral message timer is set by dc_set_chat_ephemeral_timer()
+ * on this or any other device participating in the chat.
  *
  * @memberof dc_context_t
  * @param context The context object.
@@ -1395,8 +1378,11 @@ int             dc_set_chat_name             (dc_context_t* context, uint32_t ch
 /**
  * Set the chat's ephemeral message timer.
  *
- * This timer is applied to all messages in a chat and starts when the
- * message is read. The setting is synchronized to all clients
+ * This timer is applied to all messages in a chat and starts when the message is read.
+ * For outgoing messages, the timer starts once the message is sent,
+ * for incoming messages, the timer starts once dc_markseen_msgs() is called.
+ *
+ * The setting is synchronized to all clients
  * participating in a chat.
  *
  * @memberof dc_context_t
@@ -1488,21 +1474,9 @@ char*           dc_get_mime_headers          (dc_context_t* context, uint32_t ms
  * @param context The context object
  * @param msg_ids an array of uint32_t containing all message IDs that should be deleted
  * @param msg_cnt The number of messages IDs in the msg_ids array
- * @return None.
  */
 void            dc_delete_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
-/*
- * Empty IMAP server folder: delete all messages.
- * Deprecated, use dc_set_config() with the key "delete_server_after" instead.
- *
- * @memberof dc_context_t
- * @param context The context object.
- * @param flags What to delete, a combination of the @ref DC_EMPTY flags
- * @return None.
- */
-void            dc_empty_server              (dc_context_t* context, uint32_t flags);
-
 
 /**
  * Forward messages to another chat.
@@ -1512,22 +1486,22 @@ void            dc_empty_server              (dc_context_t* context, uint32_t fl
  * @param msg_ids An array of uint32_t containing all message IDs that should be forwarded
  * @param msg_cnt The number of messages IDs in the msg_ids array
  * @param chat_id The destination chat ID.
- * @return None.
  */
 void            dc_forward_msgs              (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt, uint32_t chat_id);
 
 
 /**
- * Mark all messages sent by the given contact
- * as _noticed_.  See also dc_marknoticed_chat() and
- * dc_markseen_msgs()
+ * Mark all messages sent by the given contact as _noticed_.
+ * This function is typically used to ignore a user in the deaddrop temporarily ("Not now" button).
  *
- * Calling this function usually results in the event #DC_EVENT_MSGS_CHANGED.
+ * The contact is expected to belong to the deaddrop;
+ * only one #DC_EVENT_MSGS_NOTICED with chat_id=DC_CHAT_ID_DEADDROP may be emitted.
+ *
+ * See also dc_marknoticed_chat() and dc_markseen_msgs()
  *
  * @memberof dc_context_t
  * @param context The context object.
  * @param contact_id The contact ID of which all messages should be marked as noticed.
- * @return None.
  */
 void            dc_marknoticed_contact       (dc_context_t* context, uint32_t contact_id);
 
@@ -1538,30 +1512,19 @@ void            dc_marknoticed_contact       (dc_context_t* context, uint32_t co
  * message is only marked as NOTICED and no IMAP/MDNs is done.  See also
  * dc_marknoticed_chat() and dc_marknoticed_contact()
  *
+ * Moreover, if messages belong to a chat with ephemeral messages enabled,
+ * the ephemeral timer is started for these messages.
+ *
+ * One #DC_EVENT_MSGS_NOTICED event is emitted per modified chat.
+ *
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_ids An array of uint32_t containing all the messages IDs that should be marked as seen.
  * @param msg_cnt The number of message IDs in msg_ids.
- * @return None.
  */
 void            dc_markseen_msgs             (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
 
-/**
- * Star/unstar messages by setting the last parameter to 0 (unstar) or 1 (star).
- * Starred messages are collected in a virtual chat that can be shown using
- * dc_get_chat_msgs() using the chat_id DC_CHAT_ID_STARRED.
- *
- * @memberof dc_context_t
- * @param context The context object.
- * @param msg_ids An array of uint32_t message IDs defining the messages to star or unstar
- * @param msg_cnt The number of IDs in msg_ids
- * @param star 0=unstar the messages in msg_ids, 1=star them
- * @return None.
- */
-void            dc_star_msgs                 (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt, int star);
-
-
 /**
  * Get a single message object of the type dc_msg_t.
  * For a list of messages in a chat, see dc_get_chat_msgs()
@@ -1712,7 +1675,6 @@ dc_array_t*     dc_get_blocked_contacts      (dc_context_t* context);
  * @param context The context object.
  * @param contact_id The ID of the contact to block or unblock.
  * @param block 1=block contact, 0=unblock contact
- * @return None.
  */
 void            dc_block_contact             (dc_context_t* context, uint32_t contact_id, int block);
 
@@ -1808,7 +1770,6 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
  * @param param1 Meaning depends on the DC_IMEX_* constants. If this parameter is a directory, it should not end with
  *     a slash (otherwise you'll get double slashes when receiving #DC_EVENT_IMEX_FILE_WRITTEN). Set to NULL if not used.
  * @param param2 Meaning depends on the DC_IMEX_* constants. Set to NULL if not used.
- * @return None.
  */
 void            dc_imex                      (dc_context_t* context, int what, const char* param1, const char* param2);
 
@@ -1951,7 +1912,6 @@ int             dc_continue_key_transfer     (dc_context_t* context, uint32_t ms
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @return None.
  */
 void            dc_stop_ongoing_process      (dc_context_t* context);
 
@@ -2080,7 +2040,6 @@ uint32_t        dc_join_securejoin           (dc_context_t* context, const char*
  * @param chat_id Chat id to enable location streaming for.
  * @param seconds >0: enable location streaming for the given number of seconds;
  *     0: disable location streaming.
- * @return None.
  */
 void        dc_send_locations_to_chat       (dc_context_t* context, uint32_t chat_id, int seconds);
 
@@ -2203,7 +2162,6 @@ dc_array_t* dc_get_locations                (dc_context_t* context, uint32_t cha
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @return None.
  */
 void        dc_delete_all_locations         (dc_context_t* context);
 
@@ -2214,11 +2172,11 @@ void        dc_delete_all_locations         (dc_context_t* context);
  *   MUST NOT be released by the standard free() function;
  *   always use dc_str_unref() for this purpose.
  * - dc_str_unref() MUST NOT be called for strings not returned by deltachat-core.
- * - dc_str_unref() MUST NOT be called for other objectes returned by deltachat-core.
+ * - dc_str_unref() MUST NOT be called for other objects returned by deltachat-core.
  *
  * @memberof dc_context_t
  * @param str The string to release.
- * @return None.
+ *     If NULL is given, nothing is done.
  */
 void dc_str_unref (char* str);
 
@@ -2371,7 +2329,7 @@ dc_context_t*  dc_accounts_get_account          (dc_accounts_t* accounts, uint32
 
 /**
  * Get the currently selected account.
- * If there is at least once account in the account-manager,
+ * If there is at least one account in the account-manager,
  * there is always a selected one.
  * To change the selected account, use dc_accounts_select_account();
  * also adding/importing/migrating accounts may change the selection.
@@ -2405,7 +2363,6 @@ int            dc_accounts_select_account       (dc_accounts_t* accounts, uint32
  *
  * @memberof dc_accounts_t
  * @param accounts Account manager as created by dc_accounts_new().
- * @return None.
  */
 void           dc_accounts_start_io             (dc_accounts_t* accounts);
 
@@ -2473,7 +2430,6 @@ dc_accounts_event_emitter_t* dc_accounts_get_event_emitter (dc_accounts_t* accou
  * @param array The array object to free,
  *     created eg. by dc_get_chatlist(), dc_get_contacts() and so on.
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void             dc_array_unref              (dc_array_t* array);
 
@@ -2672,7 +2628,6 @@ int              dc_array_search_id          (const dc_array_t* array, uint32_t
  * @memberof dc_chatlist_t
  * @param chatlist The chatlist object to free, created eg. by dc_get_chatlist(), dc_search_msgs().
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void             dc_chatlist_unref           (dc_chatlist_t* chatlist);
 
@@ -2759,7 +2714,7 @@ dc_lot_t*        dc_chatlist_get_summary     (const dc_chatlist_t* chatlist, siz
  * @memberof dc_context_t
  * @param context The context object.
  * @param chat_id Chat to get a summary for.
- * @param msg_id Messasge to get a summary for.
+ * @param msg_id Message to get a summary for.
  * @return The summary as an dc_lot_t object, see dc_chatlist_get_summary() for details.
  *     Must be freed using dc_lot_unref().  NULL is never returned.
  */
@@ -2806,8 +2761,6 @@ char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat
 
 #define         DC_CHAT_ID_DEADDROP          1 // virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 #define         DC_CHAT_ID_TRASH             3 // messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
-#define         DC_CHAT_ID_MSGS_IN_CREATION  4 // a message is just in creation but not yet assigned to a chat (eg. we may need the message ID to set up blobs; this avoids unready message to be sent and shown)
-#define         DC_CHAT_ID_STARRED           5 // virtual chat showing all messages flagged with msgs.starred=2
 #define         DC_CHAT_ID_ARCHIVED_LINK     6 // only an indicator in a chatlist
 #define         DC_CHAT_ID_ALLDONE_HINT      7 // only an indicator in a chatlist
 #define         DC_CHAT_ID_LAST_SPECIAL      9 // larger chat IDs are "real" chats, their messages are "real" messages.
@@ -2825,7 +2778,6 @@ char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat
  * @memberof dc_chat_t
  * @param chat Chat object are returned eg. by dc_get_chat().
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_chat_unref                (dc_chat_t* chat);
 
@@ -2835,7 +2787,6 @@ void            dc_chat_unref                (dc_chat_t* chat);
  *
  * Special IDs:
  * - DC_CHAT_ID_DEADDROP         (1) - Virtual chat containing messages which senders are not confirmed by the user.
- * - DC_CHAT_ID_STARRED          (5) - Virtual chat containing all starred messages-
  * - DC_CHAT_ID_ARCHIVED_LINK    (6) - A link at the end of the chatlist, if present the UI should show the button "Archived chats"-
  *
  * "Normal" chat IDs are larger than these special IDs (larger than DC_CHAT_ID_LAST_SPECIAL).
@@ -3086,7 +3037,6 @@ dc_msg_t*       dc_msg_new                    (dc_context_t* context, int viewty
  * @memberof dc_msg_t
  * @param msg The message object to free.
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_msg_unref                  (dc_msg_t* msg);
 
@@ -3365,6 +3315,7 @@ int             dc_msg_get_showpadlock        (const dc_msg_t* msg);
 
 /**
  * Get ephemeral timer duration for message.
+ * This is the value of dc_get_chat_ephemeral_timer() in the moment the message was sent.
  *
  * To check if the timer is started and calculate remaining time,
  * use dc_msg_get_ephemeral_timestamp().
@@ -3384,7 +3335,8 @@ uint32_t        dc_msg_get_ephemeral_timer    (const dc_msg_t* msg);
  *
  * @memberof dc_msg_t
  * @param msg The message object.
- * @return Time of message removal, 0 if the timer is not started.
+ * @return Time of message removal, 0 if the timer is not yet started
+ *     (the timer starts on sending messages or when dc_markseen_msgs() is called)
  */
 int64_t          dc_msg_get_ephemeral_timestamp (const dc_msg_t* msg);
 
@@ -3470,21 +3422,6 @@ int             dc_msg_has_location           (const dc_msg_t* msg);
 int             dc_msg_is_sent                (const dc_msg_t* msg);
 
 
-/**
- * Check if a message is starred.  Starred messages are "favorites" marked by the user
- * with a "star" or something like that.  Starred messages can typically be shown
- * easily and are not deleted automatically.
- *
- * To star one or more messages, use dc_star_msgs(), to get a list of starred messages,
- * use dc_get_chat_msgs() using DC_CHAT_ID_STARRED as the chat_id.
- *
- * @memberof dc_msg_t
- * @param msg The message object.
- * @return 1=message is starred, 0=message not starred.
- */
-int             dc_msg_is_starred             (const dc_msg_t* msg);
-
-
 /**
  * Check if the message is a forwarded message.
  *
@@ -3587,6 +3524,27 @@ char*           dc_msg_get_setupcodebegin     (const dc_msg_t* msg);
  */
 char* dc_msg_get_videochat_url (const dc_msg_t* msg);
 
+/**
+ * Gets the error status of the message.
+ * If there is no error associated with the message, NULL is returned.
+ *
+ * A message can have an associated error status if something went wrong when sending or
+ * receiving message itself.  The error status is free-form text and should not be further parsed,
+ * rather it's presence is meant to indicate *something* went wrong with the message and the
+ * text of the error is detailed information on what.
+ * 
+ * Some common reasons error can be associated with messages are:
+ * * Lack of valid signature on an e2ee message, usually for received messages.
+ * * Failure to decrypt an e2ee message, usually for received messages.
+ * * When a message could not be delivered to one or more recipients the non-delivery
+ *   notification text can be stored in the error status.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return Error or NULL. The result must be released using dc_str_unref().
+ */
+char*           dc_msg_get_error               (const dc_msg_t* msg);
+
 
 /**
  * Get type of videochat.
@@ -3630,7 +3588,6 @@ int dc_msg_get_videochat_type (const dc_msg_t* msg);
  * @memberof dc_msg_t
  * @param msg The message object.
  * @param text Message text.
- * @return None.
  */
 void            dc_msg_set_text               (dc_msg_t* msg, const char* text);
 
@@ -3646,7 +3603,6 @@ void            dc_msg_set_text               (dc_msg_t* msg, const char* text);
  * @param file If the message object is used in dc_send_msg() later,
  *     this must be the full path of the image file to send.
  * @param filemime Mime type of the file. NULL if you don't know or don't care.
- * @return None.
  */
 void            dc_msg_set_file               (dc_msg_t* msg, const char* file, const char* filemime);
 
@@ -3660,7 +3616,6 @@ void            dc_msg_set_file               (dc_msg_t* msg, const char* file,
  * @param msg The message object.
  * @param width Width in pixels, if known. 0 if you don't know or don't care.
  * @param height Height in pixels, if known. 0 if you don't know or don't care.
- * @return None.
  */
 void            dc_msg_set_dimension          (dc_msg_t* msg, int width, int height);
 
@@ -3673,7 +3628,6 @@ void            dc_msg_set_dimension          (dc_msg_t* msg, int width, int hei
  * @memberof dc_msg_t
  * @param msg The message object.
  * @param duration Length in milliseconds. 0 if you don't know or don't care.
- * @return None.
  */
 void            dc_msg_set_duration           (dc_msg_t* msg, int duration);
 
@@ -3693,7 +3647,6 @@ void            dc_msg_set_duration           (dc_msg_t* msg, int duration);
  * @param msg The message object.
  * @param latitude North-south position of the location.
  * @param longitude East-west position of the location.
- * @return None.
  */
 void            dc_msg_set_location           (dc_msg_t* msg, double latitude, double longitude);
 
@@ -3718,11 +3671,67 @@ void            dc_msg_set_location           (dc_msg_t* msg, double latitude, d
  * @param width The new width to store in the message object. 0 if you do not want to change width and height.
  * @param height The new height to store in the message object. 0 if you do not want to change width and height.
  * @param duration The new duration to store in the message object. 0 if you do not want to change it.
- * @return None.
  */
 void            dc_msg_latefiling_mediasize   (dc_msg_t* msg, int width, int height, int duration);
 
 
+/**
+ * Set the message replying to.
+ * This allows optionally to reply to an explicit message
+ * instead of replying implicitly to the end of the chat.
+ *
+ * dc_msg_set_quote() copies some basic data from the quoted message object
+ * so that dc_msg_get_quoted_text() will always work.
+ * dc_msg_get_quoted_msg() gets back the quoted message only if it is _not_ deleted.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object to set the reply to.
+ * @param quote The quote to set for msg.
+ */
+void             dc_msg_set_quote             (dc_msg_t* msg, const dc_msg_t* quote);
+
+
+/**
+ * Get quoted text, if any.
+ * You can use this function also check if there is a quote for a message.
+ *
+ * The text is a summary of the original text,
+ * similar to what is shown in the chatlist.
+ *
+ * If available, you can get the whole quoted message object using dc_msg_get_quoted_msg().
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return The quoted text or NULL if there is no quote.
+ *     Returned strings must be released using dc_str_unref().
+ */
+char*           dc_msg_get_quoted_text        (const dc_msg_t* msg);
+
+
+/**
+ * Get quoted message, if available.
+ * UIs might use this information to offer "jumping back" to the quoted message
+ * or to enrich displaying the quote.
+ *
+ * If this function returns NULL,
+ * this does not mean there is no quote for the message -
+ * it might also mean that a quote exist but the quoted message is deleted meanwhile.
+ * Therefore, do not use this function to check if there is a quote for a message.
+ * To check if a message has a quote, use dc_msg_get_quoted_text().
+ *
+ * To display the quote in the chat, use dc_msg_get_quoted_text() as a primary source,
+ * however, one might add information from the message object (eg. an image).
+ *
+ * It is not guaranteed that the message belong to the same chat.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return The quoted message or NULL.
+ *     Must be freed using dc_msg_unref() after usage.
+ */
+dc_msg_t*       dc_msg_get_quoted_msg         (const dc_msg_t* msg);
+
+
 /**
  * @class dc_contact_t
  *
@@ -3756,7 +3765,6 @@ void            dc_msg_latefiling_mediasize   (dc_msg_t* msg, int width, int hei
  * @memberof dc_contact_t
  * @param contact The contact object as created eg. by dc_get_contact().
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_contact_unref             (dc_contact_t* contact);
 
@@ -3996,7 +4004,6 @@ void            dc_provider_unref                     (dc_provider_t* provider);
  * @memberof dc_lot_t
  * @param lot The object to free.
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_lot_unref             (dc_lot_t* lot);
 
@@ -4281,10 +4288,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 
-#define DC_EMPTY_MVBOX 0x01 // Deprecated, flag for dc_empty_server(): Clear all mvbox messages
-#define DC_EMPTY_INBOX 0x02 // Deprecated, flag for dc_empty_server(): Clear all INBOX messages
-
-
 /**
  * @class dc_event_emitter_t
  *
@@ -4314,7 +4317,6 @@ dc_event_t* dc_get_next_event(dc_event_emitter_t* emitter);
  * @memberof dc_event_emitter_t
  * @param emitter Event emitter object as returned from dc_get_event_emitter().
  *     If NULL is given, nothing is done and an error is logged.
- * @return None.
  */
 void  dc_event_emitter_unref(dc_event_emitter_t* emitter);
 
@@ -4348,7 +4350,6 @@ dc_event_t* dc_accounts_get_next_event (dc_accounts_event_emitter_t* emitter);
  * @memberof dc_accounts_event_emitter_t
  * @param emitter Event emitter object as returned from dc_accounts_get_event_emitter().
  *     If NULL is given, nothing is done and an error is logged.
- * @return None.
  */
 void dc_accounts_event_emitter_unref(dc_accounts_event_emitter_t* emitter);
 
@@ -4410,7 +4411,7 @@ int dc_event_get_data2_int(dc_event_t* event);
  *
  * @memberof dc_event_t
  * @param event Event object as returned from dc_get_next_event().
- * @return "data2" as a string,
+ * @return "data2" as a string or NULL.
  *     the meaning depends on the event type associated with this event.
  *     Once you're done with the string, you have to unref it using dc_unref_str().
  */
@@ -4435,7 +4436,6 @@ uint32_t dc_event_get_account_id(dc_event_t* event);
  *
  * @memberof dc_event_t
  * @param event Event object as returned from dc_get_next_event().
- * @return None.
  */
 void dc_event_unref(dc_event_t* event);
 
@@ -4506,14 +4506,6 @@ void dc_event_unref(dc_event_t* event);
  */
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
-/**
- * Emitted when an IMAP folder was emptied.
- *
- * @param data1 0
- * @param data2 (char*) Folder name.
- */
-#define DC_EVENT_IMAP_FOLDER_EMPTIED  106
-
 /**
  * Emitted when a new blob file was successfully written
  *
@@ -4619,6 +4611,21 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_INCOMING_MSG             2005
 
 
+/**
+ * Messages were marked noticed or seen.
+ * The ui may update badge counters or stop showing a chatlist-item with a bold font.
+ *
+ * This event is emitted eg. when calling dc_markseen_msgs(), dc_marknoticed_chat() or dc_marknoticed_contact()
+ * or when a chat is answered on another device.
+ * Do not try to derive the state of an item from just the fact you received the event;
+ * use eg. dc_msg_get_state() or dc_get_fresh_msg_cnt() for this purpose.
+ *
+ * @param data1 (int) chat_id
+ * @param data2 0
+ */
+#define DC_EVENT_MSGS_NOTICED             2008
+
+
 /**
  * A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
  * DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
@@ -4670,7 +4677,7 @@ void dc_event_unref(dc_event_t* event);
 /**
  * Contact(s) created, renamed, verified, blocked or deleted.
  *
- * @param data1 (int) If not 0, this is the contact_id of an added contact that should be selected.
+ * @param data1 (int) contact_id of the changed contact or 0 on batch-changes or deletion.
  * @param data2 0
  */
 #define DC_EVENT_CONTACTS_CHANGED         2030
@@ -4692,7 +4699,7 @@ void dc_event_unref(dc_event_t* event);
  * Inform about the configuration progress started by dc_configure().
  *
  * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done
- * @param data2 0
+ * @param data2 (char*) progress comment, error message or NULL if not applicable
  */
 #define DC_EVENT_CONFIGURE_PROGRESS       2041
 
@@ -4755,18 +4762,8 @@ void dc_event_unref(dc_event_t* event);
  */
 
 
-#define DC_EVENT_FILE_COPIED         2055 // not used anymore
-#define DC_EVENT_IS_OFFLINE          2081 // not used anymore
-#define DC_EVENT_GET_STRING          2091 // not used anymore, use dc_set_stock_translation()
-#define DC_ERROR_SEE_STRING          0    // not used anymore
-#define DC_ERROR_SELF_NOT_IN_GROUP   1    // not used anymore
-#define DC_STR_SELFNOTINGRP          21   // not used anymore
 #define DC_EVENT_DATA1_IS_STRING(e)  0    // not used anymore 
-#define DC_EVENT_DATA2_IS_STRING(e)  ((e)==DC_EVENT_IMEX_FILE_WRITTEN || ((e)>=100 && (e)<=499))
-#define DC_EVENT_RETURNS_INT(e)      ((e)==DC_EVENT_IS_OFFLINE) // not used anymore
-#define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING) // not used anymore
-#define dc_archive_chat(a,b,c)  dc_set_chat_visibility((a), (b), (c)? 1 : 0) // not used anymore
-#define dc_chat_get_archived(a) (dc_chat_get_visibility((a))==1? 1 : 0)      // not used anymore
+#define DC_EVENT_DATA2_IS_STRING(e)  ((e)==DC_EVENT_CONFIGURE_PROGRESS || (e)==DC_EVENT_IMEX_FILE_WRITTEN || ((e)>=100 && (e)<=499))
 
 
 /*
@@ -4930,7 +4927,6 @@ void dc_event_unref(dc_event_t* event);
 #define DC_STR_CONTACT_NOT_VERIFIED       36
 #define DC_STR_CONTACT_SETUP_CHANGED      37
 #define DC_STR_ARCHIVEDCHATS              40
-#define DC_STR_STARREDMSGS                41
 #define DC_STR_AC_SETUP_MSG_SUBJECT       42
 #define DC_STR_AC_SETUP_MSG_BODY          43
 #define DC_STR_CANNOT_LOGIN               60
@@ -4957,8 +4953,12 @@ void dc_event_unref(dc_event_t* event);
 #define DC_STR_EPHEMERAL_FOUR_WEEKS       81
 #define DC_STR_VIDEOCHAT_INVITATION       82
 #define DC_STR_VIDEOCHAT_INVITE_MSG_BODY  83
+#define DC_STR_CONFIGURATION_FAILED       84
+#define DC_STR_BAD_TIME_MSG_BODY          85
+#define DC_STR_UPDATE_REMINDER_MSG_BODY   86
+#define DC_STR_ERROR_NO_NETWORK           87
 
-#define DC_STR_COUNT                      83
+#define DC_STR_COUNT                      87
 
 /*
  * @}

deltachat-ffi/src/lib.rs

@@ -218,7 +218,7 @@ pub unsafe extern "C" fn dc_set_config_from_qr(
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c_char {
+pub unsafe extern "C" fn dc_get_info(context: *const dc_context_t) -> *mut libc::c_char {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_get_info()");
         return "".strdup();
@@ -316,7 +316,6 @@ pub unsafe extern "C" fn dc_get_id(context: *mut dc_context_t) -> libc::c_int {
     ctx.get_id() as libc::c_int
 }
 
-#[no_mangle]
 pub type dc_event_t = Event;
 
 #[no_mangle]
@@ -355,7 +354,6 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::SmtpMessageSent(_)
         | EventType::ImapMessageDeleted(_)
         | EventType::ImapMessageMoved(_)
-        | EventType::ImapFolderEmptied(_)
         | EventType::NewBlobFile(_)
         | EventType::DeletedBlobFile(_)
         | EventType::Warning(_)
@@ -364,6 +362,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ErrorSelfNotInGroup(_) => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
+        | EventType::MsgsNoticed(chat_id)
         | EventType::MsgDelivered { chat_id, .. }
         | EventType::MsgFailed { chat_id, .. }
         | EventType::MsgRead { chat_id, .. }
@@ -373,7 +372,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
             let id = id.unwrap_or_default();
             id as libc::c_int
         }
-        EventType::ConfigureProgress(progress) | EventType::ImexProgress(progress) => {
+        EventType::ConfigureProgress { progress, .. } | EventType::ImexProgress(progress) => {
             *progress as libc::c_int
         }
         EventType::ImexFileWritten(_) => 0,
@@ -398,7 +397,6 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::SmtpMessageSent(_)
         | EventType::ImapMessageDeleted(_)
         | EventType::ImapMessageMoved(_)
-        | EventType::ImapFolderEmptied(_)
         | EventType::NewBlobFile(_)
         | EventType::DeletedBlobFile(_)
         | EventType::Warning(_)
@@ -407,9 +405,10 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ErrorSelfNotInGroup(_)
         | EventType::ContactsChanged(_)
         | EventType::LocationChanged(_)
-        | EventType::ConfigureProgress(_)
+        | EventType::ConfigureProgress { .. }
         | EventType::ImexProgress(_)
         | EventType::ImexFileWritten(_)
+        | EventType::MsgsNoticed(_)
         | EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
@@ -438,7 +437,6 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SmtpMessageSent(msg)
         | EventType::ImapMessageDeleted(msg)
         | EventType::ImapMessageMoved(msg)
-        | EventType::ImapFolderEmptied(msg)
         | EventType::NewBlobFile(msg)
         | EventType::DeletedBlobFile(msg)
         | EventType::Warning(msg)
@@ -450,17 +448,24 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         }
         EventType::MsgsChanged { .. }
         | EventType::IncomingMsg { .. }
+        | EventType::MsgsNoticed(_)
         | EventType::MsgDelivered { .. }
         | EventType::MsgFailed { .. }
         | EventType::MsgRead { .. }
         | EventType::ChatModified(_)
         | EventType::ContactsChanged(_)
         | EventType::LocationChanged(_)
-        | EventType::ConfigureProgress(_)
         | EventType::ImexProgress(_)
         | EventType::SecurejoinInviterProgress { .. }
         | EventType::SecurejoinJoinerProgress { .. }
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
+        EventType::ConfigureProgress { comment, .. } => {
+            if let Some(comment) = comment {
+                comment.to_c_string().unwrap_or_default().into_raw()
+            } else {
+                ptr::null_mut()
+            }
+        }
         EventType::ImexFileWritten(file) => {
             let data2 = file.to_c_string().unwrap_or_default();
             data2.into_raw()
@@ -478,7 +483,6 @@ pub unsafe extern "C" fn dc_event_get_account_id(event: *mut dc_event_t) -> u32
     (*event).id
 }
 
-#[no_mangle]
 pub type dc_event_emitter_t = EventEmitter;
 
 #[no_mangle]
@@ -496,7 +500,7 @@ pub unsafe extern "C" fn dc_get_event_emitter(
 #[no_mangle]
 pub unsafe extern "C" fn dc_event_emitter_unref(emitter: *mut dc_event_emitter_t) {
     if emitter.is_null() {
-        eprintln!("ignoring careless call to dc_event_mitter_unref()");
+        eprintln!("ignoring careless call to dc_event_emitter_unref()");
         return;
     }
 
@@ -506,6 +510,7 @@ pub unsafe extern "C" fn dc_event_emitter_unref(emitter: *mut dc_event_emitter_t
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *mut dc_event_t {
     if events.is_null() {
+        eprintln!("ignoring careless call to dc_get_next_event()");
         return ptr::null_mut();
     }
     let events = &*events;
@@ -969,22 +974,6 @@ pub unsafe extern "C" fn dc_marknoticed_chat(context: *mut dc_context_t, chat_id
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_marknoticed_all_chats(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_marknoticed_all_chats()");
-        return;
-    }
-    let ctx = &*context;
-
-    block_on(async move {
-        chat::marknoticed_all_chats(&ctx)
-            .await
-            .log_err(ctx, "Failed marknoticed all chats")
-            .unwrap_or(())
-    })
-}
-
 fn from_prim<S, T>(s: S) -> Option<T>
 where
     T: FromPrimitive,
@@ -1423,17 +1412,6 @@ pub unsafe extern "C" fn dc_delete_msgs(
     block_on(message::delete_msgs(&ctx, &msg_ids))
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_empty_server(context: *mut dc_context_t, flags: u32) {
-    if context.is_null() || flags == 0 {
-        eprintln!("ignoring careless call to dc_empty_server()");
-        return;
-    }
-    let ctx = &*context;
-
-    block_on(message::dc_empty_server(&ctx, flags))
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_forward_msgs(
     context: *mut dc_context_t,
@@ -1486,23 +1464,6 @@ pub unsafe extern "C" fn dc_markseen_msgs(
     block_on(message::markseen_msgs(&ctx, msg_ids));
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_star_msgs(
-    context: *mut dc_context_t,
-    msg_ids: *const u32,
-    msg_cnt: libc::c_int,
-    star: libc::c_int,
-) {
-    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
-        eprintln!("ignoring careless call to dc_star_msgs()");
-        return;
-    }
-    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
-    let ctx = &*context;
-
-    block_on(message::star_msgs(&ctx, msg_ids, star == 1));
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) -> *mut dc_msg_t {
     if context.is_null() {
@@ -1827,7 +1788,7 @@ pub unsafe extern "C" fn dc_continue_key_transfer(
         {
             Ok(()) => 1,
             Err(err) => {
-                error!(&ctx, "dc_continue_key_transfer: {}", err);
+                warn!(&ctx, "dc_continue_key_transfer: {}", err);
                 0
             }
         }
@@ -1891,8 +1852,13 @@ pub unsafe extern "C" fn dc_join_securejoin(
     }
     let ctx = &*context;
 
-    block_on(async move { securejoin::dc_join_securejoin(&ctx, &to_string_lossy(qr)).await })
-        .to_u32()
+    block_on(async move {
+        securejoin::dc_join_securejoin(&ctx, &to_string_lossy(qr))
+            .await
+            .map(|chatid| chatid.to_u32())
+            .log_err(ctx, "failed dc_join_securejoin() call")
+            .unwrap_or_default()
+    })
 }
 
 #[no_mangle]
@@ -1992,7 +1958,6 @@ pub unsafe extern "C" fn dc_delete_all_locations(context: *mut dc_context_t) {
 
 // dc_array_t
 
-#[no_mangle]
 pub type dc_array_t = dc_array::dc_array_t;
 
 #[no_mangle]
@@ -2175,7 +2140,6 @@ pub struct ChatlistWrapper {
     list: chatlist::Chatlist,
 }
 
-#[no_mangle]
 pub type dc_chatlist_t = ChatlistWrapper;
 
 #[no_mangle]
@@ -2299,7 +2263,6 @@ pub struct ChatWrapper {
     chat: chat::Chat,
 }
 
-#[no_mangle]
 pub type dc_chat_t = ChatWrapper;
 
 #[no_mangle]
@@ -2525,7 +2488,6 @@ pub struct MessageWrapper {
     message: message::Message,
 }
 
-#[no_mangle]
 pub type dc_msg_t = MessageWrapper;
 
 #[no_mangle]
@@ -2835,16 +2797,6 @@ pub unsafe extern "C" fn dc_msg_is_sent(msg: *mut dc_msg_t) -> libc::c_int {
     ffi_msg.message.is_sent().into()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_is_starred(msg: *mut dc_msg_t) -> libc::c_int {
-    if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_is_starred()");
-        return 0;
-    }
-    let ffi_msg = &*msg;
-    ffi_msg.message.is_starred().into()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_is_forwarded(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -3010,6 +2962,74 @@ pub unsafe extern "C" fn dc_msg_latefiling_mediasize(
     });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_error(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_error()");
+        return ptr::null_mut();
+    }
+    let ffi_msg = &*msg;
+    match ffi_msg.message.error() {
+        Some(error) => error.strdup(),
+        None => ptr::null_mut(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_quote(msg: *mut dc_msg_t, quote: *const dc_msg_t) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_quote()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    let ffi_quote = &*quote;
+
+    block_on(async move {
+        ffi_msg
+            .message
+            .set_quote(&*ffi_msg.context, &ffi_quote.message)
+            .await
+            .log_err(&*ffi_msg.context, "failed to set quote")
+            .ok();
+    });
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_quoted_text(msg: *const dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_quoted_text()");
+        return ptr::null_mut();
+    }
+    let ffi_msg: &MessageWrapper = &*msg;
+    ffi_msg
+        .message
+        .quoted_text()
+        .map_or_else(ptr::null_mut, |s| s.strdup())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_quoted_msg(msg: *const dc_msg_t) -> *mut dc_msg_t {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_get_quoted_msg()");
+        return ptr::null_mut();
+    }
+    let ffi_msg: &MessageWrapper = &*msg;
+    let context = &*ffi_msg.context;
+    let res = block_on(async move {
+        ffi_msg
+            .message
+            .quoted_message(context)
+            .await
+            .log_err(context, "failed to get quoted message")
+            .unwrap_or(None)
+    });
+
+    match res {
+        Some(message) => Box::into_raw(Box::new(MessageWrapper { context, message })),
+        None => ptr::null_mut(),
+    }
+}
+
 // dc_contact_t
 
 /// FFI struct for [dc_contact_t]
@@ -3024,7 +3044,6 @@ pub struct ContactWrapper {
     contact: contact::Contact,
 }
 
-#[no_mangle]
 pub type dc_contact_t = ContactWrapper;
 
 #[no_mangle]
@@ -3157,7 +3176,6 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
 
 // dc_lot_t
 
-#[no_mangle]
 pub type dc_lot_t = lot::Lot;
 
 #[no_mangle]
@@ -3266,7 +3284,8 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
 
     fn log_err(self, ctx: &Context, message: &str) -> Result<T, E> {
         self.map_err(|err| {
-            warn!(ctx, "{}: {}", message, err);
+            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+            warn!(ctx, "{}: {:#}", message, err);
             err
         })
     }
@@ -3298,7 +3317,6 @@ fn convert_and_prune_message_ids(msg_ids: *const u32, msg_cnt: libc::c_int) -> V
 
 // dc_provider_t
 
-#[no_mangle]
 pub type dc_provider_t = provider::Provider;
 
 #[no_mangle]
@@ -3389,7 +3407,8 @@ pub unsafe extern "C" fn dc_accounts_new(
     match accs {
         Ok(accs) => Box::into_raw(Box::new(accs)),
         Err(err) => {
-            eprintln!("failed to create accounts: {}", err);
+            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+            eprintln!("failed to create accounts: {:#}", err);
             ptr::null_mut()
         }
     }
@@ -3450,7 +3469,7 @@ pub unsafe extern "C" fn dc_accounts_select_account(
     let accounts = &*accounts;
     block_on(accounts.select_account(id))
         .map(|_| 1)
-        .unwrap_or_else(|_| 0)
+        .unwrap_or(0)
 }
 
 #[no_mangle]
@@ -3462,7 +3481,7 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -
 
     let accounts = &*accounts;
 
-    block_on(accounts.add_account()).unwrap_or_else(|_| 0)
+    block_on(accounts.add_account()).unwrap_or(0)
 }
 
 #[no_mangle]
@@ -3564,7 +3583,6 @@ pub unsafe extern "C" fn dc_accounts_maybe_network(accounts: *mut dc_accounts_t)
     block_on(accounts.maybe_network());
 }
 
-#[no_mangle]
 pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
 
 #[no_mangle]
@@ -3598,9 +3616,10 @@ pub unsafe extern "C" fn dc_accounts_get_next_event(
     emitter: *mut dc_accounts_event_emitter_t,
 ) -> *mut dc_event_t {
     if emitter.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_next_event()");
         return ptr::null_mut();
     }
-    let emitter = &*emitter;
+    let emitter = &mut *emitter;
 
     emitter
         .recv_sync()

examples/repl/cmdline.rs

@@ -185,7 +185,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -193,7 +193,6 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.is_starred() { "★" } else { "" },
         if msg.get_from_id() == 1 as libc::c_uint {
             ""
         } else if msg.get_state() == MessageState::InSeen {
@@ -387,8 +386,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
-                 star <msg-id>\n\
-                 unstar <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
                  listcontacts [<query>]\n\
@@ -405,7 +402,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
-                 emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
                  ============================================="
@@ -949,12 +945,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             message::markseen_msgs(&context, msg_ids).await;
         }
-        "star" | "unstar" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = vec![MsgId::new(0); 1];
-            msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::star_msgs(&context, msg_ids, arg0 == "star").await;
-        }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
@@ -1092,11 +1082,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 seconds, device_cnt, server_cnt
             );
         }
-        "emptyserver" => {
-            ensure!(!arg1.is_empty(), "Argument <flags> missing");
-
-            message::dc_empty_server(&context, arg1.parse()?).await;
-        }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),
     }

examples/repl/main.rs

@@ -80,11 +80,21 @@ fn receive_event(event: EventType) {
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
-        EventType::ConfigureProgress(progress) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
-            );
+        EventType::ConfigureProgress { progress, comment } => {
+            if let Some(comment) = comment {
+                info!(
+                    "{}",
+                    yellow.paint(format!(
+                        "Received CONFIGURE_PROGRESS({} ‰, {})",
+                        progress, comment
+                    ))
+                );
+            } else {
+                info!(
+                    "{}",
+                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+                );
+            }
         }
         EventType::ImexProgress(progress) => {
             info!(
@@ -410,7 +420,7 @@ async fn handle_cmd(
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await;
+                dc_join_securejoin(&ctx, arg1).await?;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),

examples/simple.rs

@@ -10,7 +10,7 @@ use deltachat::EventType;
 
 fn cb(event: EventType) {
     match event {
-        EventType::ConfigureProgress(progress) => {
+        EventType::ConfigureProgress { progress, .. } => {
             log::info!("progress: {}", progress);
         }
         EventType::Info(msg) => {

python/src/deltachat/account.py

@@ -179,17 +179,6 @@ class Account(object):
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def empty_server_folders(self, inbox=False, mvbox=False):
-        """ empty server folders. """
-        flags = 0
-        if inbox:
-            flags |= const.DC_EMPTY_INBOX
-        if mvbox:
-            flags |= const.DC_EMPTY_MVBOX
-        if not flags:
-            raise ValueError("no flags set")
-        lib.dc_empty_server(self._dc_context, flags)
-
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -557,6 +546,9 @@ class Account(object):
         You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
         account is started.
 
+        If you are using this from a test, you may want to call
+        wait_all_initial_fetches() afterwards.
+
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 

python/src/deltachat/cutil.py

@@ -17,7 +17,8 @@ def iter_array(dc_array_t, constructor):
 
 
 def from_dc_charpointer(obj):
-    return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    if obj != ffi.NULL:
+        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
 
 
 class DCLot:

python/src/deltachat/direct_imap.py

@@ -24,12 +24,13 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    imap = DirectImap(account)
-    if imap.select_config_folder("mvbox"):
+    if not hasattr(account, "direct_imap"):
+        imap = DirectImap(account)
+        if imap.select_config_folder("mvbox"):
+            imap.delete(ALL, expunge=True)
+        assert imap.select_config_folder("inbox")
         imap.delete(ALL, expunge=True)
-    assert imap.select_config_folder("inbox")
-    imap.delete(ALL, expunge=True)
-    setattr(account, "direct_imap", imap)
+        setattr(account, "direct_imap", imap)
 
 
 @deltachat.global_hookimpl

python/src/deltachat/events.py

@@ -1,6 +1,7 @@
 import threading
 import time
 import re
+import os
 from queue import Queue, Empty
 
 import deltachat
@@ -48,6 +49,15 @@ class FFIEventLogger:
         if self.logid:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+
+        if os.name == "posix":
+            WARN = '\033[93m'
+            ERROR = '\033[91m'
+            ENDC = '\033[0m'
+            if message.startswith("DC_EVENT_WARNING"):
+                s = WARN + s + ENDC
+            if message.startswith("DC_EVENT_ERROR"):
+                s = ERROR + s + ENDC
         with self._loglock:
             print(s, flush=True)
 
@@ -111,6 +121,15 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
+    def wait_all_initial_fetches(self):
+        """Has to be called after start_io() to wait for fetch_existing_msgs to run
+        so that new messages are not mistaken for old ones:
+        - ac1 and ac2 are created
+        - ac1 sends a message to ac2
+        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
+        - therefore no DC_EVENT_INCOMING_MSG is sent"""
+        self.get_info_contains("Done fetching existing messages")
+
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")

python/src/deltachat/message.py

@@ -175,6 +175,27 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
+    @property
+    def quoted_text(self):
+        """Text inside the quote
+
+        :returns: Quoted text"""
+        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
+
+    @property
+    def quote(self):
+        """Quote getter
+
+        :returns: Quoted message, if found in the database"""
+        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
+        if msg:
+            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
+
+    @quote.setter
+    def quote(self, quoted_message):
+        """Quote setter"""
+        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 

python/src/deltachat/testplugin.py

@@ -342,10 +342,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
+            if hasattr(account, "direct_imap"):
+                # Attach the existing direct_imap. If we did not do this, a new one would be created and
+                # delete existing messages (see dc_account_extra_configure(configure))
+                ac.direct_imap = account.direct_imap
             ac._configtracker = ac.configure()
             return ac
 
         def wait_configure_and_start_io(self):
+            started_accounts = []
             for acc in self._accounts:
                 if hasattr(acc, "_configtracker"):
                     acc._configtracker.wait_finish()
@@ -353,8 +358,11 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc.set_config("bcc_self", "0")
                 if acc.is_configured() and not acc.is_started():
                     acc.start_io()
+                    started_accounts.append(acc)
                 print("{}: {} account was successfully setup".format(
                     acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in started_accounts:
+                acc._evtracker.wait_all_initial_fetches()
 
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__

python/tests/test_account.py

@@ -166,6 +166,16 @@ class TestOfflineContact:
         with pytest.raises(ValueError):
             ac1.create_chat(ac3)
 
+    def test_contact_rename(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact = ac1.create_contact("some1@example.com", name="some1")
+        chat = ac1.create_chat(contact)
+        assert chat.get_name() == "some1"
+        ac1.create_contact("some1@example.com", name="renamed")
+        ev = ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED")
+        assert ev.data1 == chat.id
+        assert chat.get_name() == "renamed"
+
 
 class TestOfflineChat:
     @pytest.fixture
@@ -466,6 +476,21 @@ class TestOfflineChat:
         assert not res.is_ask_verifygroup()
         assert res.contact_id == 10
 
+    def test_quote(self, chat1):
+        """Offline quoting test"""
+        msg = Message.new_empty(chat1.account, "text")
+        msg.set_text("Multi\nline\nmessage")
+        assert msg.quoted_text is None
+
+        # Prepare message to assign it a Message-Id.
+        # Messages without Message-Id cannot be quoted.
+        msg = chat1.prepare_message(msg)
+
+        reply_msg = Message.new_empty(chat1.account, "text")
+        reply_msg.set_text("reply")
+        reply_msg.quote = msg
+        assert reply_msg.quoted_text == "Multi\nline\nmessage"
+
     def test_group_chat_many_members_add_remove(self, ac1, lp):
         lp.sec("ac1: creating group chat with 10 other members")
         chat = ac1.create_group_chat(name="title1")
@@ -819,18 +844,12 @@ class TestOnlineAccount:
         assert msg_in.text == "message2"
         assert msg_in.is_forwarded()
 
-    def test_send_self_message_and_empty_folder(self, acfactory, lp):
+    def test_send_self_message(self, acfactory, lp):
         ac1 = acfactory.get_one_online_account(mvbox=True, move=True)
         lp.sec("ac1: create self chat")
         chat = ac1.get_self_contact().create_chat()
         chat.send_text("hello")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
-        ac1.empty_server_folders(inbox=True, mvbox=True)
-        ev1 = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
-        ev2 = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
-        boxes = [ev1.data2, ev2.data2]
-        boxes.remove("INBOX")
-        assert len(boxes) == 1 and boxes[0].endswith("DeltaChat")
 
     def test_send_and_receive_message_markseen(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -878,7 +897,13 @@ class TestOnlineAccount:
         lp.sec("mark messages as seen on ac2, wait for changes on ac1")
         ac2.direct_imap.idle_start()
         ac1.direct_imap.idle_start()
+
         ac2.mark_seen_messages([msg2, msg4])
+        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+        assert msg2.chat.id == msg4.chat.id
+        assert ev.data1 == msg2.chat.id
+        assert ev.data2 == 0
+
         ac2.direct_imap.idle_check(terminate=True)
         lp.step("1")
         for i in range(2):
@@ -1007,7 +1032,58 @@ class TestOnlineAccount:
         assert msg_in.text == text2
         assert ac1.get_config("addr") in [x.addr for x in msg_in.chat.get_contacts()]
 
+    def test_prefer_encrypt(self, acfactory, lp):
+        """Test quorum rule for encryption preference in 1:1 and group chat."""
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+        ac1.set_config("e2ee_enabled", "0")
+        ac2.set_config("e2ee_enabled", "1")
+        ac3.set_config("e2ee_enabled", "0")
+
+        # Make sure we do not send a copy to ourselves. This is to
+        # test that we count own preference even when we are not in
+        # the recipient list.
+        ac1.set_config("bcc_self", "0")
+        ac2.set_config("bcc_self", "0")
+        ac3.set_config("bcc_self", "0")
+
+        acfactory.introduce_each_other([ac1, ac2, ac3])
+
+        lp.sec("ac1: sending message to ac2")
+        chat1 = ac1.create_chat(ac2)
+        msg1 = chat1.send_text("message1")
+        assert not msg1.is_encrypted()
+        ac2._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac2: sending message to ac1")
+        chat2 = ac2.create_chat(ac1)
+        msg2 = chat2.send_text("message2")
+        assert not msg2.is_encrypted()
+        ac1._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac1: sending message to group chat with ac2 and ac3")
+        group = ac1.create_group_chat("hello")
+        group.add_contact(ac2)
+        group.add_contact(ac3)
+        msg3 = group.send_text("message3")
+        assert not msg3.is_encrypted()
+        ac2._evtracker.wait_next_incoming_message()
+        ac3._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac3: start preferring encryption and inform ac1")
+        ac3.set_config("e2ee_enabled", "1")
+        chat3 = ac3.create_chat(ac1)
+        msg4 = chat3.send_text("message4")
+        # ac1 still does not prefer encryption
+        assert not msg4.is_encrypted()
+        ac1._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac1: sending another message to group chat with ac2 and ac3")
+        msg5 = group.send_text("message5")
+        # Majority prefers encryption now
+        assert msg5.is_encrypted()
+
     def test_reply_encrypted(self, acfactory, lp):
+        """Test that replies to encrypted messages are encrypted."""
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         lp.sec("ac1: create chat with ac2")
@@ -1035,26 +1111,26 @@ class TestOnlineAccount:
         print("ac2: e2ee_enabled={}".format(ac2.get_config("e2ee_enabled")))
         ac1.set_config("e2ee_enabled", "0")
 
-        # Set unprepared and unencrypted draft to test that it is not
-        # taken into account when determining whether last message is
-        # encrypted.
-        msg_draft = Message.new_empty(ac1, "text")
-        msg_draft.set_text("message2 -- should be encrypted")
-        chat.set_draft(msg_draft)
+        for quoted_msg in msg1, msg3:
+            # Save the draft with a quote.
+            # It should be encrypted if quoted message is encrypted.
+            msg_draft = Message.new_empty(ac1, "text")
+            msg_draft.set_text("message reply")
+            msg_draft.quote = quoted_msg
+            chat.set_draft(msg_draft)
 
-        # Get the draft, prepare and send it.
-        msg_draft = chat.get_draft()
-        msg_out = chat.prepare_message(msg_draft)
-        chat.send_prepared(msg_out)
+            # Get the draft, prepare and send it.
+            msg_draft = chat.get_draft()
+            msg_out = chat.prepare_message(msg_draft)
+            chat.send_prepared(msg_out)
 
-        chat.set_draft(None)
-        assert chat.get_draft() is None
+            chat.set_draft(None)
+            assert chat.get_draft() is None
 
-        lp.sec("wait for ac2 to receive message")
-        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
-        msg_in = ac2.get_message_by_id(ev.data2)
-        assert msg_in.text == "message2 -- should be encrypted"
-        assert msg_in.is_encrypted()
+            msg_in = ac2._evtracker.wait_next_incoming_message()
+            assert msg_in.text == "message reply"
+            assert msg_in.quoted_text == quoted_msg.text
+            assert msg_in.is_encrypted() == quoted_msg.is_encrypted()
 
     def test_saved_mime_on_received_message(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1423,6 +1499,8 @@ class TestOnlineAccount:
         contact = ac1.create_contact(ac2)
         contact.set_blocked()
         assert contact.is_blocked()
+        ev = ac1._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        assert ev.data1 == contact.id
 
         lp.sec("ac2 sends a message to ac1 that does not arrive because it is blocked")
         ac2.create_chat(ac1).send_text("This will not arrive!")
@@ -1566,6 +1644,7 @@ class TestOnlineAccount:
         chat41 = ac4.create_chat(ac1)
         chat42 = ac4.create_chat(ac2)
         ac4.start_io()
+        ac4._evtracker.wait_all_initial_fetches()
 
         lp.sec("ac1: creating group chat with 2 other members")
         chat = ac1.create_group_chat("title", contacts=[ac2, ac3])
@@ -1733,6 +1812,37 @@ class TestOnlineAccount:
 
         assert len(imap2.get_all_messages()) == 1
 
+    def test_configure_error_msgs(self, acfactory):
+        ac1, configdict = acfactory.get_online_config()
+        ac1.update_config(configdict)
+        ac1.set_config("mail_pw", "abc")  # Wrong mail pw
+        ac1.configure()
+        while True:
+            ev = ac1._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+            if ev.data1 == 0:
+                break
+        # Password is wrong so it definitely has to say something about "password"
+        assert "password" in ev.data2
+
+        ac2, configdict = acfactory.get_online_config()
+        ac2.update_config(configdict)
+        ac2.set_config("addr", "abc@def.invalid")  # mail server can't be reached
+        ac2.configure()
+        while True:
+            ev = ac2._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+            if ev.data1 == 0:
+                break
+        # Can't connect so it probably should say something about "internet"
+        # again, should not repeat itself
+        # If this fails then probably `e.msg.to_lowercase().contains("could not resolve")`
+        # in configure/mod.rs returned false because the error message was changed
+        # (i.e. did not contain "could not resolve" anymore)
+        assert (ev.data2.count("internet") + ev.data2.count("network")) == 1
+        # Should mention that it can't connect:
+        assert ev.data2.count("connect") == 1
+        # The users do not know what "configuration" is
+        assert "configuration" not in ev.data2.lower()
+
     def test_name_changes(self, acfactory):
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac1.set_config("displayname", "Account 1")
@@ -1756,6 +1866,8 @@ class TestOnlineAccount:
         # Explicitly rename contact on ac2 to "Renamed"
         ac2.create_contact(contact, name="Renamed")
         assert contact.name == "Renamed"
+        ev = ac2._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        assert ev.data1 == contact.id
 
         # ac1 also renames itself into "Renamed"
         assert update_name() == "Renamed"
@@ -1773,6 +1885,82 @@ class TestOnlineAccount:
             # No renames should happen after explicit rename
             assert updated_name == "Renamed"
 
+    def test_group_quote(self, acfactory, lp):
+        """Test quoting in a group with a new member who have not seen the quoted message."""
+        ac1, ac2, ac3 = accounts = acfactory.get_many_online_accounts(3)
+        acfactory.introduce_each_other(accounts)
+        chat = ac1.create_group_chat(name="quote group")
+        chat.add_contact(ac2)
+
+        lp.sec("ac1: sending message")
+        out_msg = chat.send_text("hello")
+
+        lp.sec("ac2: receiving message")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        chat.add_contact(ac3)
+        ac2._evtracker.wait_next_incoming_message()
+        ac3._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac2: sending reply with a quote")
+        reply_msg = Message.new_empty(msg.chat.account, "text")
+        reply_msg.set_text("reply")
+        reply_msg.quote = msg
+        reply_msg = msg.chat.prepare_message(reply_msg)
+        assert reply_msg.quoted_text == "hello"
+        msg.chat.send_prepared(reply_msg)
+
+        lp.sec("ac3: receiving reply")
+        received_reply = ac3._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "reply"
+        assert received_reply.quoted_text == "hello"
+        # ac3 was not in the group and has not received quoted message
+        assert received_reply.quote is None
+
+        lp.sec("ac1: receiving reply")
+        received_reply = ac1._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "reply"
+        assert received_reply.quoted_text == "hello"
+        assert received_reply.quote.id == out_msg.id
+
+    @pytest.mark.parametrize("mvbox_move", [False, True])
+    def test_add_all_recipients_as_contacts(self, acfactory, lp, mvbox_move):
+        """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
+        This way, we can already offer them some email addresses they can write to.
+
+        Also test that existing emails are fetched during onboarding.
+
+        Lastly, tests that bcc_self messages moved to the mvbox are marked as read."""
+        ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure_and_start_io()
+
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("send out message with bcc to ourselves")
+        if mvbox_move:
+            ac1.direct_imap.select_config_folder("mvbox")
+        ac1.direct_imap.idle_start()
+        ac1.set_config("bcc_self", "1")
+        chat.send_text("message text")
+
+        # now wait until the bcc_self message arrives
+        # Also test that bcc_self messages moved to the mvbox are marked as read.
+        assert ac1.direct_imap.idle_wait_for_seen()
+
+        ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone._configtracker.wait_finish()
+        ac1_clone.start_io()
+
+        ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        ac2_addr = ac2.get_config("addr")
+        assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
+
+        msg = ac1_clone._evtracker.wait_next_messages_changed()
+        assert msg.text == "message text"
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):
@@ -1898,8 +2086,7 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "cannot login" in ev.data2.lower()
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_user(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -1907,8 +2094,7 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "cannot login" in ev.data2.lower()
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_domain(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -1916,5 +2102,4 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "could not connect" in ev.data2.lower()
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")

src/accounts.rs

@@ -1,10 +1,8 @@
-use std::collections::HashMap;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::task::{Context as TaskContext, Poll};
+use std::collections::BTreeMap;
 
 use async_std::fs;
 use async_std::path::PathBuf;
+use async_std::prelude::*;
 use async_std::sync::{Arc, RwLock};
 use uuid::Uuid;
 
@@ -20,7 +18,7 @@ use crate::events::Event;
 pub struct Accounts {
     dir: PathBuf,
     config: Config,
-    accounts: Arc<RwLock<HashMap<u32, Context>>>,
+    accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
 }
 
 impl Accounts {
@@ -225,61 +223,42 @@ impl Accounts {
 
     /// Unified event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
-        let emitters = self
+        let emitters: Vec<_> = self
             .accounts
             .read()
             .await
             .iter()
-            .map(|(id, a)| EmitterWrapper {
-                id: *id,
-                emitter: a.get_event_emitter(),
-                done: AtomicBool::new(false),
-            })
+            .map(|(_id, a)| a.get_event_emitter())
             .collect();
 
-        EventEmitter(emitters)
+        EventEmitter(futures::stream::select_all(emitters))
     }
 }
 
+#[derive(Debug)]
+pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
+
 impl EventEmitter {
-    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
-    pub fn recv_sync(&self) -> Option<Event> {
+    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
+    pub fn recv_sync(&mut self) -> Option<Event> {
         async_std::task::block_on(self.recv())
     }
 
-    /// Async recv of an event. Return `None` if the `Sender` has been droped.
-    pub async fn recv(&self) -> Option<Event> {
-        futures::future::poll_fn(|cx| Pin::new(self).recv_poll(cx)).await
-    }
-
-    fn recv_poll(self: Pin<&Self>, _cx: &mut TaskContext<'_>) -> Poll<Option<Event>> {
-        for e in &*self.0 {
-            if e.done.load(Ordering::Acquire) {
-                // skip emitters that are already done
-                continue;
-            }
-
-            match e.emitter.try_recv() {
-                Ok(event) => return Poll::Ready(Some(event)),
-                Err(async_std::sync::TryRecvError::Disconnected) => {
-                    e.done.store(true, Ordering::Release);
-                }
-                Err(async_std::sync::TryRecvError::Empty) => {}
-            }
-        }
-
-        Poll::Pending
+    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
+    pub async fn recv(&mut self) -> Option<Event> {
+        self.0.next().await
     }
 }
 
-#[derive(Debug)]
-pub struct EventEmitter(Vec<EmitterWrapper>);
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
 
-#[derive(Debug)]
-struct EmitterWrapper {
-    id: u32,
-    emitter: crate::events::EventEmitter,
-    done: AtomicBool,
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
+    }
 }
 
 pub const CONFIG_NAME: &str = "accounts.toml";
@@ -342,9 +321,9 @@ impl Config {
         })
     }
 
-    pub async fn load_accounts(&self) -> Result<HashMap<u32, Context>> {
+    pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
         let cfg = &*self.inner.read().await;
-        let mut accounts = HashMap::with_capacity(cfg.accounts.len());
+        let mut accounts = BTreeMap::new();
         for account_config in &cfg.accounts {
             let ctx = Context::new(
                 cfg.os_name.clone(),
@@ -530,4 +509,23 @@ mod tests {
             ctx.get_config(crate::config::Config::Addr).await.unwrap()
         );
     }
+
+    /// Tests that accounts are sorted by ID.
+    #[async_std::test]
+    async fn test_accounts_sorted() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+        for expected_id in 2..10 {
+            let id = accounts.add_account().await.unwrap();
+            assert_eq!(id, expected_id);
+        }
+
+        let ids = accounts.get_all().await;
+        for (i, expected_id) in (1..10).enumerate() {
+            assert_eq!(ids.get(i), Some(&expected_id));
+        }
+    }
 }

src/blob.rs

@@ -685,7 +685,7 @@ mod tests {
 
         let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
         assert!(stem.contains("foo"));
-        assert!(!stem.contains("?"));
+        assert!(!stem.contains('?'));
         assert_eq!(ext, ".bar");
 
         let (stem, ext) = BlobObject::sanitise_name("no-extension");
@@ -698,10 +698,10 @@ mod tests {
         assert!(!stem.contains("ignored"));
         assert!(stem.contains("this"));
         assert!(stem.contains("forbidden"));
-        assert!(!stem.contains("/"));
-        assert!(!stem.contains("\\"));
-        assert!(!stem.contains(":"));
-        assert!(!stem.contains("*"));
-        assert!(!stem.contains("?"));
+        assert!(!stem.contains('/'));
+        assert!(!stem.contains('\\'));
+        assert!(!stem.contains(':'));
+        assert!(!stem.contains('*'));
+        assert!(!stem.contains('?'));
     }
 }

src/chat.rs

@@ -3,6 +3,7 @@
 use std::convert::TryFrom;
 use std::time::{Duration, SystemTime};
 
+use anyhow::Context as _;
 use async_std::path::{Path, PathBuf};
 use itertools::Itertools;
 use num_traits::FromPrimitive;
@@ -70,10 +71,7 @@ impl ChatId {
     ///
     /// This kind of chat ID can not be used for real chats.
     pub fn is_special(self) -> bool {
-        match self.0 {
-            0..=DC_CHAT_ID_LAST_SPECIAL => true,
-            _ => false,
-        }
+        matches!(self.0, 0..=DC_CHAT_ID_LAST_SPECIAL)
     }
 
     /// Chat ID which represents the deaddrop chat.
@@ -95,13 +93,6 @@ impl ChatId {
         self.0 == DC_CHAT_ID_TRASH
     }
 
-    // DC_CHAT_ID_MSGS_IN_CREATION seems unused?
-
-    /// Virtual chat showing all starred messages.
-    pub fn is_starred(self) -> bool {
-        self.0 == DC_CHAT_ID_STARRED
-    }
-
     /// Chat ID signifying there are **any** number of archived chats.
     ///
     /// This chat ID can be returned in a [Chatlist] and signals to
@@ -324,14 +315,11 @@ impl ChatId {
     async fn do_set_draft(self, context: &Context, msg: &mut Message) -> Result<(), Error> {
         match msg.viewtype {
             Viewtype::Unknown => bail!("Can not set draft of unknown type."),
-            Viewtype::Text => match msg.text.as_ref() {
-                Some(text) => {
-                    if text.is_empty() {
-                        bail!("No text in draft");
-                    }
+            Viewtype::Text => {
+                if msg.text.is_none_or_empty() && msg.in_reply_to.is_none_or_empty() {
+                    bail!("No text and no quote in draft");
                 }
-                None => bail!("No text in draft"),
-            },
+            }
             _ => {
                 let blob = msg
                     .param
@@ -344,8 +332,8 @@ impl ChatId {
         context
             .sql
             .execute(
-                "INSERT INTO msgs (chat_id, from_id, timestamp, type, state, txt, param, hidden)
-         VALUES (?,?,?, ?,?,?,?,?);",
+                "INSERT INTO msgs (chat_id, from_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to)
+         VALUES (?,?,?, ?,?,?,?,?,?);",
                 paramsv![
                     self,
                     DC_CONTACT_ID_SELF,
@@ -355,6 +343,7 @@ impl ChatId {
                     msg.text.as_deref().unwrap_or(""),
                     msg.param.to_string(),
                     1,
+                    msg.in_reply_to.as_deref().unwrap_or_default(),
                 ],
             )
             .await?;
@@ -482,20 +471,6 @@ impl ChatId {
         }
     }
 
-    async fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
-        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?));
-        let res: Option<(String, String)> =
-            self.parent_query(context, "param, error", collect).await?;
-
-        if let Some((ref packed, ref error)) = res {
-            let param = packed.parse::<Params>()?;
-            Ok(error.is_empty() && param.exists(Param::GuaranteeE2ee))
-        } else {
-            // No messages
-            Ok(false)
-        }
-    }
-
     /// Bad evil escape hatch.
     ///
     /// Avoid using this, eventually types should be cleaned up enough
@@ -511,8 +486,6 @@ impl std::fmt::Display for ChatId {
             write!(f, "Chat#Deadrop")
         } else if self.is_trash() {
             write!(f, "Chat#Trash")
-        } else if self.is_starred() {
-            write!(f, "Chat#Starred")
         } else if self.is_archived_link() {
             write!(f, "Chat#ArchivedLink")
         } else if self.is_alldone_hint() {
@@ -613,8 +586,6 @@ impl Chat {
                     let tempname = context.stock_str(StockMessage::ArchivedChats).await;
                     let cnt = dc_get_archived_cnt(context).await;
                     chat.name = format!("{} ({})", tempname, cnt);
-                } else if chat.id.is_starred() {
-                    chat.name = context.stock_str(StockMessage::StarredMsgs).await.into();
                 } else {
                     if chat.typ == Chattype::Single {
                         let contacts = get_chat_contacts(context, chat.id).await;
@@ -781,7 +752,6 @@ impl Chat {
         timestamp: i64,
     ) -> Result<MsgId, Error> {
         let mut new_references = "".into();
-        let mut new_in_reply_to = "".into();
         let mut msg_id = 0;
         let mut to_id = 0;
         let mut location_id = 0;
@@ -804,217 +774,180 @@ impl Chat {
             bail!("Cannot set message; self not in group.");
         }
 
-        if let Some(from) = context.get_config(Config::ConfiguredAddr).await {
-            let new_rfc724_mid = {
-                let grpid = match self.typ {
-                    Chattype::Group | Chattype::VerifiedGroup => Some(self.grpid.as_str()),
-                    _ => None,
-                };
-                dc_create_outgoing_rfc724_mid(grpid, &from)
+        let from = context
+            .get_config(Config::ConfiguredAddr)
+            .await
+            .context("Cannot prepare message for sending, address is not configured.")?;
+
+        let new_rfc724_mid = {
+            let grpid = match self.typ {
+                Chattype::Group | Chattype::VerifiedGroup => Some(self.grpid.as_str()),
+                _ => None,
             };
+            dc_create_outgoing_rfc724_mid(grpid, &from)
+        };
 
-            if self.typ == Chattype::Single {
-                if let Some(id) = context
-                    .sql
-                    .query_get_value(
-                        context,
-                        "SELECT contact_id FROM chats_contacts WHERE chat_id=?;",
-                        paramsv![self.id],
-                    )
-                    .await
-                {
-                    to_id = id;
-                } else {
-                    error!(
-                        context,
-                        "Cannot send message, contact for {} not found.", self.id,
-                    );
-                    bail!("Cannot set message, contact for {} not found.", self.id);
-                }
-            } else if (self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup)
-                && self.param.get_int(Param::Unpromoted).unwrap_or_default() == 1
+        if self.typ == Chattype::Single {
+            if let Some(id) = context
+                .sql
+                .query_get_value(
+                    context,
+                    "SELECT contact_id FROM chats_contacts WHERE chat_id=?;",
+                    paramsv![self.id],
+                )
+                .await
             {
-                msg.param.set_int(Param::AttachGroupImage, 1);
-                self.param.remove(Param::Unpromoted);
-                self.update_param(context).await?;
+                to_id = id;
+            } else {
+                error!(
+                    context,
+                    "Cannot send message, contact for {} not found.", self.id,
+                );
+                bail!("Cannot set message, contact for {} not found.", self.id);
             }
+        } else if (self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup)
+            && self.param.get_int(Param::Unpromoted).unwrap_or_default() == 1
+        {
+            msg.param.set_int(Param::AttachGroupImage, 1);
+            self.param.remove(Param::Unpromoted);
+            self.update_param(context).await?;
+        }
 
-            /* check if we want to encrypt this message.  If yes and circumstances change
-            so that E2EE is no longer available at a later point (reset, changed settings),
-            we might not send the message out at all */
-            if msg.param.get_int(Param::ForcePlaintext).unwrap_or_default() == 0 {
-                let mut can_encrypt = true;
-                let mut all_mutual = context.get_config_bool(Config::E2eeEnabled).await;
+        // reset encrypt error state eg. for forwarding
+        msg.param.remove(Param::ErroneousE2ee);
 
-                // take care that this statement returns NULL rows
-                // if there is no peerstates for a chat member!
-                // for DC_PARAM_SELFTALK this statement does not return any row
-                let res = context
-                    .sql
-                    .query_map(
-                        "SELECT ps.prefer_encrypted, c.addr \
-                     FROM chats_contacts cc  \
-                     LEFT JOIN contacts c ON cc.contact_id=c.id  \
-                     LEFT JOIN acpeerstates ps ON c.addr=ps.addr  \
-                     WHERE cc.chat_id=?  AND cc.contact_id>9;",
-                        paramsv![self.id],
-                        |row| {
-                            let addr: String = row.get(1)?;
-
-                            if let Some(prefer_encrypted) = row.get::<_, Option<i32>>(0)? {
-                                // the peerstate exist, so we have either public_key or gossip_key
-                                // and can encrypt potentially
-                                if prefer_encrypted != 1 {
-                                    info!(
-                                        context,
-                                        "[autocrypt] peerstate for {} is {}",
-                                        addr,
-                                        if prefer_encrypted == 0 {
-                                            "NOPREFERENCE"
-                                        } else {
-                                            "RESET"
-                                        },
-                                    );
-                                    all_mutual = false;
-                                }
-                            } else {
-                                info!(context, "[autocrypt] no peerstate for {}", addr,);
-                                can_encrypt = false;
-                                all_mutual = false;
-                            }
-                            Ok(())
-                        },
-                        |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
-                    )
-                    .await;
-                match res {
-                    Ok(_) => {}
-                    Err(err) => {
-                        warn!(context, "chat: failed to load peerstates: {:?}", err);
-                    }
+        // set "In-Reply-To:" to identify the message to which the composed message is a reply;
+        // set "References:" to identify the "thread" of the conversation;
+        // both according to RFC 5322 3.6.4, page 25
+        //
+        // as self-talks are mainly used to transfer data between devices,
+        // we do not set In-Reply-To/References in this case.
+        if !self.is_self_talk() {
+            if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) =
+                self.id.get_parent_mime_headers(context).await
+            {
+                // "In-Reply-To:" is not changed if it is set manually.
+                // This does not affect "References:" header, it will contain "default parent" (the
+                // latest message in the thread) anyway.
+                if msg.in_reply_to.is_none() && !parent_rfc724_mid.is_empty() {
+                    msg.in_reply_to = Some(parent_rfc724_mid.clone());
                 }
 
-                if can_encrypt && (all_mutual || self.id.parent_is_encrypted(context).await?) {
-                    msg.param.set_int(Param::GuaranteeE2ee, 1);
+                // the whole list of messages referenced may be huge;
+                // only use the oldest and and the parent message
+                let parent_references = parent_references
+                    .find(' ')
+                    .and_then(|n| parent_references.get(..n))
+                    .unwrap_or(&parent_references);
+
+                if !parent_references.is_empty() && !parent_rfc724_mid.is_empty() {
+                    // angle brackets are added by the mimefactory later
+                    new_references = format!("{} {}", parent_references, parent_rfc724_mid);
+                } else if !parent_references.is_empty() {
+                    new_references = parent_references.to_string();
+                } else if !parent_in_reply_to.is_empty() && !parent_rfc724_mid.is_empty() {
+                    new_references = format!("{} {}", parent_in_reply_to, parent_rfc724_mid);
+                } else if !parent_in_reply_to.is_empty() {
+                    new_references = parent_in_reply_to;
                 }
             }
-            // reset encrypt error state eg. for forwarding
-            msg.param.remove(Param::ErroneousE2ee);
+        }
 
-            // set "In-Reply-To:" to identify the message to which the composed message is a reply;
-            // set "References:" to identify the "thread" of the conversation;
-            // both according to RFC 5322 3.6.4, page 25
-            //
-            // as self-talks are mainly used to transfer data between devices,
-            // we do not set In-Reply-To/References in this case.
-            if !self.is_self_talk() {
-                if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) =
-                    self.id.get_parent_mime_headers(context).await
-                {
-                    if !parent_rfc724_mid.is_empty() {
-                        new_in_reply_to = parent_rfc724_mid.clone();
-                    }
+        // add independent location to database
 
-                    // the whole list of messages referenced may be huge;
-                    // only use the oldest and and the parent message
-                    let parent_references = parent_references
-                        .find(' ')
-                        .and_then(|n| parent_references.get(..n))
-                        .unwrap_or(&parent_references);
-
-                    if !parent_references.is_empty() && !parent_rfc724_mid.is_empty() {
-                        // angle brackets are added by the mimefactory later
-                        new_references = format!("{} {}", parent_references, parent_rfc724_mid);
-                    } else if !parent_references.is_empty() {
-                        new_references = parent_references.to_string();
-                    } else if !parent_in_reply_to.is_empty() && !parent_rfc724_mid.is_empty() {
-                        new_references = format!("{} {}", parent_in_reply_to, parent_rfc724_mid);
-                    } else if !parent_in_reply_to.is_empty() {
-                        new_references = parent_in_reply_to;
-                    }
-                }
-            }
-
-            // add independent location to database
-
-            if msg.param.exists(Param::SetLatitude)
-                && context
-                    .sql
-                    .execute(
-                        "INSERT INTO locations \
+        if msg.param.exists(Param::SetLatitude)
+            && context
+                .sql
+                .execute(
+                    "INSERT INTO locations \
                      (timestamp,from_id,chat_id, latitude,longitude,independent)\
                      VALUES (?,?,?, ?,?,1);", // 1=DC_CONTACT_ID_SELF
-                        paramsv![
-                            timestamp,
-                            DC_CONTACT_ID_SELF,
-                            self.id,
-                            msg.param.get_float(Param::SetLatitude).unwrap_or_default(),
-                            msg.param.get_float(Param::SetLongitude).unwrap_or_default(),
-                        ],
-                    )
-                    .await
-                    .is_ok()
-            {
-                location_id = context
-                    .sql
-                    .get_rowid2(
-                        context,
-                        "locations",
-                        "timestamp",
+                    paramsv![
                         timestamp,
-                        "from_id",
-                        DC_CONTACT_ID_SELF as i32,
-                    )
-                    .await?;
-            }
+                        DC_CONTACT_ID_SELF,
+                        self.id,
+                        msg.param.get_float(Param::SetLatitude).unwrap_or_default(),
+                        msg.param.get_float(Param::SetLongitude).unwrap_or_default(),
+                    ],
+                )
+                .await
+                .is_ok()
+        {
+            location_id = context
+                .sql
+                .get_rowid2(
+                    context,
+                    "locations",
+                    "timestamp",
+                    timestamp,
+                    "from_id",
+                    DC_CONTACT_ID_SELF as i32,
+                )
+                .await?;
+        }
 
-            let ephemeral_timer = if msg.param.get_cmd() == SystemMessage::EphemeralTimerChanged {
-                EphemeralTimer::Disabled
-            } else {
-                self.id.get_ephemeral_timer(context).await?
-            };
-            let ephemeral_timestamp = match ephemeral_timer {
-                EphemeralTimer::Disabled => 0,
-                EphemeralTimer::Enabled { duration } => timestamp + i64::from(duration),
-            };
-
-            // add message to the database
-
-            if context.sql.execute(
-                        "INSERT INTO msgs (rfc724_mid, chat_id, from_id, to_id, timestamp, type, state, txt, param, hidden, mime_in_reply_to, mime_references, location_id, ephemeral_timer, ephemeral_timestamp) VALUES (?,?,?,?,?, ?,?,?,?,?, ?,?,?,?,?);",
-                        paramsv![
-                            new_rfc724_mid,
-                            self.id,
-                            DC_CONTACT_ID_SELF,
-                            to_id as i32,
-                            timestamp,
-                            msg.viewtype,
-                            msg.state,
-                            msg.text.as_ref().cloned().unwrap_or_default(),
-                            msg.param.to_string(),
-                            msg.hidden,
-                            new_in_reply_to,
-                            new_references,
-                            location_id as i32,
-                            ephemeral_timer,
-                            ephemeral_timestamp
-                        ]
-                    ).await.is_ok() {
-                        msg_id = context.sql.get_rowid(
-                            context,
-                            "msgs",
-                            "rfc724_mid",
-                            new_rfc724_mid,
-                        ).await?;
-                    } else {
-                        error!(
-                            context,
-                            "Cannot send message, cannot insert to database ({}).",
-                            self.id,
-                        );
-                    }
+        let ephemeral_timer = if msg.param.get_cmd() == SystemMessage::EphemeralTimerChanged {
+            EphemeralTimer::Disabled
         } else {
-            error!(context, "Cannot send message, not configured.",);
+            self.id.get_ephemeral_timer(context).await?
+        };
+        let ephemeral_timestamp = match ephemeral_timer {
+            EphemeralTimer::Disabled => 0,
+            EphemeralTimer::Enabled { duration } => timestamp + i64::from(duration),
+        };
+
+        // add message to the database
+
+        if context
+            .sql
+            .execute(
+                "INSERT INTO msgs (
+                        rfc724_mid,
+                        chat_id,
+                        from_id,
+                        to_id,
+                        timestamp,
+                        type,
+                        state,
+                        txt,
+                        param,
+                        hidden,
+                        mime_in_reply_to,
+                        mime_references,
+                        location_id,
+                        ephemeral_timer,
+                        ephemeral_timestamp)
+                        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);",
+                paramsv![
+                    new_rfc724_mid,
+                    self.id,
+                    DC_CONTACT_ID_SELF,
+                    to_id as i32,
+                    timestamp,
+                    msg.viewtype,
+                    msg.state,
+                    msg.text.as_ref().cloned().unwrap_or_default(),
+                    msg.param.to_string(),
+                    msg.hidden,
+                    msg.in_reply_to.as_deref().unwrap_or_default(),
+                    new_references,
+                    location_id as i32,
+                    ephemeral_timer,
+                    ephemeral_timestamp
+                ],
+            )
+            .await
+            .is_ok()
+        {
+            msg_id = context
+                .sql
+                .get_rowid(context, "msgs", "rfc724_mid", new_rfc724_mid)
+                .await?;
+        } else {
+            error!(
+                context,
+                "Cannot send message, cannot insert to database ({}).", self.id,
+            );
         }
         schedule_ephemeral_task(context).await;
 
@@ -1166,7 +1099,7 @@ pub async fn create_by_msg_id(context: &Context, msg_id: MsgId) -> Result<ChatId
 }
 
 /// Create a normal chat with a single user.  To create group chats,
-/// see dc_create_group_chat().
+/// see [Chat::create_group_chat].
 ///
 /// If a chat already exists, this ID is returned, otherwise a new chat is created;
 /// this new chat may already contain messages, eg. from the deaddrop, to get the
@@ -1730,23 +1663,6 @@ pub async fn get_chat_msgs(
                 process_rows,
             )
             .await
-    } else if chat_id.is_starred() {
-        context
-            .sql
-            .query_map(
-                "SELECT m.id AS id, m.timestamp AS timestamp
-               FROM msgs m
-               LEFT JOIN contacts ct
-                      ON m.from_id=ct.id
-              WHERE m.starred=1
-                AND m.hidden=0
-                AND ct.blocked=0
-              ORDER BY m.timestamp,m.id;",
-                paramsv![],
-                process_row,
-                process_rows,
-            )
-            .await
     } else {
         context
             .sql
@@ -1771,45 +1687,34 @@ pub async fn get_chat_msgs(
     }
 }
 
-pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<(), Error> {
-    if !context
+pub(crate) async fn marknoticed_chat_if_older_than(
+    context: &Context,
+    chat_id: ChatId,
+    timestamp: i64,
+) -> Result<(), Error> {
+    if let Some(chat_timestamp) = context
         .sql
-        .exists(
-            "SELECT id FROM msgs  WHERE chat_id=? AND state=?;",
-            paramsv![chat_id, MessageState::InFresh],
-        )
-        .await?
-    {
-        return Ok(());
-    }
-
-    context
-        .sql
-        .execute(
-            "UPDATE msgs
-            SET state=13
-          WHERE chat_id=?
-            AND state=10;",
+        .query_get_value(
+            context,
+            "SELECT MAX(timestamp) FROM msgs WHERE chat_id=?",
             paramsv![chat_id],
         )
-        .await?;
-
-    context.emit_event(EventType::MsgsChanged {
-        chat_id: ChatId::new(0),
-        msg_id: MsgId::new(0),
-    });
-
+        .await
+    {
+        if timestamp > chat_timestamp {
+            marknoticed_chat(context, chat_id).await?;
+        }
+    }
     Ok(())
 }
 
-pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
+pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<(), Error> {
+    // "WHERE" below uses the index `(state, hidden, chat_id)`, see get_fresh_msg_cnt() for reasoning
     if !context
         .sql
         .exists(
-            "SELECT id
-           FROM msgs
-          WHERE state=10;",
-            paramsv![],
+            "SELECT id FROM msgs WHERE state=? AND hidden=0 AND chat_id=?;",
+            paramsv![MessageState::InFresh, chat_id],
         )
         .await?
     {
@@ -1820,16 +1725,15 @@ pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
         .sql
         .execute(
             "UPDATE msgs
-            SET state=13
-          WHERE state=10;",
-            paramsv![],
+            SET state=?
+          WHERE state=?
+            AND hidden=0
+            AND chat_id=?;",
+            paramsv![MessageState::InNoticed, MessageState::InFresh, chat_id],
         )
         .await?;
 
-    context.emit_event(EventType::MsgsChanged {
-        msg_id: MsgId::new(0),
-        chat_id: ChatId::new(0),
-    });
+    context.emit_event(EventType::MsgsNoticed(chat_id));
 
     Ok(())
 }
@@ -2697,6 +2601,7 @@ pub(crate) async fn get_chat_cnt(context: &Context) -> usize {
     }
 }
 
+/// Returns a tuple of `(chatid, is_verified, blocked)`.
 pub(crate) async fn get_chat_id_by_grpid(
     context: &Context,
     grpid: impl AsRef<str>,
@@ -2751,14 +2656,35 @@ pub async fn add_device_msg_with_importance(
         prepare_msg_blob(context, msg).await?;
         chat_id.unarchive(context).await?;
 
+        let timestamp_sent = dc_create_smeared_timestamp(context).await;
+
+        // makes sure, the added message is the last one,
+        // even if the date is wrong (useful esp. when warning about bad dates)
+        let mut timestamp_sort = timestamp_sent;
+        if let Some(last_msg_time) = context
+            .sql
+            .query_get_value(
+                context,
+                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=?",
+                paramsv![chat_id],
+            )
+            .await
+        {
+            if timestamp_sort <= last_msg_time {
+                timestamp_sort = last_msg_time + 1;
+            }
+        }
+
         context.sql.execute(
-            "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,param,rfc724_mid) \
-             VALUES (?,?,?, ?,?,?, ?,?,?);",
+            "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,timestamp_sent,timestamp_rcvd,type,state, txt,param,rfc724_mid) \
+             VALUES (?,?,?, ?,?,?,?,?, ?,?,?);",
             paramsv![
                 chat_id,
                 DC_CONTACT_ID_DEVICE,
                 DC_CONTACT_ID_SELF,
-                dc_create_smeared_timestamp(context).await,
+                timestamp_sort,
+                timestamp_sent,
+                timestamp_sent, // timestamp_sent equals timestamp_rcvd
                 msg.viewtype,
                 MessageState::InFresh,
                 msg.text.as_ref().cloned().unwrap_or_default(),
@@ -2886,7 +2812,6 @@ pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl
 mod tests {
     use super::*;
 
-    use crate::chat;
     use crate::contact::Contact;
     use crate::test_utils::*;
 
@@ -3535,44 +3460,4 @@ mod tests {
             false
         );
     }
-
-    #[async_std::test]
-    async fn test_parent_is_encrypted() {
-        let t = TestContext::new().await;
-        let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
-            .await
-            .unwrap();
-        assert!(!chat_id.parent_is_encrypted(&t.ctx).await.unwrap());
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("hello".to_string()));
-        chat_id.set_draft(&t.ctx, Some(&mut msg)).await;
-        assert!(!chat_id.parent_is_encrypted(&t.ctx).await.unwrap());
-    }
-
-    #[async_std::test]
-    async fn test_get_fresh_msg_cnt() {
-        let t = TestContext::new().await;
-        // create 50 chats with 1k messages each
-        let mut chat_ids = Vec::new();
-        for _i in 1..50 {
-            let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
-                .await
-                .unwrap();
-            for _i in 1..1000 {
-                let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some("message text".to_string());
-                chat::send_msg(&t.ctx, chat_id, &mut msg)
-                    .await
-                    .unwrap_or_default();
-            }
-            chat_ids.push(chat_id);
-        }
-        // load all chats 20k times
-        for _i in 1..20_000 {
-            for chat_id in chat_ids.iter() {
-                chat_id.get_fresh_msg_cnt(&t.ctx).await;
-            }
-        }
-    }
 }

src/config.rs

@@ -121,9 +121,9 @@ pub enum Config {
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
 
-    #[strum(props(default = "0"))]
     /// Whether we send a warning if the password is wrong (set to false when we send a warning
     /// because we do not want to send a second warning)
+    #[strum(props(default = "0"))]
     NotifyAboutWrongPw,
 
     /// address to webrtc instance to use for videochats

src/configure/auto_mozilla.rs

@@ -268,6 +268,8 @@ pub(crate) async fn moz_autoconfigure(
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]

src/configure/auto_outlook.rs

@@ -209,6 +209,8 @@ pub(crate) async fn outlk_autodiscover(
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]

src/configure/mod.rs

@@ -7,11 +7,12 @@ mod server_params;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
+use async_std::task;
+use itertools::Itertools;
+use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::config::Config;
-use crate::constants::*;
-use crate::context::Context;
 use crate::dc_tools::*;
 use crate::imap::Imap;
 use crate::login_param::{LoginParam, ServerLoginParam};
@@ -19,19 +20,28 @@ use crate::message::Message;
 use crate::oauth2::*;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
+use crate::stock::StockMessage;
 use crate::{chat, e2ee, provider};
+use crate::{constants::*, job};
+use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
-use server_params::ServerParams;
+use server_params::{expand_param_vector, ServerParams};
 
 macro_rules! progress {
-    ($context:tt, $progress:expr) => {
+    ($context:tt, $progress:expr, $comment:expr) => {
         assert!(
             $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::EventType::ConfigureProgress($progress));
+        $context.emit_event($crate::events::EventType::ConfigureProgress {
+            progress: $progress,
+            comment: $comment,
+        });
+    };
+    ($context:tt, $progress:expr) => {
+        progress!($context, $progress, None);
     };
 }
 
@@ -110,7 +120,18 @@ impl Context {
                 Ok(())
             }
             Err(err) => {
-                progress!(self, 0);
+                progress!(
+                    self,
+                    0,
+                    Some(
+                        self.stock_string_repl_str(
+                            StockMessage::ConfigurationFailed,
+                            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                            format!("{:#}", err),
+                        )
+                        .await
+                    )
+                );
                 Err(err)
             }
         }
@@ -193,8 +214,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 500);
 
-    let servers: Vec<ServerParams> = param_autoconfig
-        .unwrap_or_else(|| {
+    let servers = expand_param_vector(
+        param_autoconfig.unwrap_or_else(|| {
             vec![
                 ServerParams {
                     protocol: Protocol::IMAP,
@@ -211,62 +232,97 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                     username: param.smtp.user.clone(),
                 },
             ]
-        })
-        .into_iter()
-        // The order of expansion is important: ports are expanded the
-        // last, so they are changed the first. Username is only
-        // changed if default value (address with domain) didn't work
-        // for all available hosts and ports.
-        .flat_map(|params| params.expand_usernames(&param.addr).into_iter())
-        .flat_map(|params| params.expand_hostnames(&param_domain).into_iter())
-        .flat_map(|params| params.expand_ports().into_iter())
+        }),
+        &param.addr,
+        &param_domain,
+    );
+
+    progress!(ctx, 550);
+
+    // Spawn SMTP configuration task
+    let mut smtp = Smtp::new();
+
+    let context_smtp = ctx.clone();
+    let mut smtp_param = param.smtp.clone();
+    let smtp_addr = param.addr.clone();
+    let smtp_servers: Vec<ServerParams> = servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::SMTP)
+        .cloned()
         .collect();
 
-    // Configure IMAP
+    let smtp_config_task = task::spawn(async move {
+        let mut smtp_configured = false;
+        let mut errors = Vec::new();
+        for smtp_server in smtp_servers {
+            smtp_param.user = smtp_server.username.clone();
+            smtp_param.server = smtp_server.hostname.clone();
+            smtp_param.port = smtp_server.port;
+            smtp_param.security = smtp_server.socket;
+
+            match try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp)
+                .await
+            {
+                Ok(_) => {
+                    smtp_configured = true;
+                    break;
+                }
+                Err(e) => errors.push(e),
+            }
+        }
+
+        if smtp_configured {
+            Ok(smtp_param)
+        } else {
+            Err(errors)
+        }
+    });
+
     progress!(ctx, 600);
+
+    // Configure IMAP
     let (_s, r) = async_std::sync::channel(1);
     let mut imap = Imap::new(r);
 
     let mut imap_configured = false;
-    for imap_server in servers
+    let imap_servers: Vec<&ServerParams> = servers
         .iter()
         .filter(|params| params.protocol == Protocol::IMAP)
-    {
+        .collect();
+    let imap_servers_count = imap_servers.len();
+    let mut errors = Vec::new();
+    for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
         param.imap.user = imap_server.username.clone();
         param.imap.server = imap_server.hostname.clone();
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
 
-        if try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
-            imap_configured = true;
-            break;
+        match try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
+            Ok(_) => {
+                imap_configured = true;
+                break;
+            }
+            Err(e) => errors.push(e),
         }
+        progress!(
+            ctx,
+            600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
+        );
     }
     if !imap_configured {
-        bail!("IMAP autoconfig did not succeed");
+        bail!(nicer_configuration_error(ctx, errors).await);
     }
 
-    // Configure SMTP
-    progress!(ctx, 750);
-    let mut smtp = Smtp::new();
+    progress!(ctx, 850);
 
-    let mut smtp_configured = false;
-    for smtp_server in servers
-        .iter()
-        .filter(|params| params.protocol == Protocol::SMTP)
-    {
-        param.smtp.user = smtp_server.username.clone();
-        param.smtp.server = smtp_server.hostname.clone();
-        param.smtp.port = smtp_server.port;
-        param.smtp.security = smtp_server.socket;
-
-        if try_smtp_one_param(ctx, &param.smtp, &param.addr, oauth2, &mut smtp).await {
-            smtp_configured = true;
-            break;
+    // Wait for SMTP configuration
+    match smtp_config_task.await {
+        Ok(smtp_param) => {
+            param.smtp = smtp_param;
+        }
+        Err(errors) => {
+            bail!(nicer_configuration_error(ctx, errors).await);
         }
-    }
-    if !smtp_configured {
-        bail!("SMTP autoconfig did not succeed");
     }
 
     progress!(ctx, 900);
@@ -294,6 +350,12 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     e2ee::ensure_secret_key_exists(ctx).await?;
     info!(ctx, "key generation completed");
 
+    job::add(
+        ctx,
+        job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
+    )
+    .await;
+
     progress!(ctx, 940);
 
     Ok(())
@@ -438,7 +500,7 @@ async fn try_imap_one_param(
     addr: &str,
     oauth2: bool,
     imap: &mut Imap,
-) -> bool {
+) -> Result<(), ConfigurationError> {
     let inf = format!(
         "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
         param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
@@ -447,10 +509,13 @@ async fn try_imap_one_param(
 
     if let Err(err) = imap.connect(context, param, addr, oauth2).await {
         info!(context, "failure: {}", err);
-        false
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
     } else {
         info!(context, "success: {}", inf);
-        true
+        Ok(())
     }
 }
 
@@ -460,7 +525,7 @@ async fn try_smtp_one_param(
     addr: &str,
     oauth2: bool,
     smtp: &mut Smtp,
-) -> bool {
+) -> Result<(), ConfigurationError> {
     let inf = format!(
         "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
         param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
@@ -469,27 +534,61 @@ async fn try_smtp_one_param(
 
     if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
         info!(context, "failure: {}", err);
-        false
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
     } else {
         info!(context, "success: {}", inf);
         smtp.disconnect().await;
-        true
+        Ok(())
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+#[error("Trying {config}…\nError: {msg}")]
+pub struct ConfigurationError {
+    config: String,
+    msg: String,
+}
+
+async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationError>) -> String {
+    let first_err = if let Some(f) = errors.first() {
+        f
+    } else {
+        return "".to_string();
+    };
+
+    if errors
+        .iter()
+        .all(|e| e.msg.to_lowercase().contains("could not resolve"))
+    {
+        return context
+            .stock_str(StockMessage::ErrorNoNetwork)
+            .await
+            .to_string();
+    }
+
+    if errors.iter().all(|e| e.msg == first_err.msg) {
+        return first_err.msg.to_string();
+    }
+
+    errors.iter().map(|e| e.to_string()).join("\n\n")
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("Invalid email address: {0:?}")]
     InvalidEmailAddress(String),
 
-    #[error("XML error at position {position}")]
+    #[error("XML error at position {position}: {error}")]
     InvalidXml {
         position: usize,
         #[source]
         error: quick_xml::Error,
     },
 
-    #[error("Failed to get URL")]
+    #[error("Failed to get URL: {0}")]
     ReadUrlError(#[from] self::read_url::Error),
 
     #[error("Number of redirection is exceeded")]
@@ -498,6 +597,7 @@ pub enum Error {
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
 
     use super::*;
     use crate::config::*;

src/configure/read_url.rs

@@ -12,7 +12,7 @@ pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
     match surf::get(url).recv_string().await {
         Ok(res) => Ok(res),
         Err(err) => {
-            info!(context, "Can\'t read URL {}", url);
+            info!(context, "Can\'t read URL {}: {}", url, err);
 
             Err(Error::GetError(err))
         }

src/configure/server_params.rs

@@ -6,7 +6,7 @@ use crate::provider::{Protocol, Socket};
 ///
 /// Can be loaded from offline provider database, online configuraiton
 /// or derived from user entered parameters.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub(crate) struct ServerParams {
     /// Protocol, such as IMAP or SMTP.
     pub protocol: Protocol,
@@ -113,3 +113,52 @@ impl ServerParams {
         res
     }
 }
+
+/// Expands vector of `ServerParams`, replacing placeholders with
+/// variants to try.
+pub(crate) fn expand_param_vector(
+    v: Vec<ServerParams>,
+    addr: &str,
+    domain: &str,
+) -> Vec<ServerParams> {
+    v.into_iter()
+        // The order of expansion is important: ports are expanded the
+        // last, so they are changed the first. Username is only
+        // changed if default value (address with domain) didn't work
+        // for all available hosts and ports.
+        .flat_map(|params| params.expand_usernames(addr).into_iter())
+        .flat_map(|params| params.expand_hostnames(domain).into_iter())
+        .flat_map(|params| params.expand_ports().into_iter())
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_expand_param_vector() {
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::IMAP,
+                hostname: "example.net".to_string(),
+                port: 0,
+                socket: Socket::SSL,
+                username: "foobar".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+
+        assert_eq!(
+            v,
+            vec![ServerParams {
+                protocol: Protocol::IMAP,
+                hostname: "example.net".to_string(),
+                port: 993,
+                socket: Socket::SSL,
+                username: "foobar".to_string(),
+            }],
+        );
+    }
+}

src/constants.rs

@@ -1,6 +1,4 @@
 //! # Constants
-#![allow(dead_code)]
-
 use deltachat_derive::*;
 use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
@@ -9,13 +7,6 @@ lazy_static! {
     pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
 }
 
-// some defaults
-const DC_E2EE_DEFAULT_ENABLED: i32 = 1;
-const DC_INBOX_WATCH_DEFAULT: i32 = 1;
-const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
-const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
-const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
-
 #[derive(
     Debug,
     Display,
@@ -117,14 +108,16 @@ pub const DC_GCL_ADD_SELF: usize = 0x02;
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 
+// warn about an outdated app after a given number of days.
+// as we use the "provider-db generation date" as reference (that might not be updated very often)
+// and as not all system get speedy updates,
+// do not use too small value that will annoy users checking for nonexistant updates.
+pub const DC_OUTDATED_WARNING_DAYS: i64 = 365;
+
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
-pub(crate) const DC_CHAT_ID_DEADDROP: u32 = 1;
+pub const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
 pub const DC_CHAT_ID_TRASH: u32 = 3;
-/// a message is just in creation but not yet assigned to a chat (eg. we may need the message ID to set up blobs; this avoids unready message to be sent and shown)
-const DC_CHAT_ID_MSGS_IN_CREATION: u32 = 4;
-/// virtual chat showing all messages flagged with msgs.starred=2
-pub const DC_CHAT_ID_STARRED: u32 = 5;
 /// only an indicator in a chatlist
 pub const DC_CHAT_ID_ARCHIVED_LINK: u32 = 6;
 /// only an indicator in a chatlist
@@ -166,9 +159,9 @@ pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
 /// approx. max. length returned by dc_msg_get_text()
-const DC_MAX_GET_TEXT_LEN: usize = 30000;
+pub const DC_MAX_GET_TEXT_LEN: usize = 30000;
 /// approx. max. length returned by dc_get_msg_info()
-const DC_MAX_GET_INFO_LEN: usize = 100_000;
+pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -202,6 +195,9 @@ pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
 
+/// How many existing messages shall be fetched after configuration.
+pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
+
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
@@ -295,16 +291,6 @@ mod tests {
     }
 }
 
-// These constants are used as events
-// reported to the callback given to dc_context_new().
-// If you do not want to handle an event, it is always safe to return 0,
-// so there is no need to add a "case" for every event.
-
-const DC_EVENT_FILE_COPIED: usize = 2055; // deprecated;
-const DC_EVENT_IS_OFFLINE: usize = 2081; // deprecated;
-const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
-const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]

src/contact.rs

@@ -16,7 +16,7 @@ use crate::error::{bail, ensure, format_err, Result};
 use crate::events::EventType;
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::message::{MessageState, MsgId};
+use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::*;
 use crate::peerstate::*;
@@ -247,13 +247,12 @@ impl Contact {
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, name, addr, Origin::ManuallyCreated).await?;
         let blocked = Contact::is_blocked_load(context, contact_id).await;
-        context.emit_event(EventType::ContactsChanged(
-            if sth_modified == Modifier::Created {
-                Some(contact_id)
-            } else {
-                None
-            },
-        ));
+        match sth_modified {
+            Modifier::None => {}
+            Modifier::Modified | Modifier::Created => {
+                context.emit_event(EventType::ContactsChanged(Some(contact_id)))
+            }
+        }
         if blocked {
             Contact::unblock(context, contact_id).await;
         }
@@ -261,10 +260,9 @@ impl Contact {
         Ok(contact_id)
     }
 
-    /// Mark all messages sent by the given contact
-    /// as *noticed*.  See also dc_marknoticed_chat() and dc_markseen_msgs()
-    ///
-    /// Calling this function usually results in the event `#DC_EVENT_MSGS_CHANGED`.
+    /// Mark messages from a contact as noticed.
+    /// The contact is expected to belong to the deaddrop,
+    /// therefore, DC_EVENT_MSGS_NOTICED(DC_CHAT_ID_DEADDROP) is emitted.
     pub async fn mark_noticed(context: &Context, id: u32) {
         if context
             .sql
@@ -275,10 +273,7 @@ impl Contact {
             .await
             .is_ok()
         {
-            context.emit_event(EventType::MsgsChanged {
-                chat_id: ChatId::new(0),
-                msg_id: MsgId::new(0),
-            });
+            context.emit_event(EventType::MsgsNoticed(ChatId::new(DC_CHAT_ID_DEADDROP)));
         }
     }
 
@@ -457,10 +452,20 @@ impl Contact {
                 if update_name {
                     // Update the contact name also if it is used as a group name.
                     // This is one of the few duplicated data, however, getting the chat list is easier this way.
-                    context.sql.execute(
-                    "UPDATE chats SET name=? WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    paramsv![new_name, Chattype::Single, row_id]
-                ).await.ok();
+                    let chat_id = context.sql.query_get_value::<i32>(
+                        context,
+                        "SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
+                        paramsv![Chattype::Single, row_id]
+                    ).await;
+                    if let Some(chat_id) = chat_id {
+                        match context.sql.execute("UPDATE chats SET name=? WHERE id=? AND name!=?1", paramsv![new_name, chat_id]).await {
+                            Err(err) => warn!(context, "Can't update chat name: {}", err),
+                            Ok(count) => if count > 0 {
+                                // Chat name updated
+                                context.emit_event(EventType::ChatModified(ChatId::new(chat_id as u32)));
+                            }
+                        }
+                    }
                 }
                 sth_modified = Modifier::Modified;
             }
@@ -1091,12 +1096,12 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
             // However, I'm not sure about this point; it may be confusing if the user wants to add other people;
             // this would result in recreating the same group...)
             if context.sql.execute(
-                    "UPDATE chats SET blocked=? WHERE type=? AND id IN (SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    paramsv![new_blocking, 100, contact_id as i32],
-                ).await.is_ok() {
-                    Contact::mark_noticed(context, contact_id).await;
-                    context.emit_event(EventType::ContactsChanged(None));
-                }
+                "UPDATE chats SET blocked=? WHERE type=? AND id IN (SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
+                paramsv![new_blocking, 100, contact_id as i32]).await.is_ok()
+            {
+                Contact::mark_noticed(context, contact_id).await;
+                context.emit_event(EventType::ContactsChanged(Some(contact_id)));
+            }
         }
     }
 }

src/context.rs

@@ -136,10 +136,7 @@ impl Context {
         let ctx = Context {
             inner: Arc::new(inner),
         };
-        ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, false).await,
-            "Failed opening sqlite database"
-        );
+        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }
@@ -573,7 +570,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir, 1).await;
         assert!(res.is_err());
     }
 

src/dc_receive_imf.rs

@@ -19,9 +19,7 @@ use crate::message::{self, MessageState, MessengerMessage, MsgId};
 use crate::mimeparser::*;
 use crate::param::*;
 use crate::peerstate::*;
-use crate::securejoin::{
-    self, handle_securejoin_handshake, observe_securejoin_on_other_device, BobStatus,
-};
+use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device};
 use crate::stock::StockMessage;
 use crate::{contact, location};
 
@@ -45,6 +43,17 @@ pub async fn dc_receive_imf(
     server_folder: impl AsRef<str>,
     server_uid: u32,
     seen: bool,
+) -> Result<()> {
+    dc_receive_imf_inner(context, imf_raw, server_folder, server_uid, seen, false).await
+}
+
+pub(crate) async fn dc_receive_imf_inner(
+    context: &Context,
+    imf_raw: &[u8],
+    server_folder: impl AsRef<str>,
+    server_uid: u32,
+    seen: bool,
+    fetching_existing_messages: bool,
 ) -> Result<()> {
     info!(
         context,
@@ -171,6 +180,7 @@ pub async fn dc_receive_imf(
             &mut insert_msg_id,
             &mut created_db_entries,
             &mut create_event_to_send,
+            fetching_existing_messages,
         )
         .await
         {
@@ -337,6 +347,7 @@ async fn add_parts(
     insert_msg_id: &mut MsgId,
     created_db_entries: &mut Vec<(ChatId, MsgId)>,
     create_event_to_send: &mut Option<CreateEvent>,
+    fetching_existing_messages: bool,
 ) -> Result<()> {
     let mut state: MessageState;
     let mut chat_id_blocked = Blocked::Not;
@@ -391,7 +402,7 @@ async fn add_parts(
     let to_id: u32;
 
     if incoming {
-        state = if seen {
+        state = if seen || fetching_existing_messages {
             MessageState::InSeen
         } else {
             MessageState::InFresh
@@ -418,8 +429,6 @@ async fn add_parts(
                 }
                 Err(err) => {
                     *hidden = true;
-
-                    context.bob.write().await.status = BobStatus::Error; // secure-join failed
                     context.stop_ongoing().await;
                     warn!(context, "Error in Secure-Join message handling: {}", err);
                     return Ok(());
@@ -536,6 +545,10 @@ async fn add_parts(
             && show_emails != ShowEmails::All
         {
             state = MessageState::InNoticed;
+        } else if fetching_existing_messages && Blocked::Deaddrop == chat_id_blocked {
+            // The fetched existing message should be shown in the chatlist-contact-request because
+            // a new user won't find the contact request in the menu
+            state = MessageState::InFresh;
         }
     } else {
         // Outgoing
@@ -632,6 +645,12 @@ async fn add_parts(
         }
     }
 
+    if fetching_existing_messages && mime_parser.decrypting_failed {
+        *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
+        // We are only gathering old messages on first start. We do not want to add loads of non-decryptable messages to the chats.
+        info!(context, "Dropping existing non-decipherable message.");
+    }
+
     // Extract ephemeral timer from the message.
     let mut ephemeral_timer = if let Some(value) = mime_parser.get(HeaderDef::EphemeralTimer) {
         match value.parse::<EphemeralTimer>() {
@@ -819,7 +838,7 @@ async fn add_parts(
                     mime_headers,
                     mime_in_reply_to,
                     mime_references,
-                    part.error,
+                    part.error.take().unwrap_or_default(),
                     ephemeral_timer,
                     ephemeral_timestamp
                 ])?;
@@ -849,6 +868,11 @@ async fn add_parts(
         "Message has {} parts and is assigned to chat #{}.", icnt, chat_id,
     );
 
+    // new outgoing message from another device marks the chat as noticed.
+    if !incoming && !*hidden && !chat_id.is_special() {
+        chat::marknoticed_chat_if_older_than(context, chat_id, sort_timestamp).await?;
+    }
+
     // check event to send
     if chat_id.is_trash() || *hidden {
         *create_event_to_send = None;
@@ -1534,7 +1558,7 @@ async fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> String {
             },
         )
         .await
-        .unwrap_or_else(|_| member_cs);
+        .unwrap_or(member_cs);
 
     hex_hash(&members)
 }
@@ -1562,7 +1586,7 @@ async fn search_chat_ids_by_contact_ids(
             }
         }
         if !contact_ids.is_empty() {
-            contact_ids.sort();
+            contact_ids.sort_unstable();
             let contact_ids_str = join(contact_ids.iter().map(|x| x.to_string()), ",");
             context.sql.query_map(
                 format!(
@@ -2213,7 +2237,7 @@ mod tests {
         } else {
             panic!("Wrong item type");
         };
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+        let msg = message::Message::load_from_db(&t.ctx, *msg_id)
             .await
             .unwrap();
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
@@ -2264,7 +2288,7 @@ mod tests {
             chat::get_chat_msgs(&t.ctx, group_id, 0, None).await.len(),
             1
         );
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+        let msg = message::Message::load_from_db(&t.ctx, *msg_id)
             .await
             .unwrap();
         assert_eq!(msg.state, MessageState::OutMdnRcvd);
@@ -2352,7 +2376,7 @@ mod tests {
         } else {
             panic!("Wrong item type");
         };
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+        let msg = message::Message::load_from_db(&t.ctx, *msg_id)
             .await
             .unwrap();
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
@@ -2459,7 +2483,7 @@ mod tests {
             "shenauithz@testrun.org",
             "Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it",
             include_bytes!("../test-data/message/tiscali_ndn.eml"),
-            "",
+            None,
         )
         .await;
     }
@@ -2471,7 +2495,7 @@ mod tests {
             "hcksocnsofoejx@five.chat",
             "Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org",
             include_bytes!("../test-data/message/testrun_ndn.eml"),
-            "Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"
+            Some("Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"),
         )
         .await;
     }
@@ -2483,7 +2507,7 @@ mod tests {
             "haeclirth.sinoenrat@yahoo.com",
             "1680295672.3657931.1591783872936@mail.yahoo.com",
             include_bytes!("../test-data/message/yahoo_ndn.eml"),
-            "Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com [...]"
+            Some("Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com [...]"),
         )
         .await;
     }
@@ -2495,7 +2519,7 @@ mod tests {
             "assidhfaaspocwaeofi@gmail.com",
             "CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com",
             include_bytes!("../test-data/message/gmail_ndn.eml"),
-            "Delivery Status Notification (Failure) – ** Die Adresse wurde nicht gefunden **\n\nIhre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.\n\nHier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser\n\nAntwort:\n\n550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient\'s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp",
+            Some("Delivery Status Notification (Failure) – ** Die Adresse wurde nicht gefunden **\n\nIhre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.\n\nHier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser\n\nAntwort:\n\n550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient\'s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp"),
         )
         .await;
     }
@@ -2507,7 +2531,7 @@ mod tests {
             "snaerituhaeirns@gmail.com",
             "9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de",
             include_bytes!("../test-data/message/gmx_ndn.eml"),
-            "Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp [...]"
+            Some("Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp [...]"),
         )
         .await;
     }
@@ -2519,7 +2543,7 @@ mod tests {
             "hanerthaertidiuea@gmx.de",
             "04422840-f884-3e37-5778-8192fe22d8e1@posteo.de",
             include_bytes!("../test-data/message/posteo_ndn.eml"),
-            "Undelivered Mail Returned to Sender – This is the mail system at host mout01.posteo.de.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550\n    Requested action not taken: mailbox unavailable (in reply to RCPT TO\n    command)",
+            Some("Undelivered Mail Returned to Sender – This is the mail system at host mout01.posteo.de.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550\n    Requested action not taken: mailbox unavailable (in reply to RCPT TO\n    command)"),
         )
         .await;
     }
@@ -2530,7 +2554,7 @@ mod tests {
         foreign_addr: &str,
         rfc724_mid_outgoing: &str,
         raw_ndn: &[u8],
-        error_msg: &str,
+        error_msg: Option<&str>,
     ) {
         let t = TestContext::new().await;
         t.configure_addr(self_addr).await;
@@ -2573,7 +2597,8 @@ mod tests {
         let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
 
         assert_eq!(msg.state, MessageState::OutFailed);
-        assert_eq!(msg.error, error_msg);
+
+        assert_eq!(msg.error(), error_msg.map(|error| error.to_string()));
     }
 
     #[async_std::test]

src/dc_tools.rs

@@ -15,9 +15,14 @@ use async_std::{fs, io};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
+use crate::chat::{add_device_msg, add_device_msg_with_importance};
+use crate::constants::{Viewtype, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::error::{bail, Error};
 use crate::events::EventType;
+use crate::message::Message;
+use crate::provider::get_provider_update_timestamp;
+use crate::stock::StockMessage;
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
@@ -151,6 +156,73 @@ pub(crate) async fn dc_create_smeared_timestamps(context: &Context, count: usize
     start
 }
 
+// if the system time is not plausible, once a day, add a device message.
+// for testing we're using time() as that is also used for message timestamps.
+// moreover, add a warning if the app is outdated.
+pub(crate) async fn maybe_add_time_based_warnings(context: &Context) {
+    if !maybe_warn_on_bad_time(context, time(), get_provider_update_timestamp()).await {
+        maybe_warn_on_outdated(context, time(), get_provider_update_timestamp()).await;
+    }
+}
+
+async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestamp: i64) -> bool {
+    if now < known_past_timestamp {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(
+            context
+                .stock_string_repl_str(
+                    StockMessage::BadTimeMsgBody,
+                    Local
+                        .timestamp(now, 0)
+                        .format("%Y-%m-%d %H:%M:%S")
+                        .to_string(),
+                )
+                .await,
+        );
+        add_device_msg_with_importance(
+            context,
+            Some(
+                format!(
+                    "bad-time-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m-%d") // repeat every day
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+            true,
+        )
+        .await
+        .ok();
+        return true;
+    }
+    false
+}
+
+async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time: i64) {
+    if now > approx_compile_time + DC_OUTDATED_WARNING_DAYS * 24 * 60 * 60 {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(
+            context
+                .stock_str(StockMessage::UpdateReminderMsgBody)
+                .await
+                .into(),
+        );
+        add_device_msg(
+            context,
+            Some(
+                format!(
+                    "outdated-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m") // repeat every month
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+        )
+        .await
+        .ok();
+    }
+}
+
 /* Message-ID tools */
 pub(crate) fn dc_create_id() -> String {
     /* generate an id. the generated ID should be as short and as unique as possible:
@@ -637,8 +709,25 @@ pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
         .to_string()
 }
 
+pub(crate) trait IsNoneOrEmpty<T> {
+    fn is_none_or_empty(&self) -> bool;
+}
+impl<T> IsNoneOrEmpty<T> for Option<T>
+where
+    T: AsRef<str>,
+{
+    fn is_none_or_empty(&self) -> bool {
+        match self {
+            Some(s) if !s.as_ref().is_empty() => false,
+            _ => true,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use std::convert::TryInto;
 
@@ -798,6 +887,9 @@ mod tests {
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
+    use crate::chat;
+    use crate::chatlist::Chatlist;
+    use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
     use proptest::prelude::*;
 
     proptest! {
@@ -991,4 +1083,123 @@ mod tests {
         assert_eq!(improve_single_line_input("Hi\naiae "), "Hi aiae");
         assert_eq!(improve_single_line_input("\r\nahte\n\r"), "ahte");
     }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_bad_time() {
+        let t = TestContext::new().await;
+        let timestamp_now = time();
+        let timestamp_future = timestamp_now + 60 * 60 * 24 * 7;
+        let timestamp_past = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 9, 1),
+            NaiveTime::from_hms(0, 0, 0),
+        )
+        .timestamp_millis()
+            / 1_000;
+
+        // a correct time must not add a device message
+        maybe_warn_on_bad_time(&t.ctx, timestamp_now, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // we cannot find out if a date in the future is wrong - a device message is not added
+        maybe_warn_on_bad_time(&t.ctx, timestamp_future, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // a date in the past must add a device message
+        maybe_warn_on_bad_time(&t.ctx, timestamp_past, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // the message should be added only once a day - test that an hour later and nearly a day later
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60 * 24 - 1,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // next day, there should be another device message
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60 * 24,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        assert_eq!(device_chat_id, chats.get_chat_id(0));
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 2);
+    }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_outdated() {
+        let t = TestContext::new().await;
+        let timestamp_now: i64 = time();
+
+        // in about 6 months, the app should not be outdated
+        // (if this fails, provider-db is not updated since 6 months)
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + 180 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // in 1 year, the app should be considered as outdated
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + 365 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // do not repeat the warning every day ...
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + (365 + 1) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // ... but every month
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + (365 + 31) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 2);
+    }
 }

src/e2ee.rs

@@ -51,23 +51,41 @@ impl EncryptHelper {
     }
 
     /// Determines if we can and should encrypt.
+    ///
+    /// For encryption to be enabled, `e2ee_guaranteed` should be true, or strictly more than a half
+    /// of peerstates should prefer encryption. Own preference is counted equally to peer
+    /// preferences, even if message copy is not sent to self.
+    ///
+    /// `e2ee_guaranteed` should be set to true for replies to encrypted messages (as required by
+    /// Autocrypt Level 1, version 1.1) and for messages sent in verified groups.
+    ///
+    /// Returns an error if `e2ee_guaranteed` is true, but one or more keys are missing.
+    ///
+    /// Always returns `false` if one of the peerstates does not support Autocrypt (is in "reset"
+    /// state) or does not have a known key.
     pub fn should_encrypt(
         &self,
         context: &Context,
         e2ee_guaranteed: bool,
         peerstates: &[(Option<Peerstate>, &str)],
     ) -> Result<bool> {
-        if !(self.prefer_encrypt == EncryptPreference::Mutual || e2ee_guaranteed) {
-            return Ok(false);
-        }
-
+        let mut prefer_encrypt_count = if self.prefer_encrypt == EncryptPreference::Mutual {
+            1
+        } else {
+            0
+        };
         for (peerstate, addr) in peerstates {
             match peerstate {
                 Some(peerstate) => {
-                    if peerstate.prefer_encrypt != EncryptPreference::Mutual && !e2ee_guaranteed {
-                        info!(context, "peerstate for {:?} is no-encrypt", addr);
-                        return Ok(false);
-                    }
+                    info!(
+                        context,
+                        "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
+                    );
+                    match peerstate.prefer_encrypt {
+                        EncryptPreference::NoPreference => {}
+                        EncryptPreference::Mutual => prefer_encrypt_count += 1,
+                        EncryptPreference::Reset => return Ok(false),
+                    };
                 }
                 None => {
                     let msg = format!("peerstate for {:?} missing, cannot encrypt", addr);
@@ -81,7 +99,11 @@ impl EncryptHelper {
             }
         }
 
-        Ok(true)
+        // Count number of recipients, including self.
+        // This does not depend on whether we send a copy to self or not.
+        let recipients_count = peerstates.len() + 1;
+
+        Ok(e2ee_guaranteed || 2 * prefer_encrypt_count > recipients_count)
     }
 
     /// Tries to encrypt the passed in `mail`.
@@ -135,41 +157,31 @@ pub async fn try_decrypt(
         .map(|from| from.addr)
         .unwrap_or_default();
 
-    let mut peerstate = None;
-    let autocryptheader = Aheader::from_headers(context, &from, &mail.headers);
-
-    if message_time > 0 {
-        peerstate = Peerstate::from_addr(context, &from).await?;
+    let mut peerstate = Peerstate::from_addr(context, &from).await?;
 
+    // Apply Autocrypt header
+    if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
         if let Some(ref mut peerstate) = peerstate {
-            if let Some(ref header) = autocryptheader {
-                peerstate.apply_header(&header, message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            } else if message_time > peerstate.last_seen_autocrypt && !contains_report(mail) {
-                peerstate.degrade_encryption(message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            }
-        } else if let Some(ref header) = autocryptheader {
+            peerstate.apply_header(&header, message_time);
+            peerstate.save_to_db(&context.sql, false).await?;
+        } else {
             let p = Peerstate::from_header(context, header, message_time);
             p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
     }
 
-    /* possibly perform decryption */
+    // Possibly perform decryption
     let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
     let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
     let mut signatures = HashSet::default();
 
-    if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-        peerstate = Peerstate::from_addr(&context, &from).await?;
-    }
-    if let Some(peerstate) = peerstate {
+    if let Some(ref mut peerstate) = peerstate {
         peerstate.handle_fingerprint_change(context).await?;
-        if let Some(key) = peerstate.public_key {
-            public_keyring_for_validate.add(key);
-        } else if let Some(key) = peerstate.gossip_key {
-            public_keyring_for_validate.add(key);
+        if let Some(key) = &peerstate.public_key {
+            public_keyring_for_validate.add(key.clone());
+        } else if let Some(key) = &peerstate.gossip_key {
+            public_keyring_for_validate.add(key.clone());
         }
     }
 
@@ -181,6 +193,18 @@ pub async fn try_decrypt(
         &mut signatures,
     )
     .await?;
+
+    if let Some(mut peerstate) = peerstate {
+        // If message is not encrypted and it is not a read receipt, degrade encryption.
+        if out_mail.is_none()
+            && message_time > peerstate.last_seen_autocrypt
+            && !contains_report(mail)
+        {
+            peerstate.degrade_encryption(message_time);
+            peerstate.save_to_db(&context.sql, false).await?;
+        }
+    }
+
     Ok((out_mail, signatures))
 }
 
@@ -319,6 +343,11 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
 mod tests {
     use super::*;
 
+    use crate::chat;
+    use crate::constants::Viewtype;
+    use crate::contact::{Contact, Origin};
+    use crate::message::Message;
+    use crate::param::Param;
     use crate::test_utils::*;
 
     mod ensure_secret_key_exists {
@@ -379,4 +408,106 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         let data = b"blas";
         assert_eq!(has_decrypted_pgp_armor(data), false);
     }
+
+    #[async_std::test]
+    async fn test_encrypted_no_autocrypt() -> crate::error::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let (contact_alice_id, _modified) = Contact::add_or_lookup(
+            &bob.ctx,
+            "Alice",
+            "alice@example.com",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+        let (contact_bob_id, _modified) = Contact::add_or_lookup(
+            &alice.ctx,
+            "Bob",
+            "bob@example.net",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+
+        let chat_alice = chat::create_by_contact_id(&alice.ctx, contact_bob_id).await?;
+        let chat_bob = chat::create_by_contact_id(&bob.ctx, contact_alice_id).await?;
+
+        // Alice sends unencrypted message to Bob
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        // Bob receives unencrypted message from Alice
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+
+        // Parsing a message is enough to update peerstate
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Bob sends encrypted message to Alice
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        chat::send_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        let sent = bob.pop_sent_msg().await;
+
+        // Alice receives encrypted message from Bob
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+
+        let peerstate_bob = Peerstate::from_addr(&alice.ctx, "bob@example.net")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_bob.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Now Alice and Bob have established keys.
+
+        // Alice sends encrypted message without Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::SkipAutocrypt, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Alice sends plaintext message with Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::ForcePlaintext, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Alice sends plaintext message without Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::ForcePlaintext, 1);
+        msg.param.set_int(Param::SkipAutocrypt, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Reset);
+
+        Ok(())
+    }
 }

src/events.rs

@@ -58,12 +58,18 @@ impl EventEmitter {
 
     /// Async recv of an event. Return `None` if the `Sender` has been droped.
     pub async fn recv(&self) -> Option<Event> {
-        // TODO: change once we can use async channels internally.
         self.0.recv().await.ok()
     }
+}
 
-    pub fn try_recv(&self) -> Result<Event, async_std::sync::TryRecvError> {
-        self.0.try_recv()
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
     }
 }
 
@@ -119,10 +125,6 @@ pub enum EventType {
     #[strum(props(id = "105"))]
     ImapMessageMoved(String),
 
-    /// Emitted when an IMAP folder was emptied
-    #[strum(props(id = "106"))]
-    ImapFolderEmptied(String),
-
     /// Emitted when an new file in the $BLOBDIR was created
     #[strum(props(id = "150"))]
     NewBlobFile(String),
@@ -190,6 +192,11 @@ pub enum EventType {
     #[strum(props(id = "2005"))]
     IncomingMsg { chat_id: ChatId, msg_id: MsgId },
 
+    /// Messages were seen or noticed.
+    /// chat id is always set.
+    #[strum(props(id = "2008"))]
+    MsgsNoticed(ChatId),
+
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
     #[strum(props(id = "2010"))]
@@ -237,10 +244,16 @@ pub enum EventType {
     LocationChanged(Option<u32>),
 
     /// Inform about the configuration progress started by configure().
-    ///
-    /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     #[strum(props(id = "2041"))]
-    ConfigureProgress(usize),
+    ConfigureProgress {
+        /// Progress.
+        ///
+        /// 0=error, 1-999=progress in permille, 1000=success and done
+        progress: usize,
+
+        /// Progress comment or error, something to display to the user.
+        comment: Option<String>,
+    },
 
     /// Inform about the import/export progress started by imex().
     ///

src/format_flowed.rs

@@ -21,21 +21,28 @@
 /// length. However, this should be rare and should not result in
 /// immediate mail rejection: SMTP (RFC 2821) limit is 998 characters,
 /// and Spam Assassin limit is 78 characters.
-fn format_line_flowed(line: &str) -> String {
+fn format_line_flowed(line: &str, prefix: &str) -> String {
     let mut result = String::new();
-    let mut buffer = String::new();
+    let mut buffer = prefix.to_string();
     let mut after_space = false;
 
     for c in line.chars() {
         if c == ' ' {
             buffer.push(c);
             after_space = true;
+        } else if c == '>' {
+            if buffer.is_empty() {
+                // Space stuffing, see RFC 3676
+                buffer.push(' ');
+            }
+            buffer.push(c);
+            after_space = false;
         } else {
-            if after_space && buffer.len() >= 72 && !c.is_whitespace() && c != '>' {
+            if after_space && buffer.len() >= 72 && !c.is_whitespace() {
                 // Flush the buffer and insert soft break (SP CRLF).
                 result += &buffer;
                 result += "\r\n";
-                buffer = String::new();
+                buffer = prefix.to_string();
             }
             buffer.push(c);
             after_space = false;
@@ -44,6 +51,28 @@ fn format_line_flowed(line: &str) -> String {
     result + &buffer
 }
 
+fn format_flowed_prefix(text: &str, prefix: &str) -> String {
+    let mut result = String::new();
+
+    for line in text.split('\n') {
+        if !result.is_empty() {
+            result += "\r\n";
+        }
+        let line = line.trim_end();
+        if prefix.len() + line.len() > 78 {
+            result += &format_line_flowed(line, prefix);
+        } else {
+            result += prefix;
+            if prefix.is_empty() && line.starts_with('>') {
+                // Space stuffing, see RFC 3676
+                result.push(' ');
+            }
+            result += line;
+        }
+    }
+    result
+}
+
 /// Returns text formatted according to RFC 3767 (format=flowed).
 ///
 /// This function accepts text separated by LF, but returns text
@@ -52,20 +81,12 @@ fn format_line_flowed(line: &str) -> String {
 /// RFC 2646 technique is used to insert soft line breaks, so DelSp
 /// SHOULD be set to "no" when sending.
 pub fn format_flowed(text: &str) -> String {
-    let mut result = String::new();
+    format_flowed_prefix(text, "")
+}
 
-    for line in text.split('\n') {
-        if !result.is_empty() {
-            result += "\r\n";
-        }
-        let line = line.trim_end();
-        if line.len() > 78 {
-            result += &format_line_flowed(line);
-        } else {
-            result += line;
-        }
-    }
-    result
+/// Same as format_flowed(), but adds "> " prefix to each line.
+pub fn format_flowed_quote(text: &str) -> String {
+    format_flowed_prefix(text, "> ")
 }
 
 /// Joins lines in format=flowed text.
@@ -122,6 +143,9 @@ mod tests {
                         To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
                         client and enter the setup code presented on the generating device.";
         assert_eq!(format_flowed(text), expected);
+
+        let text = "> Not a quote";
+        assert_eq!(format_flowed(text), " > Not a quote");
     }
 
     #[test]
@@ -133,4 +157,21 @@ mod tests {
                         unwrapped on the receiver";
         assert_eq!(unformat_flowed(text, false), expected);
     }
+
+    #[test]
+    fn test_format_flowed_quote() {
+        let quote = "this is a quoted line";
+        let expected = "> this is a quoted line";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "> foo bar baz";
+        let expected = "> > foo bar baz";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "this is a very long quote that should be wrapped using format=flowed and unwrapped on the receiver";
+        let expected =
+            "> this is a very long quote that should be wrapped using format=flowed and \r\n\
+            > unwrapped on the receiver";
+        assert_eq!(format_flowed_quote(quote), expected);
+    }
 }

src/headerdef.rs

@@ -3,7 +3,6 @@ use mailparse::{MailHeader, MailHeaderMap};
 
 #[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
 #[strum(serialize_all = "kebab_case")]
-#[allow(dead_code)]
 pub enum HeaderDef {
     MessageId,
     Subject,

src/imap/idle.rs

@@ -25,7 +25,7 @@ impl Imap {
         if !self.can_idle() {
             bail!("IMAP server does not have IDLE capability");
         }
-        self.setup_handle_if_needed(context).await?;
+        self.setup_handle(context).await?;
 
         self.select_folder(context, watch_folder.clone()).await?;
 
@@ -113,12 +113,13 @@ impl Imap {
         // in this case, we're waiting for a configure job (and an interrupt).
 
         let fake_idle_start_time = SystemTime::now();
-        info!(context, "IMAP-fake-IDLEing...");
 
         // Do not poll, just wait for an interrupt when no folder is passed in.
         if watch_folder.is_none() {
+            info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
             return self.idle_interrupt.recv().await.unwrap_or_default();
         }
+        info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
         // check every minute if there are new messages
         // TODO: grow sleep durations / make them more flexible
@@ -160,7 +161,7 @@ impl Imap {
                     // will not find any new.
 
                     if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_new_messages(context, watch_folder).await {
+                        match self.fetch_new_messages(context, watch_folder, false).await {
                             Ok(res) => {
                                 info!(context, "fetch_new_messages returned {:?}", res);
                                 if res {

src/imap/mod.rs

@@ -3,8 +3,9 @@
 //! uses [async-email/async-imap](https://github.com/async-email/async-imap)
 //! to implement connect, fetch, delete functionality with standard IMAP servers.
 
-use std::collections::BTreeMap;
+use std::{cmp, collections::BTreeMap};
 
+use anyhow::Context as _;
 use async_imap::{
     error::Result as ImapResult,
     types::{Capability, Fetch, Flag, Mailbox, Name, NameAttribute},
@@ -13,12 +14,9 @@ use async_std::prelude::*;
 use async_std::sync::Receiver;
 use num_traits::FromPrimitive;
 
-use crate::config::*;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_receive_imf::{
-    dc_receive_imf, from_field_to_contact_id, is_msgrmsg_rfc724_mid_in_list,
-};
+use crate::dc_receive_imf::{from_field_to_contact_id, is_msgrmsg_rfc724_mid_in_list};
 use crate::error::{bail, format_err, Result};
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
@@ -32,6 +30,7 @@ use crate::provider::{get_provider_info, Socket};
 use crate::{
     chat, dc_tools::dc_extract_grpid_from_rfc724_mid, scheduler::InterruptInfo, stock::StockMessage,
 };
+use crate::{config::*, dc_receive_imf::dc_receive_imf_inner};
 
 mod client;
 mod idle;
@@ -40,6 +39,7 @@ mod session;
 
 use chat::get_chat_id_by_grpid;
 use client::Client;
+use mailparse::SingleInfo;
 use message::Message;
 use session::Session;
 
@@ -68,7 +68,6 @@ const DELETE_CHECK_FLAGS: &str = "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])";
 const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])";
 const JUST_UID: &str = "(UID)";
 const BODY_FLAGS: &str = "(FLAGS BODY.PEEK[])";
-const SELECT_ALL: &str = "1:*";
 
 #[derive(Debug)]
 pub struct Imap {
@@ -162,7 +161,11 @@ impl Imap {
         self.should_reconnect = true;
     }
 
-    async fn setup_handle_if_needed(&mut self, context: &Context) -> Result<()> {
+    /// Connects or reconnects if needed.
+    ///
+    /// It is safe to call this function if already connected, actions
+    /// are performed only as needed.
+    async fn try_setup_handle(&mut self, context: &Context) -> Result<()> {
         if self.config.lp.server.is_empty() {
             bail!("IMAP operation attempted while it is torn down");
         }
@@ -226,21 +229,7 @@ impl Imap {
                 }
             }
             Err(err) => {
-                let message = {
-                    let config = &self.config;
-                    let imap_server: &str = config.lp.server.as_ref();
-                    let imap_port = config.lp.port;
-                    context
-                        .stock_string_repl_str2(
-                            StockMessage::ServerResponse,
-                            format!("IMAP {}:{}", imap_server, imap_port),
-                            err.to_string(),
-                        )
-                        .await
-                };
-                // IMAP connection failures are reported to users
-                emit_event!(context, EventType::ErrorNetwork(message));
-                bail!("IMAP connection failed: {}", err);
+                bail!(err);
             }
         };
 
@@ -262,7 +251,6 @@ impl Imap {
                     .await;
 
                 warn!(context, "{} ({})", message, err);
-                emit_event!(context, EventType::ErrorNetwork(message.clone()));
 
                 let lock = context.wrong_pw_warning_mutex.lock().await;
                 if self.login_failed_once
@@ -274,7 +262,7 @@ impl Imap {
                     drop(lock);
 
                     let mut msg = Message::new(Viewtype::Text);
-                    msg.text = Some(message);
+                    msg.text = Some(message.clone());
                     if let Err(e) =
                         chat::add_device_msg_with_importance(context, None, Some(&mut msg), true)
                             .await
@@ -286,11 +274,24 @@ impl Imap {
                 }
 
                 self.trigger_reconnect();
-                Err(format_err!("IMAP Could not login as {}", imap_user))
+                Err(format_err!("{}\n\n{}", message, err))
             }
         }
     }
 
+    /// Connects or reconnects if not already connected.
+    ///
+    /// This function emits network error if it fails.  It should not
+    /// be used during configuration to avoid showing failed attempt
+    /// errors to the user.
+    async fn setup_handle(&mut self, context: &Context) -> Result<()> {
+        let res = self.try_setup_handle(context).await;
+        if let Err(ref err) = res {
+            emit_event!(context, EventType::ErrorNetwork(err.to_string()));
+        }
+        res
+    }
+
     async fn unsetup_handle(&mut self, context: &Context) {
         // Close folder if messages should be expunged
         if let Err(err) = self.close_folder(context).await {
@@ -318,7 +319,9 @@ impl Imap {
         cfg.can_move = false;
     }
 
-    /// Connects to imap account using already-configured parameters.
+    /// Connects to IMAP account using already-configured parameters.
+    ///
+    /// Emits network error if connection fails.
     pub async fn connect_configured(&mut self, context: &Context) -> Result<()> {
         if self.is_connected() && !self.should_reconnect() {
             return Ok(());
@@ -348,6 +351,9 @@ impl Imap {
     /// Tries connecting to imap account using the specific login parameters.
     ///
     /// `addr` is used to renew token if OAuth2 authentication is used.
+    ///
+    /// Does not emit network errors, can be used to try various
+    /// parameters during autoconfiguration.
     pub async fn connect(
         &mut self,
         context: &Context,
@@ -375,8 +381,8 @@ impl Imap {
             config.oauth2 = oauth2;
         }
 
-        if let Err(err) = self.setup_handle_if_needed(context).await {
-            warn!(context, "failed to setup imap handle: {}", err);
+        if let Err(err) = self.try_setup_handle(context).await {
+            warn!(context, "try_setup_handle: {}", err);
             self.free_connect_params().await;
             return Err(err);
         }
@@ -422,7 +428,10 @@ impl Imap {
         if teardown {
             self.disconnect(context).await;
 
-            bail!("IMAP disconnected immediately after connecting due to error");
+            warn!(
+                context,
+                "IMAP disconnected immediately after connecting due to error"
+            );
         }
         Ok(())
     }
@@ -437,9 +446,12 @@ impl Imap {
             // probably shutdown
             bail!("IMAP operation attempted while it is torn down");
         }
-        self.setup_handle_if_needed(context).await?;
+        self.setup_handle(context).await?;
 
-        while self.fetch_new_messages(context, &watch_folder).await? {
+        while self
+            .fetch_new_messages(context, &watch_folder, false)
+            .await?
+        {
             // We fetch until no more new messages are there.
         }
         Ok(())
@@ -455,16 +467,8 @@ impl Imap {
             // the entry has the format `imap.mailbox.<folder>=<uidvalidity>:<lastseenuid>`
             let mut parts = entry.split(':');
             (
-                parts
-                    .next()
-                    .unwrap_or_default()
-                    .parse()
-                    .unwrap_or_else(|_| 0),
-                parts
-                    .next()
-                    .unwrap_or_default()
-                    .parse()
-                    .unwrap_or_else(|_| 0),
+                parts.next().unwrap_or_default().parse().unwrap_or(0),
+                parts.next().unwrap_or_default().parse().unwrap_or(0),
             )
         } else {
             (0, 0)
@@ -642,10 +646,11 @@ impl Imap {
         Ok((new_uid_validity, new_last_seen_uid))
     }
 
-    async fn fetch_new_messages<S: AsRef<str>>(
+    pub(crate) async fn fetch_new_messages<S: AsRef<str>>(
         &mut self,
         context: &Context,
         folder: S,
+        fetch_existing_msgs: bool,
     ) -> Result<bool> {
         let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await)
             .unwrap_or_default();
@@ -654,7 +659,11 @@ impl Imap {
             .select_with_uidvalidity(context, folder.as_ref())
             .await?;
 
-        let msgs = self.fetch_after(context, last_seen_uid).await?;
+        let msgs = if fetch_existing_msgs {
+            self.fetch_existing_msgs_prefetch().await?
+        } else {
+            self.fetch_after(context, last_seen_uid).await?
+        };
         let read_cnt = msgs.len();
         let folder: &str = folder.as_ref();
 
@@ -694,8 +703,9 @@ impl Imap {
         }
 
         // check passed, go fetch the emails
-        let (new_last_seen_uid_processed, error_cnt) =
-            self.fetch_many_msgs(context, &folder, &uids).await;
+        let (new_last_seen_uid_processed, error_cnt) = self
+            .fetch_many_msgs(context, &folder, &uids, fetch_existing_msgs)
+            .await;
         read_errors += error_cnt;
 
         // determine which last_seen_uid to use to update  to
@@ -720,17 +730,66 @@ impl Imap {
         Ok(read_cnt > 0)
     }
 
+    /// Gets the from, to and bcc addresses from all existing outgoing emails.
+    pub async fn get_all_recipients(&mut self, context: &Context) -> Result<Vec<SingleInfo>> {
+        if self.session.is_none() {
+            bail!("IMAP No Connection established");
+        }
+
+        let session = self.session.as_mut().unwrap();
+        let self_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await
+            .ok_or_else(|| format_err!("Not configured"))?;
+
+        let search_command = format!("FROM \"{}\"", self_addr);
+        let uids = session.uid_search(search_command).await?;
+        let uid_strings: Vec<String> = uids.into_iter().map(|s| s.to_string()).collect();
+
+        let mut result = Vec::new();
+        // We fetch the emails in chunks of 100 because according to https://tools.ietf.org/html/rfc2683#section-3.2.1.5
+        // command lines should not be much more than 1000 chars and UIDs can get up to 9- or 10-digit
+        // (servers should allow at least 8000 chars)
+        for uid_chunk in uid_strings.chunks(100) {
+            let uid_set = uid_chunk.join(",");
+
+            let mut list = session
+                .uid_fetch(uid_set, "(UID BODY.PEEK[HEADER.FIELDS (FROM TO CC BCC)])")
+                .await
+                .map_err(|err| {
+                    format_err!("IMAP Could not fetch (get_all_recipients()): {}", err)
+                })?;
+
+            while let Some(fetch) = list.next().await {
+                let msg = fetch?;
+                match get_fetch_headers(&msg) {
+                    Ok(headers) => {
+                        let (from_id, _, _) =
+                            from_field_to_contact_id(context, &mimeparser::get_from(&headers))
+                                .await?;
+                        if from_id == DC_CONTACT_ID_SELF {
+                            result.extend(mimeparser::get_recipients(&headers));
+                        }
+                    }
+
+                    Err(err) => {
+                        warn!(context, "{}", err);
+                        continue;
+                    }
+                };
+            }
+        }
+        Ok(result)
+    }
+
     /// Fetch all uids larger than the passed in. Returns a sorted list of fetch results.
     async fn fetch_after(
         &mut self,
         context: &Context,
         uid: u32,
     ) -> Result<BTreeMap<u32, async_imap::types::Fetch>> {
-        if self.session.is_none() {
-            bail!("IMAP No Connection established");
-        }
-
-        let session = self.session.as_mut().unwrap();
+        let session = self.session.as_mut();
+        let session = session.context("fetch_after(): IMAP No Connection established")?;
 
         // fetch messages with larger UID than the last one seen
         // `(UID FETCH lastseenuid+1:*)`, see RFC 4549
@@ -768,6 +827,40 @@ impl Imap {
         Ok(new_msgs)
     }
 
+    /// Like fetch_after(), but not for new messages but existing ones (the DC_FETCH_EXISTING_MSGS_COUNT newest messages)
+    async fn fetch_existing_msgs_prefetch(
+        &mut self,
+    ) -> Result<BTreeMap<u32, async_imap::types::Fetch>> {
+        let exists: i64 = {
+            let mailbox = self.config.selected_mailbox.as_ref();
+            let mailbox = mailbox.context("fetch_existing_msgs_prefetch(): no mailbox selected")?;
+            mailbox.exists.into()
+        };
+        let session = self.session.as_mut();
+        let session =
+            session.context("fetch_existing_msgs_prefetch(): IMAP No Connection established")?;
+
+        // Fetch last DC_FETCH_EXISTING_MSGS_COUNT (100) messages.
+        // Sequence numbers are sequential. If there are 1000 messages in the inbox,
+        // we can fetch the sequence numbers 900-1000 and get the last 100 messages.
+        let first = cmp::max(1, exists - DC_FETCH_EXISTING_MSGS_COUNT);
+        let set = format!("{}:*", first);
+        let mut list = session
+            .fetch(&set, PREFETCH_FLAGS)
+            .await
+            .map_err(|err| format_err!("IMAP Could not fetch: {}", err))?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(fetch) = list.next().await {
+            let msg = fetch?;
+            if let Some(msg_uid) = msg.uid {
+                msgs.insert(msg_uid, msg);
+            }
+        }
+
+        Ok(msgs)
+    }
+
     async fn set_config_last_seen_uid<S: AsRef<str>>(
         &self,
         context: &Context,
@@ -794,6 +887,7 @@ impl Imap {
         context: &Context,
         folder: S,
         server_uids: &[u32],
+        fetching_existing_messages: bool,
     ) -> (Option<u32>, usize) {
         let set = match server_uids {
             [] => return (None, 0),
@@ -867,7 +961,16 @@ impl Imap {
             let body = msg.body().unwrap();
             let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
-            match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
+            match dc_receive_imf_inner(
+                &context,
+                &body,
+                &folder,
+                server_uid,
+                is_seen,
+                fetching_existing_messages,
+            )
+            .await
+            {
                 Ok(_) => last_uid = Some(server_uid),
                 Err(err) => {
                     warn!(context, "dc_receive_imf error: {}", err);
@@ -1245,7 +1348,9 @@ impl Imap {
                 } else if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
                     // Always takes precedent
                     sentbox_folder = Some(folder.name().to_string());
-                } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
+                } else if let FolderMeaning::SentObjects =
+                    get_folder_meaning_by_name(&folder.name())
+                {
                     // only set iff none has been already set
                     if sentbox_folder.is_none() {
                         sentbox_folder = Some(folder.name().to_string());
@@ -1316,60 +1421,6 @@ impl Imap {
         info!(context, "FINISHED configuring IMAP-folders.");
         Ok(())
     }
-
-    pub async fn empty_folder(&mut self, context: &Context, folder: &str) {
-        info!(context, "emptying folder {}", folder);
-
-        // we want to report all error to the user
-        // (no retry should be attempted)
-        if folder.is_empty() {
-            error!(context, "cannot perform empty, folder not set");
-            return;
-        }
-        if let Err(err) = self.setup_handle_if_needed(context).await {
-            error!(context, "could not setup imap connection: {:?}", err);
-            return;
-        }
-        if let Err(err) = self.select_folder(context, Some(&folder)).await {
-            error!(
-                context,
-                "Could not select {} for expunging: {:?}", folder, err
-            );
-            return;
-        }
-
-        if !self
-            .add_flag_finalized_with_set(context, SELECT_ALL, "\\Deleted")
-            .await
-        {
-            error!(context, "Cannot mark messages for deletion {}", folder);
-            return;
-        }
-
-        // we now trigger expunge to actually delete messages
-        self.config.selected_folder_needs_expunge = true;
-        match self.select_folder::<String>(context, None).await {
-            Ok(()) => {
-                emit_event!(context, EventType::ImapFolderEmptied(folder.to_string()));
-            }
-            Err(err) => {
-                error!(context, "expunge failed {}: {:?}", folder, err);
-            }
-        }
-        if let Err(err) = context
-            .sql
-            .execute(
-                "UPDATE msgs SET server_folder='',server_uid=0 WHERE server_folder=?",
-                paramsv![folder],
-            )
-            .await
-        {
-            warn!(
-                context,
-                "Failed to reset server_uid and server_folder for deleted messages: {}", err
-            );
-        }
-    }
 }
 
 /// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.
@@ -1378,11 +1429,43 @@ impl Imap {
 // only watching this folder is not working. at least, this is no show stopper.
 // CAVE: if possible, take care not to add a name here that is "sent" in one language
 // but sth. different in others - a hard job.
-fn get_folder_meaning_by_name(folder_name: &Name) -> FolderMeaning {
-    let sent_names = vec!["sent", "sentmail", "sent objects", "gesendet"];
-    let lower = folder_name.name().to_lowercase();
+fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
+    // source: https://stackoverflow.com/questions/2185391/localized-gmail-imap-folders
+    let sent_names = vec![
+        "sent",
+        "sentmail",
+        "sent objects",
+        "gesendet",
+        "Sent Mail",
+        "Sendte e-mails",
+        "Enviados",
+        "Messages envoyés",
+        "Messages envoyes",
+        "Posta inviata",
+        "Verzonden berichten",
+        "Wyslane",
+        "E-mails enviados",
+        "Correio enviado",
+        "Enviada",
+        "Enviado",
+        "Gönderildi",
+        "Inviati",
+        "Odeslaná pošta",
+        "Sendt",
+        "Skickat",
+        "Verzonden",
+        "Wysłane",
+        "Éléments envoyés",
+        "Απεσταλμένα",
+        "Отправленные",
+        "寄件備份",
+        "已发送邮件",
+        "送信済み",
+        "보낸편지함",
+    ];
+    let lower = folder_name.to_lowercase();
 
-    if sent_names.into_iter().any(|s| s == lower) {
+    if sent_names.into_iter().any(|s| s.to_lowercase() == lower) {
         FolderMeaning::SentObjects
     } else {
         FolderMeaning::Unknown
@@ -1476,17 +1559,16 @@ async fn precheck_imf(
 
         if old_server_folder != server_folder || old_server_uid != server_uid {
             update_server_uid(context, rfc724_mid, server_folder, server_uid).await;
-            if let Ok(MessageState::InSeen) = msg_id.get_state(context).await {
-                job::add(
-                    context,
-                    job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
-                )
-                .await;
-            };
-            context
-                .interrupt_inbox(InterruptInfo::new(false, Some(msg_id)))
-                .await;
-            info!(context, "Updating server_uid and interrupting")
+            if let Ok(message_state) = msg_id.get_state(context).await {
+                if message_state == MessageState::InSeen || message_state.is_outgoing() {
+                    job::add(
+                        context,
+                        job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+                    )
+                    .await;
+                }
+            }
+            info!(context, "Updating server_uid and adding markseen job");
         }
         Ok(true)
     } else {
@@ -1628,3 +1710,32 @@ async fn message_needs_processing(
 fn get_fallback_folder(delimiter: &str) -> String {
     format!("INBOX{}DeltaChat", delimiter)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_get_folder_meaning_by_name() {
+        assert_eq!(
+            get_folder_meaning_by_name("Gesendet"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(
+            get_folder_meaning_by_name("GESENDET"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(
+            get_folder_meaning_by_name("gesendet"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(
+            get_folder_meaning_by_name("Messages envoyés"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(
+            get_folder_meaning_by_name("mEsSaGes envoyÉs"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(get_folder_meaning_by_name("xxx"), FolderMeaning::Unknown);
+    }
+}

src/imex.rs

@@ -6,6 +6,7 @@ use std::{
     ffi::OsStr,
 };
 
+use anyhow::Context as _;
 use async_std::path::{Path, PathBuf};
 use async_std::{
     fs::{self, File},
@@ -94,7 +95,8 @@ pub async fn imex(
             }
             Err(err) => {
                 cleanup_aborted_imex(context, what).await;
-                error!(context, "{}", err);
+                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                error!(context, "{:#}", err);
                 context.emit_event(EventType::ImexProgress(0));
                 bail!("IMEX FAILED to complete: {}", err);
             }
@@ -118,7 +120,9 @@ async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
         dc_delete_files_in_dir(context, context.get_blobdir()).await;
     }
     if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
-        context.sql.open(context, context.get_dbfile(), false).await;
+        if let Err(e) = context.sql.open(context, context.get_dbfile(), false).await {
+            warn!(context, "Re-opening db after imex failed: {}", e);
+        }
     }
 }
 
@@ -166,17 +170,23 @@ pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Re
             let name = name.to_string_lossy();
             if name.starts_with("delta-chat") && name.ends_with(".bak") {
                 let sql = Sql::new();
-                if sql.open(context, &path, true).await {
-                    let curr_backup_time = sql
-                        .get_raw_config_int(context, "backup_time")
-                        .await
-                        .unwrap_or_default();
-                    if curr_backup_time > newest_backup_time {
-                        newest_backup_path = Some(path);
-                        newest_backup_time = curr_backup_time;
+                match sql.open(context, &path, true).await {
+                    Ok(_) => {
+                        let curr_backup_time = sql
+                            .get_raw_config_int(context, "backup_time")
+                            .await
+                            .unwrap_or_default();
+                        if curr_backup_time > newest_backup_time {
+                            newest_backup_path = Some(path);
+                            newest_backup_time = curr_backup_time;
+                        }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close().await;
                     }
-                    info!(context, "backup_time of {} is {}", name, curr_backup_time);
-                    sql.close().await;
+                    Err(e) => warn!(
+                        context,
+                        "Found backup file {} which could not be opened: {}", name, e
+                    ),
                 }
             }
         }
@@ -220,10 +230,8 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.param.set_int(
-        Param::ForcePlaintext,
-        ForcePlaintext::NoAutocryptHeader as i32,
-    );
+    msg.param.set_int(Param::ForcePlaintext, 1);
+    msg.param.set_int(Param::SkipAutocrypt, 1);
 
     let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
     info!(context, "Wait for setup message being sent ...",);
@@ -522,13 +530,11 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         }
     }
 
-    ensure!(
-        context
-            .sql
-            .open(&context, &context.get_dbfile(), false)
-            .await,
-        "could not re-open db"
-    );
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
 
     delete_and_reset_all_device_msgs(&context).await?;
 
@@ -560,13 +566,11 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
     );
     /* error already logged */
     /* re-open copied database file */
-    ensure!(
-        context
-            .sql
-            .open(&context, &context.get_dbfile(), false)
-            .await,
-        "could not re-open db"
-    );
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
 
     delete_and_reset_all_device_msgs(&context).await?;
 
@@ -745,7 +749,7 @@ async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<(
     context
         .sql
         .open(&context, &context.get_dbfile(), false)
-        .await;
+        .await?;
 
     if !copied {
         bail!(
@@ -755,11 +759,11 @@ async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<(
         );
     }
     let dest_sql = Sql::new();
-    ensure!(
-        dest_sql.open(context, &dest_path_filename, false).await,
-        "could not open exported database {}",
-        dest_path_string
-    );
+    dest_sql
+        .open(context, &dest_path_filename, false)
+        .await
+        .with_context(|| format!("could not open exported database {}", dest_path_string))?;
+
     let res = match add_files_to_export(context, &dest_sql).await {
         Err(err) => {
             dc_delete_file(context, &dest_path_filename).await;

src/job.rs

@@ -14,10 +14,8 @@ use async_smtp::smtp::response::Category;
 use async_smtp::smtp::response::Code;
 use async_smtp::smtp::response::Detail;
 
-use crate::blob::BlobObject;
 use crate::chat::{self, ChatId};
 use crate::config::Config;
-use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::*;
@@ -31,6 +29,7 @@ use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
+use crate::{blob::BlobObject, contact::normalize_name, contact::Modifier, contact::Origin};
 use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
@@ -93,7 +92,7 @@ pub enum Action {
 
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
-    EmptyServer = 107,
+    FetchExistingMsgs = 110,
     MarkseenMsgOnImap = 130,
 
     // Moving message is prioritized lower than deletion so we don't
@@ -126,9 +125,9 @@ impl From<Action> for Thread {
             Unknown => Thread::Unknown,
 
             Housekeeping => Thread::Imap,
+            FetchExistingMsgs => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
             ResyncFolders => Thread::Imap,
-            EmptyServer => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
 
@@ -622,6 +621,38 @@ impl Job {
         }
     }
 
+    /// Read the recipients from old emails sent by the user and add them as contacts.
+    /// This way, we can already offer them some email addresses they can write to.
+    ///
+    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
+    /// and show them in the chat list.
+    async fn fetch_existing_msgs(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredSentboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredMvboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredInboxFolder).await;
+
+        for config in &[
+            Config::ConfiguredMvboxFolder,
+            Config::ConfiguredInboxFolder,
+            Config::ConfiguredSentboxFolder,
+        ] {
+            if let Some(folder) = context.get_config(*config).await {
+                if let Err(e) = imap.fetch_new_messages(context, folder, true).await {
+                    // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                    warn!(context, "Could not fetch messages, retrying: {:#}", e);
+                    return Status::RetryLater;
+                };
+            }
+        }
+        info!(context, "Done fetching existing messages.");
+        Status::Finished(Ok(()))
+    }
+
     /// Synchronizes UIDs for sentbox, inbox and mvbox, in this order.
     ///
     /// If a copy of the message is present in multiple folders, mvbox
@@ -660,25 +691,6 @@ impl Job {
         Status::Finished(Ok(()))
     }
 
-    async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.connect_configured(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
-        if self.foreign_id & DC_EMPTY_MVBOX > 0 {
-            if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
-                imap.empty_folder(context, &mvbox_folder).await;
-            }
-        }
-        if self.foreign_id & DC_EMPTY_INBOX > 0 {
-            if let Some(inbox_folder) = &context.get_config(Config::ConfiguredInboxFolder).await {
-                imap.empty_folder(context, &inbox_folder).await;
-            }
-        }
-        Status::Finished(Ok(()))
-    }
-
     async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.connect_configured(context).await {
             warn!(context, "could not connect: {:?}", err);
@@ -781,6 +793,50 @@ async fn set_delivered(context: &Context, msg_id: MsgId) {
     context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
 }
 
+async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
+    let mailbox = if let Some(m) = context.get_config(folder).await {
+        m
+    } else {
+        return;
+    };
+    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
+        warn!(context, "Could not select {}: {}", mailbox, e);
+        return;
+    }
+    match imap.get_all_recipients(context).await {
+        Ok(contacts) => {
+            let mut any_modified = false;
+            for contact in contacts {
+                let display_name_normalized = contact
+                    .display_name
+                    .as_ref()
+                    .map(normalize_name)
+                    .unwrap_or_default();
+
+                match Contact::add_or_lookup(
+                    context,
+                    display_name_normalized,
+                    contact.addr,
+                    Origin::OutgoingTo,
+                )
+                .await
+                {
+                    Ok((_, modified)) => {
+                        if modified != Modifier::None {
+                            any_modified = true;
+                        }
+                    }
+                    Err(e) => warn!(context, "Could not add recipient: {}", e),
+                }
+            }
+            if any_modified {
+                context.emit_event(EventType::ContactsChanged(None));
+            }
+        }
+        Err(e) => warn!(context, "Could not add recipients: {}", e),
+    };
+}
+
 /// Constructs a job for sending a message.
 ///
 /// Returns `None` if no messages need to be sent out.
@@ -1025,11 +1081,11 @@ async fn perform_job_action(
         Action::MaybeSendLocationsEnded => {
             location::job_maybe_send_locations_ended(context, job).await
         }
-        Action::EmptyServer => job.empty_server(context, connection.inbox()).await,
         Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
+        Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::Housekeeping => {
             sql::housekeeping(context).await;
             Status::Finished(Ok(()))
@@ -1092,10 +1148,10 @@ pub async fn add(context: &Context, job: Job) {
         match action {
             Action::Unknown => unreachable!(),
             Action::Housekeeping
-            | Action::EmptyServer
             | Action::DeleteMsgOnImap
             | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
+            | Action::FetchExistingMsgs
             | Action::MoveMsg => {
                 info!(context, "interrupt: imap");
                 context

src/key.rs

@@ -222,7 +222,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     let addr = context
         .get_config(Config::ConfiguredAddr)
         .await
-        .ok_or_else(|| Error::NoConfiguredAddr)?;
+        .ok_or(Error::NoConfiguredAddr)?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -355,7 +355,7 @@ pub async fn store_self_keypair(
 }
 
 /// A key fingerprint
-#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+#[derive(Clone, Eq, PartialEq, Hash)]
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {
@@ -375,6 +375,14 @@ impl Fingerprint {
     }
 }
 
+impl fmt::Debug for Fingerprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Fingerprint")
+            .field("hex", &self.hex())
+            .finish()
+    }
+}
+
 /// Make a human-readable fingerprint.
 impl fmt::Display for Fingerprint {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -529,11 +537,11 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[test]
     fn test_from_slice_bad_data() {
         let mut bad_data: [u8; 4096] = [0; 4096];
-        for i in 0..4096 {
-            bad_data[i] = (i & 0xff) as u8;
+        for (i, v) in bad_data.iter_mut().enumerate() {
+            *v = (i & 0xff) as u8;
         }
         for j in 0..(4096 / 40) {
-            let slice = &bad_data[j..j + 4096 / 2 + j];
+            let slice = &bad_data.get(j..j + 4096 / 2 + j).unwrap();
             assert!(SignedPublicKey::from_slice(slice).is_err());
             assert!(SignedSecretKey::from_slice(slice).is_err());
         }
@@ -593,7 +601,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         let ctx0 = ctx.clone();
         let thr0 =
             thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
-        let ctx1 = ctx.clone();
+        let ctx1 = ctx;
         let thr1 =
             thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
         let res0 = thr0.join().unwrap();

src/location.rs

@@ -723,6 +723,8 @@ pub(crate) async fn job_maybe_send_locations_ended(
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use crate::test_utils::TestContext;
 
@@ -772,7 +774,7 @@ mod tests {
         assert!(locations_ref[0].latitude < 51.423724f64);
         assert!(locations_ref[0].longitude >= 8.552556f64);
         assert!(locations_ref[0].longitude < 8.552557f64);
-        assert_eq!(locations_ref[0].accuracy, 0.0f64);
+        assert!(locations_ref[0].accuracy.abs() < f64::EPSILON);
         assert_eq!(locations_ref[0].timestamp, timestamp);
     }
 }

src/message.rs

@@ -2,7 +2,7 @@
 
 use async_std::path::{Path, PathBuf};
 use deltachat_derive::{FromSql, ToSql};
-use lazy_static::lazy_static;
+use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
@@ -19,10 +19,7 @@ use crate::mimeparser::{FailureReport, SystemMessage};
 use crate::param::*;
 use crate::pgp::*;
 use crate::stock::StockMessage;
-
-lazy_static! {
-    static ref UNWRAP_RE: regex::Regex = regex::Regex::new(r"\s+").unwrap();
-}
+use std::collections::BTreeMap;
 
 // In practice, the user additionally cuts the string themselves
 // pixel-accurate.
@@ -266,10 +263,9 @@ pub struct Message {
     pub(crate) server_folder: Option<String>,
     pub(crate) server_uid: u32,
     pub(crate) is_dc_message: MessengerMessage,
-    pub(crate) starred: bool,
     pub(crate) chat_blocked: Blocked,
     pub(crate) location_id: u32,
-    pub(crate) error: String,
+    error: Option<String>,
     pub(crate) param: Params,
 }
 
@@ -310,7 +306,6 @@ impl Message {
                     "    m.msgrmsg AS msgrmsg,",
                     "    m.txt AS txt,",
                     "    m.param AS param,",
-                    "    m.starred AS starred,",
                     "    m.hidden AS hidden,",
                     "    m.location_id AS location,",
                     "    c.blocked AS blocked",
@@ -336,7 +331,8 @@ impl Message {
                     msg.ephemeral_timestamp = row.get("ephemeral_timestamp")?;
                     msg.viewtype = row.get("type")?;
                     msg.state = row.get("state")?;
-                    msg.error = row.get("error")?;
+                    let error: String = row.get("error")?;
+                    msg.error = Some(error).filter(|error| !error.is_empty());
                     msg.is_dc_message = row.get("msgrmsg")?;
 
                     let text;
@@ -360,7 +356,6 @@ impl Message {
                     msg.text = Some(text);
 
                     msg.param = row.get::<_, String>("param")?.parse().unwrap_or_default();
-                    msg.starred = row.get("starred")?;
                     msg.hidden = row.get("hidden")?;
                     msg.location_id = row.get("location")?;
                     msg.chat_blocked = row
@@ -494,7 +489,7 @@ impl Message {
     pub fn get_text(&self) -> Option<String> {
         self.text
             .as_ref()
-            .map(|text| dc_truncate(text, 30000).to_string())
+            .map(|text| dc_truncate(text, DC_MAX_GET_TEXT_LEN).to_string())
     }
 
     pub fn get_filename(&self) -> Option<String> {
@@ -585,10 +580,6 @@ impl Message {
         self.state as i32 >= MessageState::OutDelivered as i32
     }
 
-    pub fn is_starred(&self) -> bool {
-        self.starred
-    }
-
     pub fn is_forwarded(&self) -> bool {
         0 != self.param.get_int(Param::Forwarded).unwrap_or_default()
     }
@@ -750,6 +741,61 @@ impl Message {
         self.update_param(context).await;
     }
 
+    /// Sets message quote.
+    ///
+    /// Message-Id is used to set Reply-To field, message text is used for quote.
+    ///
+    /// Encryption is required if quoted message was encrypted.
+    ///
+    /// The message itself is not required to exist in the database,
+    /// it may even be deleted from the database by the time the message is prepared.
+    pub async fn set_quote(&mut self, context: &Context, quote: &Message) -> Result<(), Error> {
+        ensure!(
+            !quote.rfc724_mid.is_empty(),
+            "Message without Message-Id cannot be quoted"
+        );
+        self.in_reply_to = Some(quote.rfc724_mid.clone());
+
+        if quote
+            .param
+            .get_bool(Param::GuaranteeE2ee)
+            .unwrap_or_default()
+        {
+            self.param.set(Param::GuaranteeE2ee, "1");
+        }
+
+        let text = quote.get_text().unwrap_or_default();
+        self.param.set(
+            Param::Quote,
+            if text.is_empty() {
+                // Use summary, similar to "Image" to avoid sending empty quote.
+                quote.get_summarytext(context, 500).await
+            } else {
+                text
+            },
+        );
+
+        Ok(())
+    }
+
+    pub fn quoted_text(&self) -> Option<String> {
+        self.param.get(Param::Quote).map(|s| s.to_string())
+    }
+
+    pub async fn quoted_message(&self, context: &Context) -> Result<Option<Message>, Error> {
+        if self.param.get(Param::Quote).is_some() {
+            if let Some(in_reply_to) = &self.in_reply_to {
+                let rfc724_mid = in_reply_to.trim_start_matches('<').trim_end_matches('>');
+                if !rfc724_mid.is_empty() {
+                    if let Some((_, _, msg_id)) = rfc724_mid_exists(context, rfc724_mid).await? {
+                        return Ok(Some(Message::load_from_db(context, msg_id).await?));
+                    }
+                }
+            }
+        }
+        Ok(None)
+    }
+
     pub async fn update_param(&mut self, context: &Context) -> bool {
         context
             .sql
@@ -760,6 +806,22 @@ impl Message {
             .await
             .is_ok()
     }
+
+    /// Gets the error status of the message.
+    ///
+    /// A message can have an associated error status if something went wrong when sending or
+    /// receiving message itself.  The error status is free-form text and should not be further parsed,
+    /// rather it's presence is meant to indicate *something* went wrong with the message and the
+    /// text of the error is detailed information on what.
+    ///
+    /// Some common reasons error can be associated with messages are:
+    /// * Lack of valid signature on an e2ee message, usually for received messages.
+    /// * Failure to decrypt an e2ee message, usually for received messages.
+    /// * When a message could not be delivered to one or more recipients the non-delivery
+    ///    notification text can be stored in the error status.
+    pub fn error(&self) -> Option<String> {
+        self.error.clone()
+    }
 }
 
 #[derive(
@@ -866,13 +928,18 @@ impl From<MessageState> for LotState {
 
 impl MessageState {
     pub fn can_fail(self) -> bool {
-        match self {
-            MessageState::OutPreparing
-            | MessageState::OutPending
-            | MessageState::OutDelivered
-            | MessageState::OutMdnRcvd => true, // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
-            _ => false,
-        }
+        use MessageState::*;
+        matches!(
+            self,
+            OutPreparing | OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
+        )
+    }
+    pub fn is_outgoing(self) -> bool {
+        use MessageState::*;
+        matches!(
+            self,
+            OutPreparing | OutDraft | OutPending | OutFailed | OutDelivered | OutMdnRcvd
+        )
     }
 }
 
@@ -969,7 +1036,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         return ret;
     }
     let rawtxt = rawtxt.unwrap_or_default();
-    let rawtxt = dc_truncate(rawtxt.trim(), 100_000);
+    let rawtxt = dc_truncate(rawtxt.trim(), DC_MAX_GET_INFO_LEN);
 
     let fts = dc_timestamp_to_str(msg.get_timestamp());
     ret += &format!("Sent: {}", fts);
@@ -1054,8 +1121,8 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
 
     ret += "\n";
 
-    if !msg.error.is_empty() {
-        ret += &format!("Error: {}", msg.error);
+    if let Some(error) = msg.error.as_ref() {
+        ret += &format!("Error: {}", error);
     }
 
     if let Some(path) = msg.get_file(context) {
@@ -1230,6 +1297,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
         .with_conn(move |conn| {
             let mut stmt = conn.prepare_cached(concat!(
                 "SELECT",
+                "    m.chat_id AS chat_id,",
                 "    m.state AS state,",
                 "    c.blocked AS blocked",
                 " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
@@ -1240,6 +1308,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
             for id in msg_ids.into_iter() {
                 let query_res = stmt.query_row(paramsv![id], |row| {
                     Ok((
+                        row.get::<_, ChatId>("chat_id")?,
                         row.get::<_, MessageState>("state")?,
                         row.get::<_, Option<Blocked>>("blocked")?
                             .unwrap_or_default(),
@@ -1248,8 +1317,8 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
                 if let Err(rusqlite::Error::QueryReturnedNoRows) = query_res {
                     continue;
                 }
-                let (state, blocked) = query_res.map_err(Into::<anyhow::Error>::into)?;
-                msgs.push((id, state, blocked));
+                let (chat_id, state, blocked) = query_res.map_err(Into::<anyhow::Error>::into)?;
+                msgs.push((id, chat_id, state, blocked));
             }
 
             Ok(msgs)
@@ -1257,9 +1326,9 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
         .await
         .unwrap_or_default();
 
-    let mut send_event = false;
+    let mut updated_chat_ids = BTreeMap::new();
 
-    for (id, curr_state, curr_blocked) in msgs.into_iter() {
+    for (id, curr_chat_id, curr_state, curr_blocked) in msgs.into_iter() {
         if let Err(err) = id.start_ephemeral_timer(context).await {
             error!(
                 context,
@@ -1278,19 +1347,16 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
                     job::Job::new(Action::MarkseenMsgOnImap, id.to_u32(), Params::new(), 0),
                 )
                 .await;
-                send_event = true;
+                updated_chat_ids.insert(curr_chat_id, true);
             }
         } else if curr_state == MessageState::InFresh {
             update_msg_state(context, id, MessageState::InNoticed).await;
-            send_event = true;
+            updated_chat_ids.insert(ChatId::new(DC_CHAT_ID_DEADDROP), true);
         }
     }
 
-    if send_event {
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
+    for updated_chat_id in updated_chat_ids.keys() {
+        context.emit_event(EventType::MsgsNoticed(*updated_chat_id));
     }
 
     true
@@ -1307,23 +1373,6 @@ pub async fn update_msg_state(context: &Context, msg_id: MsgId, state: MessageSt
         .is_ok()
 }
 
-pub async fn star_msgs(context: &Context, msg_ids: Vec<MsgId>, star: bool) -> bool {
-    if msg_ids.is_empty() {
-        return false;
-    }
-    context
-        .sql
-        .with_conn(move |conn| {
-            let mut stmt = conn.prepare("UPDATE msgs SET starred=? WHERE id=?;")?;
-            for msg_id in msg_ids.into_iter() {
-                stmt.execute(paramsv![star as i32, msg_id])?;
-            }
-            Ok(())
-        })
-        .await
-        .is_ok()
-}
-
 /// Returns a summary text.
 pub async fn get_summarytext_by_raw(
     viewtype: Viewtype,
@@ -1402,7 +1451,7 @@ pub async fn get_summarytext_by_raw(
         prefix
     };
 
-    UNWRAP_RE.replace_all(&summary, " ").to_string()
+    summary.split_whitespace().join(" ")
 }
 
 // as we do not cut inside words, this results in about 32-42 characters.
@@ -1797,16 +1846,6 @@ pub async fn update_server_uid(
     }
 }
 
-#[allow(dead_code)]
-pub async fn dc_empty_server(context: &Context, flags: u32) {
-    job::kill_action(context, Action::EmptyServer).await;
-    job::add(
-        context,
-        job::Job::new(Action::EmptyServer, flags, Params::new(), 0),
-    )
-    .await;
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1847,12 +1886,29 @@ mod tests {
         assert_eq!(_msg2.get_filemime(), None);
     }
 
+    /// Tests that message cannot be prepared if account has no configured address.
+    #[async_std::test]
+    async fn test_prepare_not_configured() {
+        let d = test::TestContext::new().await;
+        let ctx = &d.ctx;
+
+        let contact = Contact::create(ctx, "", "dest@example.com")
+            .await
+            .expect("failed to create contact");
+
+        let chat = chat::create_by_contact_id(ctx, contact).await.unwrap();
+
+        let mut msg = Message::new(Viewtype::Text);
+
+        assert!(chat::prepare_msg(ctx, chat, &mut msg).await.is_err());
+    }
+
     #[async_std::test]
     async fn test_get_summarytext_by_raw() {
         let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
-        let some_text = Some("bla bla".to_string());
+        let some_text = Some(" bla \t\n\tbla\n\t".to_string());
         let empty_text = Some("".to_string());
         let no_text: Option<String> = None;
 
@@ -1893,8 +1949,7 @@ mod tests {
         );
 
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::Voice, no_text.as_ref(), &mut some_file, 50, &ctx)
-                .await,
+            get_summarytext_by_raw(Viewtype::Voice, no_text.as_ref(), &some_file, 50, &ctx).await,
             "Voice message" // file names are not added for voice messages
         );
 
@@ -1904,8 +1959,7 @@ mod tests {
         );
 
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::Audio, no_text.as_ref(), &mut some_file, 50, &ctx)
-                .await,
+            get_summarytext_by_raw(Viewtype::Audio, no_text.as_ref(), &some_file, 50, &ctx).await,
             "Audio \u{2013} foo.bar" // file name is added for audio
         );
 
@@ -1921,8 +1975,7 @@ mod tests {
         );
 
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::File, some_text.as_ref(), &mut some_file, 50, &ctx)
-                .await,
+            get_summarytext_by_raw(Viewtype::File, some_text.as_ref(), &some_file, 50, &ctx).await,
             "File \u{2013} foo.bar \u{2013} bla bla" // file name is added for files
         );
 
@@ -1930,7 +1983,7 @@ mod tests {
         asm_file.set(Param::File, "foo.bar");
         asm_file.set_cmd(SystemMessage::AutocryptSetupMessage);
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), &mut asm_file, 50, &ctx).await,
+            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), &asm_file, 50, &ctx).await,
             "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
         );
     }
@@ -2007,7 +2060,7 @@ mod tests {
         let chatitems = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
         for chatitem in chatitems {
             if let ChatItem::Message { msg_id } = chatitem {
-                if let Ok(msg) = Message::load_from_db(&t.ctx, msg_id.clone()).await {
+                if let Ok(msg) = Message::load_from_db(&t.ctx, msg_id).await {
                     if msg.get_viewtype() == Viewtype::Image {
                         has_image = true;
                         // just check that width/height are inside some reasonable ranges
@@ -2021,4 +2074,43 @@ mod tests {
         }
         assert!(has_image);
     }
+
+    #[async_std::test]
+    async fn test_quote() {
+        use crate::config::Config;
+
+        let d = test::TestContext::new().await;
+        let ctx = &d.ctx;
+
+        let contact = Contact::create(ctx, "", "dest@example.com")
+            .await
+            .expect("failed to create contact");
+
+        let res = ctx
+            .set_config(Config::ConfiguredAddr, Some("self@example.com"))
+            .await;
+        assert!(res.is_ok());
+
+        let chat = chat::create_by_contact_id(ctx, contact).await.unwrap();
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("Quoted message".to_string()));
+
+        // Prepare message for sending, so it gets a Message-Id.
+        assert!(msg.rfc724_mid.is_empty());
+        let msg_id = chat::prepare_msg(ctx, chat, &mut msg).await.unwrap();
+        let msg = Message::load_from_db(ctx, msg_id).await.unwrap();
+        assert!(!msg.rfc724_mid.is_empty());
+
+        let mut msg2 = Message::new(Viewtype::Text);
+        msg2.set_quote(ctx, &msg).await.expect("can't set quote");
+        assert!(msg2.quoted_text() == msg.get_text());
+
+        let quoted_msg = msg2
+            .quoted_message(ctx)
+            .await
+            .expect("error while retrieving quoted message")
+            .expect("quoted message not found");
+        assert!(quoted_msg.get_text() == msg2.quoted_text());
+    }
 }

src/mimefactory.rs

@@ -11,7 +11,7 @@ use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
-use crate::format_flowed::format_flowed;
+use crate::format_flowed::{format_flowed, format_flowed_quote};
 use crate::location;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
@@ -145,7 +145,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             selfstatus: context
                 .get_config(Config::Selfstatus)
                 .await
-                .unwrap_or_else(|| default_str),
+                .unwrap_or(default_str),
             recipients,
             timestamp: msg.timestamp_sort,
             loaded: Loaded::Message { chat },
@@ -183,7 +183,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let selfstatus = context
             .get_config(Config::Selfstatus)
             .await
-            .unwrap_or_else(|| default_str);
+            .unwrap_or(default_str);
         let timestamp = dc_create_smeared_timestamp(context).await;
 
         let res = MimeFactory::<'a, 'b> {
@@ -237,22 +237,16 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     return true;
                 }
 
-                let force_plaintext = self
+                !self
                     .msg
                     .param
-                    .get_int(Param::ForcePlaintext)
-                    .unwrap_or_default();
-
-                if force_plaintext == 0 {
-                    return self
+                    .get_bool(Param::ForcePlaintext)
+                    .unwrap_or_default()
+                    && self
                         .msg
                         .param
-                        .get_int(Param::GuaranteeE2ee)
+                        .get_bool(Param::GuaranteeE2ee)
                         .unwrap_or_default()
-                        != 0;
-                }
-
-                false
             }
             Loaded::MDN { .. } => false,
         }
@@ -271,19 +265,30 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
     }
 
-    fn should_force_plaintext(&self) -> i32 {
+    fn should_force_plaintext(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
                 if chat.typ == Chattype::VerifiedGroup {
-                    0
+                    false
                 } else {
                     self.msg
                         .param
-                        .get_int(Param::ForcePlaintext)
+                        .get_bool(Param::ForcePlaintext)
                         .unwrap_or_default()
                 }
             }
-            Loaded::MDN { .. } => ForcePlaintext::NoAutocryptHeader as i32,
+            Loaded::MDN { .. } => true,
+        }
+    }
+
+    fn should_skip_autocrypt(&self) -> bool {
+        match &self.loaded {
+            Loaded::Message { .. } => self
+                .msg
+                .param
+                .get_bool(Param::SkipAutocrypt)
+                .unwrap_or_default(),
+            Loaded::MDN { .. } => true,
         }
     }
 
@@ -476,6 +481,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let min_verified = self.min_verified();
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
+        let skip_autocrypt = self.should_skip_autocrypt();
         let subject_str = self.subject_str().await;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let encrypt_helper = EncryptHelper::new(self.context).await?;
@@ -499,7 +505,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::MDN { .. } => self.render_mdn().await?,
         };
 
-        if force_plaintext != ForcePlaintext::NoAutocryptHeader as i32 {
+        if !skip_autocrypt {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
             unprotected_headers.push(Header::new("Autocrypt".into(), aheader));
@@ -510,7 +516,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let peerstates = self.peerstates_for_recipients().await?;
         let should_encrypt =
             encrypt_helper.should_encrypt(self.context, e2ee_guaranteed, &peerstates)?;
-        let is_encrypted = should_encrypt && force_plaintext == 0;
+        let is_encrypted = should_encrypt && !force_plaintext;
 
         let rfc724_mid = match self.loaded {
             Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
@@ -911,12 +917,17 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         };
 
+        let quoted_text = self
+            .msg
+            .quoted_text()
+            .map(|quote| format_flowed_quote(&quote) + "\r\n\r\n");
         let flowed_text = format_flowed(final_text);
 
         let footer = &self.selfstatus;
         let message_text = format!(
-            "{}{}{}{}{}",
+            "{}{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
+            quoted_text.unwrap_or_default(),
             escape_message_footer_marks(&flowed_text),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"

src/mimeparser.rs

@@ -223,7 +223,7 @@ impl MimeMessage {
 
         if warn_empty_signature && parser.signatures.is_empty() {
             for part in parser.parts.iter_mut() {
-                part.error = "No valid signature".to_string();
+                part.error = Some("No valid signature".to_string());
             }
         }
 
@@ -589,7 +589,7 @@ impl MimeMessage {
                 part.typ = Viewtype::Text;
                 part.msg_raw = Some(txt.clone());
                 part.msg = txt;
-                part.error = "Decryption failed".to_string();
+                part.error = Some("Decryption failed".to_string());
 
                 self.parts.push(part);
 
@@ -704,8 +704,8 @@ impl MimeMessage {
                             }
                         };
 
-                        let (simplified_txt, is_forwarded) = if decoded_data.is_empty() {
-                            ("".into(), false)
+                        let (simplified_txt, is_forwarded, top_quote) = if decoded_data.is_empty() {
+                            ("".to_string(), false, None)
                         } else {
                             let is_html = mime_type == mime::TEXT_HTML;
                             let out = if is_html {
@@ -723,7 +723,7 @@ impl MimeMessage {
                             false
                         };
 
-                        let simplified_txt = if mime_type.type_() == mime::TEXT
+                        let (simplified_txt, simplified_quote) = if mime_type.type_() == mime::TEXT
                             && mime_type.subtype() == mime::PLAIN
                             && is_format_flowed
                         {
@@ -732,9 +732,11 @@ impl MimeMessage {
                             } else {
                                 false
                             };
-                            unformat_flowed(&simplified_txt, delsp)
+                            let unflowed_text = unformat_flowed(&simplified_txt, delsp);
+                            let unflowed_quote = top_quote.map(|q| unformat_flowed(&q, delsp));
+                            (unflowed_text, unflowed_quote)
                         } else {
-                            simplified_txt
+                            (simplified_txt, top_quote)
                         };
 
                         if !simplified_txt.is_empty() {
@@ -742,6 +744,9 @@ impl MimeMessage {
                             part.typ = Viewtype::Text;
                             part.mimetype = Some(mime_type);
                             part.msg = simplified_txt;
+                            if let Some(quote) = simplified_quote {
+                                part.param.set(Param::Quote, quote);
+                            }
                             part.msg_raw = Some(decoded_data);
                             self.do_add_single_part(part);
                         }
@@ -1153,11 +1158,21 @@ pub(crate) fn parse_message_id(ids: &str) -> Result<String> {
 }
 
 fn is_known(key: &str) -> bool {
-    match key {
-        "return-path" | "date" | "from" | "sender" | "reply-to" | "to" | "cc" | "bcc"
-        | "message-id" | "in-reply-to" | "references" | "subject" => true,
-        _ => false,
-    }
+    matches!(
+        key,
+        "return-path"
+            | "date"
+            | "from"
+            | "sender"
+            | "reply-to"
+            | "to"
+            | "cc"
+            | "bcc"
+            | "message-id"
+            | "in-reply-to"
+            | "references"
+            | "subject"
+    )
 }
 
 #[derive(Debug, Default, Clone)]
@@ -1169,7 +1184,7 @@ pub struct Part {
     pub bytes: usize,
     pub param: Params,
     org_filename: Option<String>,
-    pub error: String,
+    pub error: Option<String>,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1271,9 +1286,9 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String
 }
 
 /// Returned addresses are normalized and lowercased.
-fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
+pub(crate) fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
     get_all_addresses_from_header(headers, |header_key| {
-        header_key == "to" || header_key == "cc"
+        header_key == "to" || header_key == "cc" || header_key == "bcc"
     })
 }
 
@@ -1318,6 +1333,8 @@ where
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use crate::test_utils::*;
 
@@ -2107,12 +2124,39 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
         assert!(test.is_empty());
     }
 
-    #[test]
-    fn test_mime_parse_format_flowed() {
-        let mime_type = "text/plain; charset=utf-8; Format=Flowed; DelSp=No"
-            .parse::<mime::Mime>()
+    #[async_std::test]
+    async fn parse_format_flowed_quote() {
+        let context = TestContext::new().await;
+        let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+Subject: Re: swipe-to-reply
+MIME-Version: 1.0
+In-Reply-To: <bar@example.org>
+Date: Tue, 06 Oct 2020 00:00:00 +0000
+Chat-Version: 1.0
+Message-ID: <foo@example.org>
+To: bob <bob@example.org>
+From: alice <alice@example.org>
+
+> Long 
+> quote.
+
+Reply
+"##;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
             .unwrap();
-        let format_param = mime_type.get_param("format").unwrap();
-        assert_eq!(format_param.as_str().to_ascii_lowercase(), "flowed");
+        assert_eq!(
+            message.get_subject(),
+            Some("Re: swipe-to-reply".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Text);
+        assert_eq!(
+            message.parts[0].param.get(Param::Quote).unwrap(),
+            "Long quote."
+        );
+        assert_eq!(message.parts[0].msg, "Reply");
     }
 }

src/oauth2.rs

@@ -170,16 +170,14 @@ pub async fn dc_get_oauth2_access_token(
         }
 
         // ... and POST
-        let response = surf::post(post_url).body_form(&post_param);
-        if response.is_err() {
-            warn!(
-                context,
-                "Error calling OAuth2 at {}: {:?}", token_url, response
-            );
+        let mut req = surf::post(post_url).build();
+        if let Err(err) = req.body_form(&post_param) {
+            warn!(context, "Error calling OAuth2 at {}: {:?}", token_url, err);
             return None;
         }
 
-        let parsed: Result<Response, _> = response.unwrap().recv_json().await;
+        let client = surf::Client::new();
+        let parsed: Result<Response, _> = client.recv_json(req).await;
         if parsed.is_err() {
             warn!(
                 context,
@@ -305,7 +303,7 @@ impl Oauth2 {
 
                     let mut fqdn: String = String::from(domain.as_ref());
                     if !fqdn.ends_with('.') {
-                        fqdn.push_str(".");
+                        fqdn.push('.');
                     }
 
                     if let Ok(res) = resolver.mx_lookup(fqdn).await {
@@ -323,7 +321,7 @@ impl Oauth2 {
     }
 
     async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
-        let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
+        let userinfo_url = self.get_userinfo.unwrap_or("");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
 
         // should returns sth. as

src/param.rs

@@ -3,12 +3,13 @@ use std::fmt;
 use std::str;
 
 use async_std::path::PathBuf;
+use itertools::Itertools;
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
-use crate::error::{self, bail, ensure};
+use crate::error::{self, bail};
 use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 
@@ -40,16 +41,21 @@ pub enum Param {
     /// 'c' nor 'e' are preset, the messages is only transport encrypted.
     ErroneousE2ee = b'e',
 
-    /// For Messages: force unencrypted message, either `ForcePlaintext::AddAutocryptHeader` (1),
-    /// `ForcePlaintext::NoAutocryptHeader` (2) or 0.
+    /// For Messages: force unencrypted message, a value from `ForcePlaintext` enum.
     ForcePlaintext = b'u',
 
+    /// For Messages: do not include Autocrypt header.
+    SkipAutocrypt = b'o',
+
     /// For Messages
     WantsMdn = b'r',
 
     /// For Messages
     Forwarded = b'a',
 
+    /// For Messages: quoted text.
+    Quote = b'q',
+
     /// For Messages
     Cmd = b'S',
 
@@ -128,14 +134,6 @@ pub enum Param {
     MsgId = b'I',
 }
 
-/// Possible values for `Param::ForcePlaintext`.
-#[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive)]
-#[repr(u8)]
-pub enum ForcePlaintext {
-    AddAutocryptHeader = 1,
-    NoAutocryptHeader = 2,
-}
-
 /// An object for handling key=value parameter lists.
 ///
 /// The structure is serialized by calling `to_string()` on it.
@@ -152,7 +150,12 @@ impl fmt::Display for Params {
             if i > 0 {
                 writeln!(f)?;
             }
-            write!(f, "{}={}", *key as u8 as char, value)?;
+            write!(
+                f,
+                "{}={}",
+                *key as u8 as char,
+                value.split('\n').join("\n\n")
+            )?;
         }
         Ok(())
     }
@@ -163,27 +166,28 @@ impl str::FromStr for Params {
 
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let mut inner = BTreeMap::new();
-        for pair in s.trim().lines() {
-            let pair = pair.trim();
-            if pair.is_empty() {
-                continue;
-            }
-            // TODO: probably nicer using a regex
-            ensure!(pair.len() > 1, "Invalid key pair: '{}'", pair);
-            let mut split = pair.splitn(2, '=');
-            let key = split.next();
-            let value = split.next();
+        let mut lines = s.lines().peekable();
 
-            ensure!(key.is_some(), "Missing key");
-            ensure!(value.is_some(), "Missing value");
+        while let Some(line) = lines.next() {
+            if let [key, value] = line.splitn(2, '=').collect::<Vec<_>>()[..] {
+                let key = key.to_string();
+                let mut value = value.to_string();
+                while let Some(s) = lines.peek() {
+                    if !s.is_empty() {
+                        break;
+                    }
+                    lines.next();
+                    value.push('\n');
+                    value += lines.next().unwrap_or_default();
+                }
 
-            let key = key.unwrap_or_default().trim();
-            let value = value.unwrap_or_default().trim();
-
-            if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
-                inner.insert(key, value.to_string());
+                if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
+                    inner.insert(key, value);
+                } else {
+                    bail!("Unknown key: {}", key);
+                }
             } else {
-                bail!("Unknown key: {}", key);
+                bail!("Not a key-value pair: {:?}", line);
             }
         }
 
@@ -379,7 +383,7 @@ mod tests {
 
     #[test]
     fn test_dc_param() {
-        let mut p1: Params = "\r\n\r\na=1\nf=2\n\nc = 3 ".parse().unwrap();
+        let mut p1: Params = "a=1\nf=2\nc=3".parse().unwrap();
 
         assert_eq!(p1.get_int(Param::Forwarded), Some(1));
         assert_eq!(p1.get_int(Param::File), Some(2));
@@ -413,6 +417,14 @@ mod tests {
         assert_eq!(p1.len(), 0)
     }
 
+    #[test]
+    fn test_roundtrip() {
+        let mut params = Params::new();
+        params.set(Param::Height, "foo\nbar=baz\nquux");
+        params.set(Param::Width, "\n\n\na=\n=");
+        assert_eq!(params.to_string().parse::<Params>().unwrap(), params);
+    }
+
     #[test]
     fn test_regression() {
         let p1: Params = "a=cli%40deltachat.de\nn=\ni=TbnwJ6lSvD5\ns=0ejvbdFSQxB"
@@ -475,8 +487,8 @@ mod tests {
         );
 
         // Blob in blobdir, expect blob.
-        let bar = t.ctx.get_blobdir().join("bar");
-        p.set(Param::File, bar.to_str().unwrap());
+        let bar_path = t.ctx.get_blobdir().join("bar");
+        p.set(Param::File, bar_path.to_str().unwrap());
         let blob = p
             .get_blob(Param::File, &t.ctx, false)
             .await

src/peerstate.rs

@@ -94,47 +94,42 @@ pub enum ToSave {
 }
 
 impl<'a> Peerstate<'a> {
-    pub fn new(context: &'a Context, addr: String) -> Self {
+    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
         Peerstate {
             context,
-            addr,
-            last_seen: 0,
-            last_seen_autocrypt: 0,
-            prefer_encrypt: Default::default(),
-            public_key: None,
-            public_key_fingerprint: None,
+            addr: header.addr.clone(),
+            last_seen: message_time,
+            last_seen_autocrypt: message_time,
+            prefer_encrypt: header.prefer_encrypt,
+            public_key: Some(header.public_key.clone()),
+            public_key_fingerprint: Some(header.public_key.fingerprint()),
             gossip_key: None,
             gossip_key_fingerprint: None,
             gossip_timestamp: 0,
             verified_key: None,
             verified_key_fingerprint: None,
-            to_save: None,
+            to_save: Some(ToSave::All),
             fingerprint_changed: false,
         }
     }
 
-    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, header.addr.clone());
-
-        res.last_seen = message_time;
-        res.last_seen_autocrypt = message_time;
-        res.to_save = Some(ToSave::All);
-        res.prefer_encrypt = header.prefer_encrypt;
-        res.public_key = Some(header.public_key.clone());
-        res.recalc_fingerprint();
-
-        res
-    }
-
     pub fn from_gossip(context: &'a Context, gossip_header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, gossip_header.addr.clone());
-
-        res.gossip_timestamp = message_time;
-        res.to_save = Some(ToSave::All);
-        res.gossip_key = Some(gossip_header.public_key.clone());
-        res.recalc_fingerprint();
-
-        res
+        Peerstate {
+            context,
+            addr: gossip_header.addr.clone(),
+            last_seen: 0,
+            last_seen_autocrypt: 0,
+            prefer_encrypt: Default::default(),
+            public_key: None,
+            public_key_fingerprint: None,
+            gossip_key: Some(gossip_header.public_key.clone()),
+            gossip_key_fingerprint: Some(gossip_header.public_key.fingerprint()),
+            gossip_timestamp: message_time,
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        }
     }
 
     pub async fn from_addr(context: &'a Context, addr: &str) -> Result<Option<Peerstate<'a>>> {
@@ -175,40 +170,44 @@ impl<'a> Peerstate<'a> {
                 public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
                 gossip_key_fingerprint, verified_key, verified_key_fingerprint
                 */
-                let mut res = Self::new(context, row.get(0)?);
 
-                res.last_seen = row.get(1)?;
-                res.last_seen_autocrypt = row.get(2)?;
-                res.prefer_encrypt = EncryptPreference::from_i32(row.get(3)?).unwrap_or_default();
-                res.gossip_timestamp = row.get(5)?;
-
-                res.public_key_fingerprint = row
-                    .get::<_, Option<String>>(7)?
-                    .map(|s| s.parse::<Fingerprint>())
-                    .transpose()
-                    .unwrap_or_default();
-                res.gossip_key_fingerprint = row
-                    .get::<_, Option<String>>(8)?
-                    .map(|s| s.parse::<Fingerprint>())
-                    .transpose()
-                    .unwrap_or_default();
-                res.verified_key_fingerprint = row
-                    .get::<_, Option<String>>(10)?
-                    .map(|s| s.parse::<Fingerprint>())
-                    .transpose()
-                    .unwrap_or_default();
-                res.public_key = row
-                    .get(4)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
-                res.gossip_key = row
-                    .get(6)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
-                res.verified_key = row
-                    .get(9)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
+                let res = Peerstate {
+                    context,
+                    addr: row.get(0)?,
+                    last_seen: row.get(1)?,
+                    last_seen_autocrypt: row.get(2)?,
+                    prefer_encrypt: EncryptPreference::from_i32(row.get(3)?).unwrap_or_default(),
+                    public_key: row
+                        .get(4)
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    public_key_fingerprint: row
+                        .get::<_, Option<String>>(7)?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    gossip_key: row
+                        .get(6)
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    gossip_key_fingerprint: row
+                        .get::<_, Option<String>>(8)?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    gossip_timestamp: row.get(5)?,
+                    verified_key: row
+                        .get(9)
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    verified_key_fingerprint: row
+                        .get::<_, Option<String>>(10)?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    to_save: None,
+                    fingerprint_changed: false,
+                };
 
                 Ok(res)
             })
@@ -291,7 +290,7 @@ impl<'a> Peerstate<'a> {
             return;
         }
 
-        if message_time > self.last_seen_autocrypt {
+        if message_time > self.last_seen {
             self.last_seen = message_time;
             self.last_seen_autocrypt = message_time;
             self.to_save = Some(ToSave::Timestamps);
@@ -490,7 +489,6 @@ mod tests {
     use super::*;
     use crate::test_utils::*;
     use pretty_assertions::assert_eq;
-    use tempfile::TempDir;
 
     #[async_std::test]
     async fn test_peerstate_save_to_db() {
@@ -637,10 +635,44 @@ mod tests {
         assert_eq!(peerstate.verified_key_fingerprint, None);
     }
 
-    // TODO: don't copy this from stress.rs
-    #[allow(dead_code)]
-    struct TestContext {
-        ctx: Context,
-        dir: TempDir,
+    #[async_std::test]
+    async fn test_peerstate_degrade_reordering() {
+        let context = crate::test_utils::TestContext::new().await.ctx;
+        let addr = "example@example.org";
+        let pub_key = alice_keypair().public;
+        let header = Aheader::new(addr.to_string(), pub_key, EncryptPreference::Mutual);
+
+        let mut peerstate = Peerstate {
+            context: &context,
+            addr: addr.to_string(),
+            last_seen: 0,
+            last_seen_autocrypt: 0,
+            prefer_encrypt: EncryptPreference::NoPreference,
+            public_key: None,
+            public_key_fingerprint: None,
+            gossip_key: None,
+            gossip_timestamp: 0,
+            gossip_key_fingerprint: None,
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: None,
+            fingerprint_changed: false,
+        };
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::NoPreference);
+
+        peerstate.apply_header(&header, 100);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
+
+        peerstate.degrade_encryption(300);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Reset);
+
+        // This has message time 200, while encryption was degraded at timestamp 300.
+        // Because of reordering, header should not be applied.
+        peerstate.apply_header(&header, 200);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Reset);
+
+        // Same header will be applied in the future.
+        peerstate.apply_header(&header, 400);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
     }
 }

src/pgp.rs

@@ -439,9 +439,9 @@ mod tests {
             let bob = bob_keypair();
             TestKeys {
                 alice_secret: alice.secret.clone(),
-                alice_public: alice.public.clone(),
+                alice_public: alice.public,
                 bob_secret: bob.secret.clone(),
-                bob_public: bob.public.clone(),
+                bob_public: bob.public,
             }
         }
     }
@@ -458,7 +458,7 @@ mod tests {
             let mut keyring = Keyring::new();
             keyring.add(KEYS.alice_public.clone());
             keyring.add(KEYS.bob_public.clone());
-            smol::block_on(pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))).unwrap()
+            futures_lite::future::block_on(pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))).unwrap()
         };
 
         /// A cyphertext encrypted to Alice & Bob, not signed.
@@ -466,7 +466,7 @@ mod tests {
             let mut keyring = Keyring::new();
             keyring.add(KEYS.alice_public.clone());
             keyring.add(KEYS.bob_public.clone());
-            smol::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
+            futures_lite::future::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
         };
     }
 

src/provider/data.rs

@@ -279,6 +279,24 @@ lazy_static::lazy_static! {
         oauth2_authorizer: None,
     };
 
+    // hermes.radio.md: hermes.radio
+    static ref P_HERMES_RADIO: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/hermes-radio",
+        server: vec![
+        ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::MdnsEnabled, value: "0" },
+            ConfigDefault { key: Config::E2eeEnabled, value: "0" },
+            ConfigDefault { key: Config::MediaQuality, value: "1" },
+            ConfigDefault { key: Config::ShowEmails, value: "2" },
+        ]),
+        strict_tls: false,
+        oauth2_authorizer: None,
+    };
+
     // hey.com.md: hey.com
     static ref P_HEY_COM: Provider = Provider {
         status: Status::BROKEN,
@@ -674,6 +692,7 @@ lazy_static::lazy_static! {
         ("gmx.info", &*P_GMX_NET),
         ("gmx.biz", &*P_GMX_NET),
         ("gmx.com", &*P_GMX_NET),
+        ("hermes.radio", &*P_HERMES_RADIO),
         ("hey.com", &*P_HEY_COM),
         ("i.ua", &*P_I_UA),
         ("icloud.com", &*P_ICLOUD),
@@ -807,4 +826,6 @@ lazy_static::lazy_static! {
         ("narod.ru", &*P_YANDEX_RU),
         ("ziggo.nl", &*P_ZIGGO_NL),
     ].iter().copied().collect();
+
+    pub static ref PROVIDER_UPDATED: chrono::NaiveDate = chrono::NaiveDate::from_ymd(2020, 9, 22);
 }

src/provider/mod.rs

@@ -4,7 +4,8 @@ mod data;
 
 use crate::config::Config;
 use crate::dc_tools::EmailAddress;
-use crate::provider::data::PROVIDER_DATA;
+use crate::provider::data::{PROVIDER_DATA, PROVIDER_UPDATED};
+use chrono::{NaiveDateTime, NaiveTime};
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
@@ -91,9 +92,18 @@ pub fn get_provider_info(addr: &str) -> Option<&Provider> {
     None
 }
 
+// returns update timestamp in seconds, UTC, compatible for comparison with time() and database times
+pub fn get_provider_update_timestamp() -> i64 {
+    NaiveDateTime::new(*PROVIDER_UPDATED, NaiveTime::from_hms(0, 0, 0)).timestamp_millis() / 1_000
+}
+
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
+    use crate::dc_tools::time;
+    use chrono::NaiveDate;
 
     #[test]
     fn test_get_provider_info_unexistant() {
@@ -136,4 +146,16 @@ mod tests {
         let provider = get_provider_info("user@googlemail.com").unwrap();
         assert!(provider.status == Status::PREPARATION);
     }
+
+    #[test]
+    fn test_get_provider_update_timestamp() {
+        let timestamp_past = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 9, 9),
+            NaiveTime::from_hms(0, 0, 0),
+        )
+        .timestamp_millis()
+            / 1_000;
+        assert!(get_provider_update_timestamp() <= time());
+        assert!(get_provider_update_timestamp() > timestamp_past);
+    }
 }

src/provider/update.py

@@ -4,6 +4,7 @@
 import sys
 import os
 import yaml
+import datetime
 
 out_all = ""
 out_domains = ""
@@ -169,6 +170,12 @@ if __name__ == "__main__":
 
     out_all += "    pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [\n"
     out_all += out_domains;
-    out_all += "    ].iter().copied().collect();\n}"
+    out_all += "    ].iter().copied().collect();\n\n"
+
+    now = datetime.datetime.utcnow()
+    out_all += "    pub static ref PROVIDER_UPDATED: chrono::NaiveDate = "\
+                        "chrono::NaiveDate::from_ymd("+str(now.year)+", "+str(now.month)+", "+str(now.day)+");\n"
+
+    out_all += "}"
 
     print(out_all)

src/scheduler.rs

@@ -3,6 +3,7 @@ use async_std::sync::{channel, Receiver, Sender};
 use async_std::task;
 
 use crate::context::Context;
+use crate::dc_tools::maybe_add_time_based_warnings;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
 use crate::{config::Config, message::MsgId, smtp::Smtp};
@@ -81,6 +82,8 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                         warn!(ctx, "failed to close folder: {:?}", err);
                     }
 
+                    maybe_add_time_based_warnings(&ctx).await;
+
                     info = if ctx.get_config_bool(Config::InboxWatch).await {
                         fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
                     } else {
@@ -128,7 +131,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
             // connect and fake idle if unable to connect
             if let Err(err) = connection.connect_configured(&ctx).await {
                 warn!(ctx, "imap connection failed: {}", err);
-                return connection.fake_idle(&ctx, None).await;
+                return connection.fake_idle(&ctx, Some(watch_folder)).await;
             }
 
             // fetch
@@ -412,10 +415,7 @@ impl Scheduler {
 
     /// Check if the scheduler is running.
     pub fn is_running(&self) -> bool {
-        match self {
-            Scheduler::Running { .. } => true,
-            _ => false,
-        }
+        matches!(self, Scheduler::Running { .. })
     }
 }
 

src/securejoin.rs

@@ -1,7 +1,8 @@
 //! Verified contact protocol implementation as [specified by countermitm project](https://countermitm.readthedocs.io/en/stable/new.html#setup-contact-protocol)
 
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
+use anyhow::{bail, Error};
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 
 use crate::aheader::EncryptPreference;
@@ -11,7 +12,6 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::e2ee::*;
-use crate::error::{bail, Error};
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
@@ -21,6 +21,7 @@ use crate::mimeparser::*;
 use crate::param::*;
 use crate::peerstate::*;
 use crate::qr::check_qr;
+use crate::sql;
 use crate::stock::StockMessage;
 use crate::token;
 
@@ -67,18 +68,6 @@ macro_rules! get_qr_attr {
     };
 }
 
-#[derive(Debug, PartialEq)]
-pub(crate) enum BobStatus {
-    Error,
-    Success,
-}
-
-impl Default for BobStatus {
-    fn default() -> Self {
-        Self::Error
-    }
-}
-
 /// State for setup-contact/secure-join protocol joiner's side.
 ///
 /// The setup-contact protocol needs to carry state for both the inviter (Alice) and the
@@ -88,11 +77,6 @@ impl Default for BobStatus {
 pub(crate) struct Bob {
     /// The next message expected by the protocol.
     expects: SecureJoinStep,
-    /// The final status of the last-run setup-contact/secure-join protocol.
-    ///
-    /// This is only meaningful if you know you have exited the protocol but have not
-    /// started a new one.
-    pub status: BobStatus,
     /// The QR-scanned information of the currently running protocol.
     pub qr_scan: Option<Lot>,
 }
@@ -204,78 +188,79 @@ async fn get_self_fingerprint(context: &Context) -> Option<Fingerprint> {
     }
 }
 
-async fn cleanup(
-    context: &Context,
-    contact_chat_id: ChatId,
-    ongoing_allocated: bool,
-    join_vg: bool,
-) -> ChatId {
+async fn cleanup(context: &Context, ongoing_allocated: bool) {
     let mut bob = context.bob.write().await;
     bob.expects = SecureJoinStep::NotActive;
-    let ret_chat_id: ChatId = if bob.status == BobStatus::Success {
-        if join_vg {
-            chat::get_chat_id_by_grpid(
-                context,
-                bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap(),
-            )
-            .await
-            .unwrap_or((ChatId::new(0), false, Blocked::Not))
-            .0
-        } else {
-            contact_chat_id
-        }
-    } else {
-        ChatId::new(0)
-    };
     bob.qr_scan = None;
-
     if ongoing_allocated {
         context.free_ongoing().await;
     }
-    ret_chat_id
 }
 
-/// Take a scanned QR-code and do the setup-contact/join-group handshake.
-/// See the ffi-documentation for more details.
-pub async fn dc_join_securejoin(context: &Context, qr: &str) -> ChatId {
+#[derive(Debug, thiserror::Error)]
+pub enum JoinError {
+    #[error("Unknown QR-code")]
+    QrCode,
+    #[error("Aborted by user")]
+    Aborted,
+    #[error("Failed to send handshake message")]
+    SendMessage(#[from] SendMsgError),
+    // Note that this can currently only occur if there is a bug in the QR/Lot code as this
+    // is supposed to create a contact for us.
+    #[error("Unknown contact (this is a bug)")]
+    UnknownContact,
+    // Note that this can only occur if we failed to create the chat correctly.
+    #[error("No Chat found for group (this is a bug)")]
+    MissingChat(#[source] sql::Error),
+}
+
+/// Take a scanned QR-code and do the setup-contact/join-group/invite handshake.
+///
+/// This is the start of the process for the joiner.  See the module and ffi documentation
+/// for more details.
+///
+/// When joining a group this will start an "ongoing" process and will block until the
+/// process is completed, the [ChatId] for the new group is not known any sooner.  When
+/// verifying a contact this returns immediately.
+pub async fn dc_join_securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
     if context.alloc_ongoing().await.is_err() {
-        return cleanup(&context, ChatId::new(0), false, false).await;
+        cleanup(&context, false).await;
+        return Err(JoinError::Aborted);
     }
 
     securejoin(context, qr).await
 }
 
-async fn securejoin(context: &Context, qr: &str) -> ChatId {
+async fn securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
     /*========================================================
     ====             Bob - the joiner's side             =====
     ====   Step 2 in "Setup verified contact" protocol   =====
     ========================================================*/
 
-    let mut contact_chat_id = ChatId::new(0);
-    let mut join_vg: bool = false;
-
     info!(context, "Requesting secure-join ...",);
     ensure_secret_key_exists(context).await.ok();
     let qr_scan = check_qr(context, &qr).await;
     if qr_scan.state != LotState::QrAskVerifyContact && qr_scan.state != LotState::QrAskVerifyGroup
     {
         error!(context, "Unknown QR code.",);
-        return cleanup(&context, contact_chat_id, true, join_vg).await;
+        cleanup(&context, true).await;
+        return Err(JoinError::QrCode);
     }
-    contact_chat_id = match chat::create_by_contact_id(context, qr_scan.id).await {
+    let contact_chat_id = match chat::create_by_contact_id(context, qr_scan.id).await {
         Ok(chat_id) => chat_id,
         Err(_) => {
             error!(context, "Unknown contact.");
-            return cleanup(&context, contact_chat_id, true, join_vg).await;
+            cleanup(&context, true).await;
+            return Err(JoinError::UnknownContact);
         }
     };
     if context.shall_stop_ongoing().await {
-        return cleanup(&context, contact_chat_id, true, join_vg).await;
+        cleanup(&context, true).await;
+        return Err(JoinError::Aborted);
     }
-    join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
+    let join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
     {
         let mut bob = context.bob.write().await;
-        bob.status = BobStatus::Error;
         bob.qr_scan = Some(qr_scan);
     }
     if fingerprint_equals_sender(
@@ -325,7 +310,8 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         .await
         {
             error!(context, "failed to send handshake message: {}", err);
-            return cleanup(&context, contact_chat_id, true, join_vg).await;
+            cleanup(&context, true).await;
+            return Err(JoinError::SendMessage(err));
         }
     } else {
         context.bob.write().await.expects = SecureJoinStep::AuthRequired;
@@ -342,7 +328,8 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         .await
         {
             error!(context, "failed to send handshake message: {}", err);
-            return cleanup(&context, contact_chat_id, true, join_vg).await;
+            cleanup(&context, true).await;
+            return Err(JoinError::SendMessage(err));
         }
     }
 
@@ -351,15 +338,45 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         while !context.shall_stop_ongoing().await {
             async_std::task::sleep(Duration::from_millis(50)).await;
         }
-        cleanup(&context, contact_chat_id, true, join_vg).await
+
+        // handle_securejoin_handshake() calls Context::stop_ongoing before the group chat
+        // is created (it is created after handle_securejoin_handshake() returns by
+        // dc_receive_imf()).  As a hack we just wait a bit for it to appear.
+        let start = Instant::now();
+        let chatid = loop {
+            {
+                let bob = context.bob.read().await;
+                let grpid = bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap();
+                match chat::get_chat_id_by_grpid(context, grpid).await {
+                    Ok((chatid, _is_verified, _blocked)) => break chatid,
+                    Err(err) => {
+                        if start.elapsed() > Duration::from_secs(7) {
+                            return Err(JoinError::MissingChat(err));
+                        }
+                    }
+                }
+            }
+            async_std::task::sleep(Duration::from_millis(50)).await;
+        };
+
+        cleanup(&context, true).await;
+        Ok(chatid)
     } else {
         // for a one-to-one-chat, the chat is already known, return the chat-id,
         // the verification runs in background
         context.free_ongoing().await;
-        contact_chat_id
+        Ok(contact_chat_id)
     }
 }
 
+/// Error for [send_handshake_msg].
+///
+/// Wrapping the [anyhow::Error] means we can "impl From" more easily on errors from this
+/// function.
+#[derive(Debug, thiserror::Error)]
+#[error("Failed sending handshake message")]
+pub struct SendMsgError(#[from] anyhow::Error);
+
 async fn send_handshake_msg(
     context: &Context,
     contact_chat_id: ChatId,
@@ -367,7 +384,7 @@ async fn send_handshake_msg(
     param2: impl AsRef<str>,
     fingerprint: Option<Fingerprint>,
     grpid: impl AsRef<str>,
-) -> Result<(), HandshakeError> {
+) -> Result<(), SendMsgError> {
     let mut msg = Message::default();
     msg.viewtype = Viewtype::Text;
     msg.text = Some(format!("Secure-Join: {}", step));
@@ -388,18 +405,12 @@ async fn send_handshake_msg(
         msg.param.set(Param::Arg4, grpid.as_ref());
     }
     if step == "vg-request" || step == "vc-request" {
-        msg.param.set_int(
-            Param::ForcePlaintext,
-            ForcePlaintext::AddAutocryptHeader as i32,
-        );
+        msg.param.set_int(Param::ForcePlaintext, 1);
     } else {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
     }
 
-    chat::send_msg(context, contact_chat_id, &mut msg)
-        .await
-        .map_err(HandshakeError::MsgSendFailed)?;
-
+    chat::send_msg(context, contact_chat_id, &mut msg).await?;
     Ok(())
 }
 
@@ -459,7 +470,7 @@ pub(crate) enum HandshakeError {
     #[error("No configured self address found")]
     NoSelfAddr,
     #[error("Failed to send message")]
-    MsgSendFailed(#[source] Error),
+    MsgSendFailed(#[from] SendMsgError),
     #[error("Failed to parse fingerprint")]
     BadFingerprint(#[from] crate::key::FingerprintError),
 }
@@ -597,7 +608,6 @@ pub(crate) async fn handle_securejoin_handshake(
                     },
                 )
                 .await;
-                context.bob.write().await.status = BobStatus::Error; // secure-join failed
                 context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
@@ -610,7 +620,6 @@ pub(crate) async fn handle_securejoin_handshake(
                     "Fingerprint mismatch on joiner-side.",
                 )
                 .await;
-                context.bob.write().await.status = BobStatus::Error; // secure-join failed
                 context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
@@ -808,7 +817,6 @@ pub(crate) async fn handle_securejoin_handshake(
                     "Contact confirm message not encrypted.",
                 )
                 .await;
-                context.bob.write().await.status = BobStatus::Error;
                 return Ok(abort_retval);
             }
 
@@ -857,7 +865,6 @@ pub(crate) async fn handle_securejoin_handshake(
             )
             .await?;
 
-            context.bob.write().await.status = BobStatus::Success;
             context.stop_ongoing().await;
             Ok(if join_vg {
                 HandshakeMessage::Propagate
@@ -877,6 +884,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     return Ok(HandshakeMessage::Ignore);
                 }
                 if join_vg {
+                    // Responsible for showing "$Bob securely joined $group" message
                     inviter_progress!(context, contact_id, 800);
                     inviter_progress!(context, contact_id, 1000);
                     let field_grpid = mime_message
@@ -1088,3 +1096,337 @@ fn encrypted_and_signed(
         false
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::chat;
+    use crate::peerstate::Peerstate;
+    use crate::test_utils::TestContext;
+
+    #[async_std::test]
+    async fn test_setup_contact() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // Generate QR-code, ChatId(0) indicates setup-contact
+        let qr = dc_get_securejoin_qr(&alice.ctx, ChatId::new(0))
+            .await
+            .unwrap();
+
+        // Bob scans QR-code, sends vc-request
+        let bob_chatid = dc_join_securejoin(&bob.ctx, &qr).await.unwrap();
+
+        let sent = bob.pop_sent_msg().await;
+        assert_eq!(sent.id(), bob_chatid);
+        assert_eq!(sent.recipient(), "alice@example.com".parse().unwrap());
+        let msg = alice.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vc-request");
+        assert!(msg.get(HeaderDef::SecureJoinInvitenumber).is_some());
+
+        // Alice receives vc-request, sends vc-auth-required
+        alice.recv_msg(&sent).await;
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vc-auth-required");
+
+        // Bob receives vc-auth-required, sends vc-request-with-auth
+        bob.recv_msg(&sent).await;
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-request-with-auth"
+        );
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
+        let bob_fp = SignedPublicKey::load_self(&bob.ctx)
+            .await
+            .unwrap()
+            .fingerprint();
+        assert_eq!(
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
+            bob_fp.hex()
+        );
+
+        // Alice should not yet have Bob verified
+        let contact_bob_id =
+            Contact::lookup_id_by_addr(&alice.ctx, "bob@example.net", Origin::Unknown).await;
+        let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Alice receives vc-request-with-auth, sends vc-contact-confirm
+        alice.recv_msg(&sent).await;
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm"
+        );
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob.ctx, "alice@example.com", Origin::Unknown).await;
+        let contact_alice = Contact::load_from_db(&bob.ctx, contact_alice_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&bob.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Bob receives vc-contact-confirm, sends vc-contact-confirm-received
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm-received"
+        );
+    }
+
+    #[async_std::test]
+    async fn test_setup_contact_bad_qr() {
+        let bob = TestContext::new_bob().await;
+        let ret = dc_join_securejoin(&bob.ctx, "not a qr code").await;
+        assert!(matches!(ret, Err(JoinError::QrCode)));
+    }
+
+    #[async_std::test]
+    async fn test_setup_contact_bob_knows_alice() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // Ensure Bob knows Alice_FP
+        let alice_pubkey = SignedPublicKey::load_self(&alice.ctx).await.unwrap();
+        let peerstate = Peerstate {
+            context: &bob.ctx,
+            addr: "alice@example.com".into(),
+            last_seen: 10,
+            last_seen_autocrypt: 10,
+            prefer_encrypt: EncryptPreference::Mutual,
+            public_key: Some(alice_pubkey.clone()),
+            public_key_fingerprint: Some(alice_pubkey.fingerprint()),
+            gossip_key: Some(alice_pubkey.clone()),
+            gossip_timestamp: 10,
+            gossip_key_fingerprint: Some(alice_pubkey.fingerprint()),
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        };
+        peerstate.save_to_db(&bob.ctx.sql, true).await.unwrap();
+
+        // Generate QR-code, ChatId(0) indicates setup-contact
+        let qr = dc_get_securejoin_qr(&alice.ctx, ChatId::new(0))
+            .await
+            .unwrap();
+
+        // Bob scans QR-code, sends vc-request-with-auth, skipping vc-request
+        dc_join_securejoin(&bob.ctx, &qr).await.unwrap();
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-request-with-auth"
+        );
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
+        let bob_fp = SignedPublicKey::load_self(&bob.ctx)
+            .await
+            .unwrap()
+            .fingerprint();
+        assert_eq!(
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
+            bob_fp.hex()
+        );
+
+        // Alice should not yet have Bob verified
+        let (contact_bob_id, _modified) = Contact::add_or_lookup(
+            &alice.ctx,
+            "Bob",
+            "bob@example.net",
+            Origin::ManuallyCreated,
+        )
+        .await
+        .unwrap();
+        let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Alice receives vc-request-with-auth, sends vc-contact-confirm
+        alice.recv_msg(&sent).await;
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm"
+        );
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob.ctx, "alice@example.com", Origin::Unknown).await;
+        let contact_alice = Contact::load_from_db(&bob.ctx, contact_alice_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&bob.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Bob receives vc-contact-confirm, sends vc-contact-confirm-received
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm-received"
+        );
+    }
+
+    #[async_std::test]
+    async fn test_secure_join() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chatid = chat::create_group_chat(&alice.ctx, VerifiedStatus::Verified, "the chat")
+            .await
+            .unwrap();
+
+        // Generate QR-code, secure-join implied by chatid
+        let qr = dc_get_securejoin_qr(&alice.ctx, chatid).await.unwrap();
+
+        // Bob scans QR-code, sends vg-request; blocks on ongoing process
+        let joiner = {
+            let qr = qr.clone();
+            let ctx = bob.ctx.clone();
+            async_std::task::spawn(async move { dc_join_securejoin(&ctx, &qr).await.unwrap() })
+        };
+
+        let sent = bob.pop_sent_msg().await;
+        assert_eq!(sent.recipient(), "alice@example.com".parse().unwrap());
+        let msg = alice.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-request");
+        assert!(msg.get(HeaderDef::SecureJoinInvitenumber).is_some());
+
+        // Alice receives vg-request, sends vg-auth-required
+        alice.recv_msg(&sent).await;
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-auth-required");
+
+        // Bob receives vg-auth-required, sends vg-request-with-auth
+        bob.recv_msg(&sent).await;
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vg-request-with-auth"
+        );
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
+        let bob_fp = SignedPublicKey::load_self(&bob.ctx)
+            .await
+            .unwrap()
+            .fingerprint();
+        assert_eq!(
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
+            bob_fp.hex()
+        );
+
+        // Alice should not yet have Bob verified
+        let contact_bob_id =
+            Contact::lookup_id_by_addr(&alice.ctx, "bob@example.net", Origin::Unknown).await;
+        let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Alice receives vg-request-with-auth, sends vg-member-added
+        alice.recv_msg(&sent).await;
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-member-added");
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob.ctx, "alice@example.com", Origin::Unknown).await;
+        let contact_alice = Contact::load_from_db(&bob.ctx, contact_alice_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&bob.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Bob receives vg-member-added, sends vg-member-added-received
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vg-member-added-received"
+        );
+
+        let bob_chatid = joiner.await;
+        let bob_chat = Chat::load_from_db(&bob.ctx, bob_chatid).await.unwrap();
+        assert!(bob_chat.is_verified());
+    }
+}

src/simplify.rs

@@ -1,3 +1,5 @@
+use itertools::Itertools;
+
 // protect lines starting with `--` against being treated as a footer.
 // for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B);
 // this should be invisible on most systems and there is no need to unescape it again
@@ -64,7 +66,7 @@ fn split_lines(buf: &str) -> Vec<&str> {
 
 /// Simplify message text for chat display.
 /// Remove quotes, signatures, trailing empty lines etc.
-pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
+pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool, Option<String>) {
     input.retain(|c| c != '\r');
     let lines = split_lines(&input);
     let (lines, is_forwarded) = skip_forward_header(&lines);
@@ -72,25 +74,21 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
     let original_lines = &lines;
 
     let lines = remove_message_footer(lines);
+    let (lines, top_quote) = remove_top_quote(lines);
 
     let text = if is_chat_message {
-        render_message(lines, false, false)
+        render_message(lines, false)
     } else {
         let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
         let (lines, has_bottom_quote) = remove_bottom_quote(lines);
-        let (lines, has_top_quote) = remove_top_quote(lines);
 
         if lines.iter().all(|it| it.trim().is_empty()) {
-            render_message(original_lines, false, false)
+            render_message(original_lines, false)
         } else {
-            render_message(
-                lines,
-                has_top_quote,
-                has_nonstandard_footer || has_bottom_quote,
-            )
+            render_message(lines, has_nonstandard_footer || has_bottom_quote)
         }
     };
-    (text, is_forwarded)
+    (text, is_forwarded, top_quote)
 }
 
 /// Skips "forwarded message" header.
@@ -134,11 +132,15 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
 }
 
 #[allow(clippy::indexing_slicing)]
-fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<String>) {
+    let mut first_quoted_line = 0;
     let mut last_quoted_line = None;
     let mut has_quoted_headline = false;
     for (l, line) in lines.iter().enumerate() {
         if is_plain_quote(line) {
+            if last_quoted_line.is_none() {
+                first_quoted_line = l;
+            }
             last_quoted_line = Some(l)
         } else if !is_empty_line(line) {
             if is_quoted_headline(line) && !has_quoted_headline && last_quoted_line.is_none() {
@@ -150,17 +152,25 @@ fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
         }
     }
     if let Some(last_quoted_line) = last_quoted_line {
-        (&lines[last_quoted_line + 1..], true)
+        (
+            &lines[last_quoted_line + 1..],
+            Some(
+                lines[first_quoted_line..last_quoted_line + 1]
+                    .iter()
+                    .map(|s| {
+                        s.strip_prefix(">")
+                            .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u))
+                    })
+                    .join("\n"),
+            ),
+        )
     } else {
-        (lines, false)
+        (lines, None)
     }
 }
 
-fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) -> String {
+fn render_message(lines: &[&str], is_cut_at_end: bool) -> String {
     let mut ret = String::new();
-    if is_cut_at_begin {
-        ret += "[...]";
-    }
     /* we write empty lines only in case and non-empty line follows */
     let mut pending_linebreaks = 0;
     let mut empty_body = true;
@@ -183,7 +193,7 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) ->
             pending_linebreaks = 1
         }
     }
-    if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
+    if is_cut_at_end && !empty_body {
         ret += " [...]";
     }
     // redo escaping done by escape_message_footer_marks()
@@ -231,7 +241,7 @@ mod tests {
         #[test]
         // proptest does not support [[:graphical:][:space:]] regex.
         fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let (output, _is_forwarded) = simplify(input, true);
+            let (output, _is_forwarded, _) = simplify(input, true);
             assert!(output.split('\n').all(|s| s != "-- "));
         }
     }
@@ -239,7 +249,7 @@ mod tests {
     #[test]
     fn test_dont_remove_whole_message() {
         let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, _) = simplify(input, false);
         assert_eq!(
             plain,
             "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
@@ -250,7 +260,7 @@ mod tests {
     #[test]
     fn test_chat_message() {
         let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
-        let (plain, is_forwarded) = simplify(input, true);
+        let (plain, is_forwarded, _) = simplify(input, true);
         assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
         assert!(!is_forwarded);
     }
@@ -258,7 +268,7 @@ mod tests {
     #[test]
     fn test_simplify_trim() {
         let input = "line1\n\r\r\rline2".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, _) = simplify(input, false);
 
         assert_eq!(plain, "line1\nline2");
         assert!(!is_forwarded);
@@ -267,7 +277,7 @@ mod tests {
     #[test]
     fn test_simplify_forwarded_message() {
         let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, _) = simplify(input, false);
 
         assert_eq!(plain, "Forwarded message");
         assert!(is_forwarded);
@@ -287,17 +297,17 @@ mod tests {
 
     #[test]
     fn test_remove_top_quote() {
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second"]);
+        let (lines, top_quote) = remove_top_quote(&["> first", "> second"]);
         assert!(lines.is_empty());
-        assert!(has_top_quote);
+        assert_eq!(top_quote.unwrap(), "first\nsecond");
 
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
+        let (lines, top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
         assert_eq!(lines, &["not a quote"]);
-        assert!(has_top_quote);
+        assert_eq!(top_quote.unwrap(), "first\nsecond");
 
-        let (lines, has_top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
+        let (lines, top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
         assert_eq!(lines, &["not a quote", "> first", "> second"]);
-        assert!(!has_top_quote);
+        assert!(top_quote.is_none());
     }
 
     #[test]
@@ -312,41 +322,41 @@ mod tests {
     #[test]
     fn test_remove_message_footer() {
         let input = "text\n--\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n--\nno footer");
 
         let input = "text\n\n--\n\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\n\nno footer");
 
         let input = "text\n\n-- no footer\n\n".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n\n-- no footer");
 
         let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\nno footer");
 
         let input = "text\n\n--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, _) = simplify(input.clone(), true);
         assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
+        let (plain, _, _) = simplify(escaped, true);
         assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
 
         // Nonstandard footer sent by https://siju.es/
         let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
-        let (plain, _) = simplify(input.clone(), false);
+        let (plain, _, _) = simplify(input.clone(), false);
         assert_eq!(plain, "Message text here [...]");
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, _) = simplify(input.clone(), true);
         assert_eq!(plain, input);
 
         let input = "--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, _) = simplify(input.clone(), true);
         assert_eq!(plain, ""); // see remove_message_footer() for some explanations
 
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
+        let (plain, _, _) = simplify(escaped, true);
         assert_eq!(plain, "--\ntreated as footer when unescaped");
     }
 }

src/smtp/mod.rs

@@ -30,7 +30,7 @@ pub enum Error {
         error: error::Error,
     },
 
-    #[error("SMTP: failed to connect: {0:?}")]
+    #[error("SMTP: failed to connect: {0}")]
     ConnectionFailure(#[source] smtp::error::Error),
 
     #[error("SMTP: failed to setup connection {0:?}")]
@@ -101,13 +101,26 @@ impl Smtp {
         }
 
         let lp = LoginParam::from_database(context, "configured_").await;
-        self.connect(
-            context,
-            &lp.smtp,
-            &lp.addr,
-            lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
-        )
-        .await
+        let res = self
+            .connect(
+                context,
+                &lp.smtp,
+                &lp.addr,
+                lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+            )
+            .await;
+        if let Err(ref err) = res {
+            let message = context
+                .stock_string_repl_str2(
+                    StockMessage::ServerResponse,
+                    format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
+                    err.to_string(),
+                )
+                .await;
+
+            context.emit_event(EventType::ErrorNetwork(message));
+        };
+        res
     }
 
     /// Connect using the provided login params.
@@ -124,7 +137,6 @@ impl Smtp {
         }
 
         if lp.server.is_empty() || lp.port == 0 {
-            context.emit_event(EventType::ErrorNetwork("SMTP bad parameters.".into()));
             return Err(Error::BadParameters);
         }
 
@@ -197,15 +209,6 @@ impl Smtp {
 
         let mut trans = client.into_transport();
         if let Err(err) = trans.connect().await {
-            let message = context
-                .stock_string_repl_str2(
-                    StockMessage::ServerResponse,
-                    format!("SMTP {}:{}", domain, port),
-                    err.to_string(),
-                )
-                .await;
-
-            emit_event!(context, EventType::ErrorNetwork(message));
             return Err(Error::ConnectionFailure(err));
         }
 

src/smtp/send.rs

@@ -5,6 +5,7 @@ use async_smtp::*;
 
 use crate::context::Context;
 use crate::events::EventType;
+use itertools::Itertools;
 use std::time::Duration;
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -33,11 +34,7 @@ impl Smtp {
     ) -> Result<()> {
         let message_len_bytes = message.len();
 
-        let recipients_display = recipients
-            .iter()
-            .map(|x| format!("{}", x))
-            .collect::<Vec<String>>()
-            .join(",");
+        let recipients_display = recipients.iter().map(|x| x.to_string()).join(",");
 
         let envelope =
             Envelope::new(self.from.clone(), recipients).map_err(Error::EnvelopeError)?;

src/sql.rs

@@ -14,6 +14,7 @@ use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH};
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::ephemeral::start_ephemeral_timers;
+use crate::error::format_err;
 use crate::param::*;
 use crate::peerstate::*;
 
@@ -77,18 +78,29 @@ impl Sql {
         // drop closes the connection
     }
 
-    // return true on success, false on failure
-    pub async fn open<T: AsRef<Path>>(&self, context: &Context, dbfile: T, readonly: bool) -> bool {
-        match open(context, self, dbfile, readonly).await {
-            Ok(_) => true,
-            Err(err) => match err.downcast_ref::<Error>() {
-                Some(Error::SqlAlreadyOpen) => false,
+    pub async fn open<T: AsRef<Path>>(
+        &self,
+        context: &Context,
+        dbfile: T,
+        readonly: bool,
+    ) -> crate::error::Result<()> {
+        let res = open(context, self, &dbfile, readonly).await;
+        if let Err(err) = &res {
+            match err.downcast_ref::<Error>() {
+                Some(Error::SqlAlreadyOpen) => {}
                 _ => {
                     self.close().await;
-                    false
                 }
-            },
+            }
         }
+        res.map_err(|e| {
+            format_err!(
+                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                "Could not open db file {}: {:#}",
+                dbfile.as_ref().to_string_lossy(),
+                e
+            )
+        })
     }
 
     pub async fn execute<S: AsRef<str>>(
@@ -130,7 +142,7 @@ impl Sql {
         &self,
     ) -> Result<r2d2::PooledConnection<r2d2_sqlite::SqliteConnectionManager>> {
         let lock = self.pool.read().await;
-        let pool = lock.as_ref().ok_or_else(|| Error::SqlNoConnection)?;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
         let conn = pool.get()?;
 
         Ok(conn)
@@ -144,7 +156,7 @@ impl Sql {
             + FnOnce(r2d2::PooledConnection<r2d2_sqlite::SqliteConnectionManager>) -> Result<H>,
     {
         let lock = self.pool.read().await;
-        let pool = lock.as_ref().ok_or_else(|| Error::SqlNoConnection)?;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
         let conn = pool.get()?;
 
         g(conn)
@@ -156,7 +168,7 @@ impl Sql {
         Fut: Future<Output = Result<H>> + Send,
     {
         let lock = self.pool.read().await;
-        let pool = lock.as_ref().ok_or_else(|| Error::SqlNoConnection)?;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
 
         let conn = pool.get()?;
         g(conn).await
@@ -666,7 +678,10 @@ async fn open(
         .with_flags(open_flags)
         .with_init(|c| {
             c.execute_batch(&format!(
-                "PRAGMA secure_delete=on; PRAGMA busy_timeout = {};",
+                "PRAGMA secure_delete=on;
+                 PRAGMA busy_timeout = {};
+                 PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
+                 ",
                 Duration::from_secs(10).as_millis()
             ))?;
             Ok(())
@@ -693,156 +708,208 @@ async fn open(
             .ok();
 
         let mut exists_before_update = false;
-        let mut dbversion_before_update: i32 = 0;
-        /* Init tables to dbversion=0 */
+        // Init tables to dbversion=68
+        let mut dbversion_before_update: i32 = 68;
         if !sql.table_exists("config").await? {
             info!(
                 context,
                 "First time init: creating tables in {:?}.",
                 dbfile.as_ref(),
             );
-            sql.execute(
-                "CREATE TABLE config (id INTEGER PRIMARY KEY, keyname TEXT, value TEXT);",
-                paramsv![],
-            )
+            sql.with_conn(move |mut conn| {
+                let tx = conn.transaction()?;
+                tx.execute_batch(
+                    r#"
+CREATE TABLE config (id INTEGER PRIMARY KEY, keyname TEXT, value TEXT);
+CREATE INDEX config_index1 ON config (keyname);
+CREATE TABLE contacts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT DEFAULT '',
+    addr TEXT DEFAULT '' COLLATE NOCASE,
+    origin INTEGER DEFAULT 0,
+    blocked INTEGER DEFAULT 0,
+    last_seen INTEGER DEFAULT 0,
+    param TEXT DEFAULT '',
+    authname TEXT DEFAULT '',
+    selfavatar_sent INTEGER DEFAULT 0
+);
+CREATE INDEX contacts_index1 ON contacts (name COLLATE NOCASE);
+CREATE INDEX contacts_index2 ON contacts (addr COLLATE NOCASE);
+INSERT INTO contacts (id,name,origin) VALUES
+(1,'self',262144), (2,'info',262144), (3,'rsvd',262144),
+(4,'rsvd',262144), (5,'device',262144), (6,'rsvd',262144),
+(7,'rsvd',262144), (8,'rsvd',262144), (9,'rsvd',262144);
+
+CREATE TABLE chats (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    type INTEGER DEFAULT 0,
+    name TEXT DEFAULT '',
+    draft_timestamp INTEGER DEFAULT 0,
+    draft_txt TEXT DEFAULT '',
+    blocked INTEGER DEFAULT 0,
+    grpid TEXT DEFAULT '',
+    param TEXT DEFAULT '',
+    archived INTEGER DEFAULT 0,
+    gossiped_timestamp INTEGER DEFAULT 0,
+    locations_send_begin INTEGER DEFAULT 0,
+    locations_send_until INTEGER DEFAULT 0,
+    locations_last_sent INTEGER DEFAULT 0,
+    created_timestamp INTEGER DEFAULT 0,
+    muted_until INTEGER DEFAULT 0,
+    ephemeral_timer INTEGER
+);
+CREATE INDEX chats_index1 ON chats (grpid);
+CREATE INDEX chats_index2 ON chats (archived);
+CREATE INDEX chats_index3 ON chats (locations_send_until);
+INSERT INTO chats (id,type,name) VALUES
+(1,120,'deaddrop'), (2,120,'rsvd'), (3,120,'trash'),
+(4,120,'msgs_in_creation'), (5,120,'starred'), (6,120,'archivedlink'),
+(7,100,'rsvd'), (8,100,'rsvd'), (9,100,'rsvd');
+
+CREATE TABLE chats_contacts (chat_id INTEGER, contact_id INTEGER);
+CREATE INDEX chats_contacts_index1 ON chats_contacts (chat_id);
+CREATE INDEX chats_contacts_index2 ON chats_contacts (contact_id);
+
+CREATE TABLE msgs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    rfc724_mid TEXT DEFAULT '',
+    server_folder TEXT DEFAULT '',
+    server_uid INTEGER DEFAULT 0,
+    chat_id INTEGER DEFAULT 0,
+    from_id INTEGER DEFAULT 0,
+    to_id INTEGER DEFAULT 0,
+    timestamp INTEGER DEFAULT 0,
+    type INTEGER DEFAULT 0,
+    state INTEGER DEFAULT 0,
+    msgrmsg INTEGER DEFAULT 1,
+    bytes INTEGER DEFAULT 0,
+    txt TEXT DEFAULT '',
+    txt_raw TEXT DEFAULT '',
+    param TEXT DEFAULT '',
+    starred INTEGER DEFAULT 0,
+    timestamp_sent INTEGER DEFAULT 0,
+    timestamp_rcvd INTEGER DEFAULT 0,
+    hidden INTEGER DEFAULT 0,
+    mime_headers TEXT,
+    mime_in_reply_to TEXT,
+    mime_references TEXT,
+    move_state INTEGER DEFAULT 1,
+    location_id INTEGER DEFAULT 0,
+    error TEXT DEFAULT '',
+
+-- Timer value in seconds. For incoming messages this
+-- timer starts when message is read, so we want to have
+-- the value stored here until the timer starts.
+    ephemeral_timer INTEGER DEFAULT 0,
+
+-- Timestamp indicating when the message should be
+-- deleted. It is convenient to store it here because UI
+-- needs this value to display how much time is left until
+-- the message is deleted.
+    ephemeral_timestamp INTEGER DEFAULT 0
+);
+
+CREATE INDEX msgs_index1 ON msgs (rfc724_mid);
+CREATE INDEX msgs_index2 ON msgs (chat_id);
+CREATE INDEX msgs_index3 ON msgs (timestamp);
+CREATE INDEX msgs_index4 ON msgs (state);
+CREATE INDEX msgs_index5 ON msgs (starred);
+CREATE INDEX msgs_index6 ON msgs (location_id);
+CREATE INDEX msgs_index7 ON msgs (state, hidden, chat_id);
+INSERT INTO msgs (id,msgrmsg,txt) VALUES
+(1,0,'marker1'), (2,0,'rsvd'), (3,0,'rsvd'),
+(4,0,'rsvd'), (5,0,'rsvd'), (6,0,'rsvd'), (7,0,'rsvd'),
+(8,0,'rsvd'), (9,0,'daymarker');
+
+CREATE TABLE jobs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    added_timestamp INTEGER,
+    desired_timestamp INTEGER DEFAULT 0,
+    action INTEGER,
+    foreign_id INTEGER,
+    param TEXT DEFAULT '',
+    thread INTEGER DEFAULT 0,
+    tries INTEGER DEFAULT 0
+);
+CREATE INDEX jobs_index1 ON jobs (desired_timestamp);
+
+CREATE TABLE leftgrps (
+    id INTEGER PRIMARY KEY,
+    grpid TEXT DEFAULT ''
+);
+CREATE INDEX leftgrps_index1 ON leftgrps (grpid);
+
+CREATE TABLE keypairs (
+    id INTEGER PRIMARY KEY,
+    addr TEXT DEFAULT '' COLLATE NOCASE,
+    is_default INTEGER DEFAULT 0,
+    private_key,
+    public_key,
+    created INTEGER DEFAULT 0
+);
+
+CREATE TABLE acpeerstates (
+    id INTEGER PRIMARY KEY,
+    addr TEXT DEFAULT '' COLLATE NOCASE,
+    last_seen INTEGER DEFAULT 0,
+    last_seen_autocrypt INTEGER DEFAULT 0,
+    public_key,
+    prefer_encrypted INTEGER DEFAULT 0,
+    gossip_timestamp INTEGER DEFAULT 0,
+    gossip_key,
+    public_key_fingerprint TEXT DEFAULT '',
+    gossip_key_fingerprint TEXT DEFAULT '',
+    verified_key,
+    verified_key_fingerprint TEXT DEFAULT ''
+);
+CREATE INDEX acpeerstates_index1 ON acpeerstates (addr);
+CREATE INDEX acpeerstates_index3 ON acpeerstates (public_key_fingerprint);
+CREATE INDEX acpeerstates_index4 ON acpeerstates (gossip_key_fingerprint);
+CREATE INDEX acpeerstates_index5 ON acpeerstates (verified_key_fingerprint);
+
+CREATE TABLE msgs_mdns (
+    msg_id INTEGER,
+    contact_id INTEGER,
+    timestamp_sent INTEGER DEFAULT 0
+);
+CREATE INDEX msgs_mdns_index1 ON msgs_mdns (msg_id);
+
+CREATE TABLE tokens (
+    id INTEGER PRIMARY KEY,
+    namespc INTEGER DEFAULT 0,
+    foreign_id INTEGER DEFAULT 0,
+    token TEXT DEFAULT '',
+    timestamp INTEGER DEFAULT 0
+);
+
+CREATE TABLE locations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    latitude REAL DEFAULT 0.0,
+    longitude REAL DEFAULT 0.0,
+    accuracy REAL DEFAULT 0.0,
+    timestamp INTEGER DEFAULT 0,
+    chat_id INTEGER DEFAULT 0,
+    from_id INTEGER DEFAULT 0,
+    independent INTEGER DEFAULT 0
+);
+CREATE INDEX locations_index1 ON locations (from_id);
+CREATE INDEX locations_index2 ON locations (timestamp);
+
+CREATE TABLE devmsglabels (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    label TEXT,
+    msg_id INTEGER DEFAULT 0
+);
+CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
+"#,
+                )?;
+                tx.commit()?;
+                Ok(())
+            })
             .await?;
-            sql.execute(
-                "CREATE INDEX config_index1 ON config (keyname);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE contacts (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT, \
-                 name TEXT DEFAULT '', \
-                 addr TEXT DEFAULT '' COLLATE NOCASE, \
-                 origin INTEGER DEFAULT 0, \
-                 blocked INTEGER DEFAULT 0, \
-                 last_seen INTEGER DEFAULT 0, \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX contacts_index1 ON contacts (name COLLATE NOCASE);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX contacts_index2 ON contacts (addr COLLATE NOCASE);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "INSERT INTO contacts (id,name,origin) VALUES \
-                 (1,'self',262144), (2,'info',262144), (3,'rsvd',262144), \
-                 (4,'rsvd',262144), (5,'device',262144), (6,'rsvd',262144), \
-                 (7,'rsvd',262144), (8,'rsvd',262144), (9,'rsvd',262144);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE chats (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT,  \
-                 type INTEGER DEFAULT 0, \
-                 name TEXT DEFAULT '', \
-                 draft_timestamp INTEGER DEFAULT 0, \
-                 draft_txt TEXT DEFAULT '', \
-                 blocked INTEGER DEFAULT 0, \
-                 grpid TEXT DEFAULT '', \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute("CREATE INDEX chats_index1 ON chats (grpid);", paramsv![])
+
+            sql.set_raw_config_int(context, "dbversion", dbversion_before_update)
                 .await?;
-            sql.execute(
-                "CREATE TABLE chats_contacts (chat_id INTEGER, contact_id INTEGER);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX chats_contacts_index1 ON chats_contacts (chat_id);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "INSERT INTO chats (id,type,name) VALUES \
-                 (1,120,'deaddrop'), (2,120,'rsvd'), (3,120,'trash'), \
-                 (4,120,'msgs_in_creation'), (5,120,'starred'), (6,120,'archivedlink'), \
-                 (7,100,'rsvd'), (8,100,'rsvd'), (9,100,'rsvd');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE msgs (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT, \
-                 rfc724_mid TEXT DEFAULT '', \
-                 server_folder TEXT DEFAULT '', \
-                 server_uid INTEGER DEFAULT 0, \
-                 chat_id INTEGER DEFAULT 0, \
-                 from_id INTEGER DEFAULT 0, \
-                 to_id INTEGER DEFAULT 0, \
-                 timestamp INTEGER DEFAULT 0, \
-                 type INTEGER DEFAULT 0, \
-                 state INTEGER DEFAULT 0, \
-                 msgrmsg INTEGER DEFAULT 1, \
-                 bytes INTEGER DEFAULT 0, \
-                 txt TEXT DEFAULT '', \
-                 txt_raw TEXT DEFAULT '', \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute("CREATE INDEX msgs_index1 ON msgs (rfc724_mid);", paramsv![])
-                .await?;
-            sql.execute("CREATE INDEX msgs_index2 ON msgs (chat_id);", paramsv![])
-                .await?;
-            sql.execute("CREATE INDEX msgs_index3 ON msgs (timestamp);", paramsv![])
-                .await?;
-            sql.execute("CREATE INDEX msgs_index4 ON msgs (state);", paramsv![])
-                .await?;
-            sql.execute(
-                "INSERT INTO msgs (id,msgrmsg,txt) VALUES \
-                 (1,0,'marker1'), (2,0,'rsvd'), (3,0,'rsvd'), \
-                 (4,0,'rsvd'), (5,0,'rsvd'), (6,0,'rsvd'), (7,0,'rsvd'), \
-                 (8,0,'rsvd'), (9,0,'daymarker');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE jobs (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT, \
-                 added_timestamp INTEGER, \
-                 desired_timestamp INTEGER DEFAULT 0, \
-                 action INTEGER, \
-                 foreign_id INTEGER, \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX jobs_index1 ON jobs (desired_timestamp);",
-                paramsv![],
-            )
-            .await?;
-            if !sql.table_exists("config").await?
-                || !sql.table_exists("contacts").await?
-                || !sql.table_exists("chats").await?
-                || !sql.table_exists("chats_contacts").await?
-                || !sql.table_exists("msgs").await?
-                || !sql.table_exists("jobs").await?
-            {
-                error!(
-                    context,
-                    "Cannot create tables in new database \"{:?}\".",
-                    dbfile.as_ref(),
-                );
-                // cannot create the tables - maybe we cannot write?
-                return Err(Error::SqlFailedToOpen.into());
-            } else {
-                sql.set_raw_config_int(context, "dbversion", 0).await?;
-            }
         } else {
             exists_before_update = true;
             dbversion_before_update = sql
@@ -858,7 +925,7 @@ async fn open(
 
         let mut dbversion = dbversion_before_update;
         let mut recalc_fingerprints = false;
-        let mut update_icons = false;
+        let mut update_icons = !exists_before_update;
 
         if dbversion < 1 {
             info!(context, "[migration] v1");
@@ -872,7 +939,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 1;
             sql.set_raw_config_int(context, "dbversion", 1).await?;
         }
         if dbversion < 2 {
@@ -882,7 +948,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 2;
             sql.set_raw_config_int(context, "dbversion", 2).await?;
         }
         if dbversion < 7 {
@@ -898,7 +963,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 7;
             sql.set_raw_config_int(context, "dbversion", 7).await?;
         }
         if dbversion < 10 {
@@ -919,7 +983,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 10;
             sql.set_raw_config_int(context, "dbversion", 10).await?;
         }
         if dbversion < 12 {
@@ -934,7 +997,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 12;
             sql.set_raw_config_int(context, "dbversion", 12).await?;
         }
         if dbversion < 17 {
@@ -946,6 +1008,8 @@ async fn open(
             .await?;
             sql.execute("CREATE INDEX chats_index2 ON chats (archived);", paramsv![])
                 .await?;
+            // 'starred' column is not used currently
+            // (dropping is not easily doable and stop adding it will make reusing it complicated)
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN starred INTEGER DEFAULT 0;",
                 paramsv![],
@@ -953,7 +1017,6 @@ async fn open(
             .await?;
             sql.execute("CREATE INDEX msgs_index5 ON msgs (starred);", paramsv![])
                 .await?;
-            dbversion = 17;
             sql.set_raw_config_int(context, "dbversion", 17).await?;
         }
         if dbversion < 18 {
@@ -968,7 +1031,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 18;
             sql.set_raw_config_int(context, "dbversion", 18).await?;
         }
         if dbversion < 27 {
@@ -992,7 +1054,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 27;
             sql.set_raw_config_int(context, "dbversion", 27).await?;
         }
         if dbversion < 34 {
@@ -1028,7 +1089,6 @@ async fn open(
             )
             .await?;
             recalc_fingerprints = true;
-            dbversion = 34;
             sql.set_raw_config_int(context, "dbversion", 34).await?;
         }
         if dbversion < 39 {
@@ -1052,7 +1112,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 39;
             sql.set_raw_config_int(context, "dbversion", 39).await?;
         }
         if dbversion < 40 {
@@ -1062,14 +1121,12 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 40;
             sql.set_raw_config_int(context, "dbversion", 40).await?;
         }
         if dbversion < 44 {
             info!(context, "[migration] v44");
             sql.execute("ALTER TABLE msgs ADD COLUMN mime_headers TEXT;", paramsv![])
                 .await?;
-            dbversion = 44;
             sql.set_raw_config_int(context, "dbversion", 44).await?;
         }
         if dbversion < 46 {
@@ -1094,7 +1151,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 47;
             sql.set_raw_config_int(context, "dbversion", 47).await?;
         }
         if dbversion < 48 {
@@ -1105,8 +1161,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-
-            dbversion = 48;
             sql.set_raw_config_int(context, "dbversion", 48).await?;
         }
         if dbversion < 49 {
@@ -1116,7 +1170,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 49;
             sql.set_raw_config_int(context, "dbversion", 49).await?;
         }
         if dbversion < 50 {
@@ -1128,7 +1181,6 @@ async fn open(
                 sql.set_raw_config_int(context, "show_emails", ShowEmails::All as i32)
                     .await?;
             }
-            dbversion = 50;
             sql.set_raw_config_int(context, "dbversion", 50).await?;
         }
         if dbversion < 53 {
@@ -1169,7 +1221,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 53;
             sql.set_raw_config_int(context, "dbversion", 53).await?;
         }
         if dbversion < 54 {
@@ -1184,7 +1235,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 54;
             sql.set_raw_config_int(context, "dbversion", 54).await?;
         }
         if dbversion < 55 {
@@ -1264,18 +1314,11 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            // Timer value in seconds. For incoming messages this
-            // timer starts when message is read, so we want to have
-            // the value stored here until the timer starts.
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN ephemeral_timer INTEGER DEFAULT 0",
                 paramsv![],
             )
             .await?;
-            // Timestamp indicating when the message should be
-            // deleted. It is convenient to store it here because UI
-            // needs this value to display how much time is left until
-            // the message is deleted.
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0",
                 paramsv![],
@@ -1317,7 +1360,7 @@ async fn open(
         }
         if dbversion < 68 {
             info!(context, "[migration] v68");
-            // the index is used to speed up get_fresh_msg_cnt(), see comment there for more details
+            // the index is used to speed up get_fresh_msg_cnt() (see comment there for more details) and marknoticed_chat()
             sql.execute(
                 "CREATE INDEX IF NOT EXISTS msgs_index7 ON msgs (state, hidden, chat_id);",
                 paramsv![],

src/stock.rs

@@ -119,9 +119,6 @@ pub enum StockMessage {
     #[strum(props(fallback = "Archived chats"))]
     ArchivedChats = 40,
 
-    #[strum(props(fallback = "Starred messages"))]
-    StarredMsgs = 41,
-
     #[strum(props(fallback = "Autocrypt Setup Message"))]
     AcSetupMsgSubject = 42,
 
@@ -216,6 +213,26 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "You are invited to a video chat, click %1$s to join."))]
     VideochatInviteMsgBody = 83,
+
+    #[strum(props(fallback = "Error:\n\n“%1$s”"))]
+    ConfigurationFailed = 84,
+
+    #[strum(props(
+        fallback = "⚠️ Date or time of your device seem to be inaccurate (%1$s).\n\n\
+                    Adjust your clock ⏰🔧 to ensure your messages are received correctly."
+    ))]
+    BadTimeMsgBody = 85,
+
+    #[strum(props(fallback = "⚠️ Your Delta Chat version might be outdated.\n\n\
+                    This may cause problems because your chat partners use newer versions - \
+                    and you are missing the latest features 😳\n\
+                    Please check https://get.delta.chat or your app store for updates."))]
+    UpdateReminderMsgBody = 86,
+
+    #[strum(props(
+        fallback = "Could not find your mail server.\n\nPlease check your internet connection."
+    ))]
+    ErrorNoNetwork = 87,
 }
 
 /*
@@ -384,7 +401,7 @@ impl Context {
     pub async fn update_device_chats(&self) -> Result<(), Error> {
         // check for the LAST added device message - if it is present, we can skip message creation.
         // this is worthwhile as this function is typically called
-        // by the ui on every probram start or even on every opening of the chatlist.
+        // by the UI on every program start or even on every opening of the chatlist.
         if chat::was_device_msg_ever_added(&self, "core-welcome").await? {
             return Ok(());
         }

src/test_utils.rs

@@ -2,20 +2,33 @@
 //!
 //! This module is only compiled for test runs.
 
+use std::str::FromStr;
+use std::time::{Duration, Instant};
+
+use async_std::path::PathBuf;
+use async_std::sync::RwLock;
 use tempfile::{tempdir, TempDir};
 
+use crate::chat::ChatId;
 use crate::config::Config;
 use crate::context::Context;
+use crate::dc_receive_imf::dc_receive_imf;
 use crate::dc_tools::EmailAddress;
+use crate::job::Action;
 use crate::key::{self, DcKey};
+use crate::mimeparser::MimeMessage;
+use crate::param::{Param, Params};
 
 /// A Context and temporary directory.
 ///
 /// The temporary directory can be used to store the SQLite database,
 /// see e.g. [test_context] which does this.
+#[derive(Debug)]
 pub(crate) struct TestContext {
     pub ctx: Context,
     pub dir: TempDir,
+    /// Counter for fake IMAP UIDs in [recv_msg], for private use in that function only.
+    recv_idx: RwLock<u32>,
 }
 
 impl TestContext {
@@ -35,7 +48,11 @@ impl TestContext {
         let ctx = Context::new("FakeOS".into(), dbfile.into(), id)
             .await
             .unwrap();
-        Self { ctx, dir }
+        Self {
+            ctx,
+            dir,
+            recv_idx: RwLock::new(0),
+        }
     }
 
     /// Create a new configured [TestContext].
@@ -48,6 +65,19 @@ impl TestContext {
         t
     }
 
+    /// Create a new configured [TestContext].
+    ///
+    /// This is a shortcut which configures bob@example.net with a fixed key.
+    pub async fn new_bob() -> Self {
+        let t = Self::new().await;
+        let keypair = bob_keypair();
+        t.configure_addr(&keypair.addr.to_string()).await;
+        key::store_self_keypair(&t.ctx, &keypair, key::KeyPairUse::Default)
+            .await
+            .expect("Failed to save Bob's key");
+        t
+    }
+
     /// Configure with alice@example.com.
     ///
     /// The context will be fake-configured as the alice user, with a pre-generated secret
@@ -76,6 +106,122 @@ impl TestContext {
             .await
             .unwrap();
     }
+
+    /// Retrieve a sent message from the jobs table.
+    ///
+    /// This retrieves and removes a message which has been scheduled to send from the jobs
+    /// table.  Messages are returned in the order they have been sent.
+    ///
+    /// Panics if there is no message or on any error.
+    pub async fn pop_sent_msg(&self) -> SentMessage {
+        let start = Instant::now();
+        let (rowid, foreign_id, raw_params) = loop {
+            let row = self
+                .ctx
+                .sql
+                .query_row(
+                    r#"
+                    SELECT id, foreign_id, param
+                      FROM jobs
+                     WHERE action=?
+                  ORDER BY desired_timestamp;
+                "#,
+                    paramsv![Action::SendMsgToSmtp],
+                    |row| {
+                        let id: i64 = row.get(0)?;
+                        let foreign_id: i64 = row.get(1)?;
+                        let param: String = row.get(2)?;
+                        Ok((id, foreign_id, param))
+                    },
+                )
+                .await;
+            if let Ok(row) = row {
+                break row;
+            }
+            if start.elapsed() < Duration::from_secs(3) {
+                async_std::task::sleep(Duration::from_millis(100)).await;
+            } else {
+                panic!("no sent message found in jobs table");
+            }
+        };
+        let id = ChatId::new(foreign_id as u32);
+        let params = Params::from_str(&raw_params).unwrap();
+        let blob_path = params
+            .get_blob(Param::File, &self.ctx, false)
+            .await
+            .expect("failed to parse blob from param")
+            .expect("no Param::File found in Params")
+            .to_abs_path();
+        self.ctx
+            .sql
+            .execute("DELETE FROM jobs WHERE id=?;", paramsv![rowid])
+            .await
+            .expect("failed to remove job");
+        SentMessage {
+            id,
+            params,
+            blob_path,
+        }
+    }
+
+    /// Parse a message.
+    ///
+    /// Parsing a message does not run the entire receive pipeline, but is not without
+    /// side-effects either.  E.g. if the message includes autocrypt headers the relevant
+    /// peerstates will be updated.  Later receiving the message using [recv_msg] is
+    /// unlikely to be affected as the peerstate would be processed again in exactly the
+    /// same way.
+    pub async fn parse_msg(&self, msg: &SentMessage) -> MimeMessage {
+        MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes())
+            .await
+            .unwrap()
+    }
+
+    /// Receive a message.
+    ///
+    /// Receives a message using the `dc_receive_imf()` pipeline.
+    pub async fn recv_msg(&self, msg: &SentMessage) {
+        let mut idx = self.recv_idx.write().await;
+        *idx += 1;
+        dc_receive_imf(&self.ctx, msg.payload().as_bytes(), "INBOX", *idx, false)
+            .await
+            .unwrap();
+    }
+}
+
+/// A raw message as it was scheduled to be sent.
+///
+/// This is a raw message, probably in the shape DC was planning to send it but not having
+/// passed through a SMTP-IMAP pipeline.
+#[derive(Debug, Clone)]
+pub struct SentMessage {
+    id: ChatId,
+    params: Params,
+    blob_path: PathBuf,
+}
+
+impl SentMessage {
+    /// The ChatId the message belonged to.
+    pub fn id(&self) -> ChatId {
+        self.id
+    }
+
+    /// A recipient the message was destined for.
+    ///
+    /// If there are multiple recipients this is just a random one, so is not very useful.
+    pub fn recipient(&self) -> EmailAddress {
+        let raw = self
+            .params
+            .get(Param::Recipients)
+            .expect("no recipients in params");
+        let rcpt = raw.split(' ').next().expect("no recipient found");
+        rcpt.parse().expect("failed to parse email address")
+    }
+
+    /// The raw message payload.
+    pub fn payload(&self) -> String {
+        std::fs::read_to_string(&self.blob_path).unwrap()
+    }
 }
 
 /// Load a pre-generated keypair for alice@example.com from disk.

tests/stress.rs

@@ -2,7 +2,7 @@
 
 use deltachat::config;
 use deltachat::context::*;
-use tempfile::{tempdir, TempDir};
+use tempfile::tempdir;
 
 /* some data used for testing
  ******************************************************************************/
@@ -92,26 +92,19 @@ async fn stress_functions(context: &Context) {
     // free(qr.cast());
 }
 
-#[allow(dead_code)]
-struct TestContext {
-    ctx: Context,
-    dir: TempDir,
-}
-
-async fn create_test_context() -> TestContext {
+async fn create_test_context() -> Context {
     use rand::Rng;
 
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = rand::thread_rng().gen();
-    let ctx = Context::new("FakeOs".into(), dbfile.into(), id)
+    Context::new("FakeOs".into(), dbfile.into(), id)
         .await
-        .unwrap();
-    TestContext { ctx, dir }
+        .unwrap()
 }
 
 #[async_std::test]
 async fn test_stress_tests() {
     let context = create_test_context().await;
-    stress_functions(&context.ctx).await;
+    stress_functions(&context).await;
 }