re-enabling smtp_config_task - still fine

Previously empty autoconfig resulted in no servers being tried and no
error displayed.

.circleci/config.yml

@@ -189,8 +189,6 @@ workflows:
               only: /.*/
 
       - remote_python_packaging:
-          requires: 
-             - remote_tests_python
           filters:
             branches:
               only: master

.github/workflows/ci.yml

@@ -47,7 +47,9 @@ jobs:
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        # macOS disabled due to random failures related to caching
+        #os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest]
         rust: [1.45.0]
         experimental: [false]
 #        include:

CHANGELOG.md

@@ -1,5 +1,74 @@
 # Changelog 
 
+## 1.47.0
+
+- breaking change: `dc_update_device_chats()` removed;
+  this is now done automatically during configure
+  unless the new config-option `bot` is set #1957
+
+- breaking change: split `DC_EVENT_MSGS_NOTICED` off `DC_EVENT_MSGS_CHANGED`
+  and remove `dc_marknoticed_all_chats()` #1942 #1981
+
+- breaking change: remove unused starring options #1965
+
+- breaking change: `DC_CHAT_TYPE_VERIFIED_GROUP` replaced by
+  `dc_chat_is_protected()`; also single-chats may be protected now, this may
+  happen over the wire even if the UI do not offer an option for that #1968
+
+- breaking change: split quotes off message text,
+  UIs should use at least `dc_msg_get_quoted_text()` to show quotes now #1975
+
+- new api for quote handling: `dc_msg_set_quote()`, `dc_msg_get_quoted_text()`,
+  `dc_msg_get_quoted_msg()` #1975 #1984 #1985 #1987 #1989 #2004
+
+- require quorum to enable encryption #1946
+
+- speed up and clean up account creation #1912 #1927 #1960 #1961
+
+- configure now collects recent contacts and fetches last messages
+  unless disabled by `fetch_existing` config-option #1913 #2003
+
+- emit `DC_EVENT_CHAT_MODIFIED` on contact rename
+  and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
+
+- add `dc_set_chat_protection()`; the `protect` parameter in
+  `dc_create_group_chat()` will be removed in an upcoming release;
+  up to then, UIs using the "verified group" paradigm
+  should not use `dc_set_chat_protection()` #1968 #2014 #2001 #2012 #2007
+
+- remove unneeded `DC_STR_COUNT` #1991
+
+- mark all failed messages as failed when receiving an NDN #1993
+
+- fix import temporary directory usage #1929
+
+- fix forcing encryption for reset peers #1998
+
+- fix: do not allow to save drafts in non-writeable chats #1997
+
+- fix: do not show HTML if there is no content and there is an attachment #1988
+
+- fix recovering offline/lost connections, fixes background receive bug #1983
+
+- fix ordering of accounts returned by `dc_accounts_get_all()` #1909
+
+- fix whitespace for summaries #1938
+
+- fix: improve sentbox name guessing #1941
+
+- fix: avoid manual poll impl for accounts events #1944
+
+- fix encoding newlines in param as a preparation for storing quotes #1945
+
+- fix: internal and ffi error handling #1967 #1966 #1959 #1911 #1916 #1917 #1915
+
+- fix ci #1928 #1931 #1932 #1933 #1934 #1943
+
+- update provider-database #1940 #2005 #2006
+
+- update dependencies #1919 #1908 #1950 #1963 #1996 #2010 #2013
+
+
 ## 1.46.0
 
 - breaking change: `dc_configure()` report errors in
@@ -745,4 +814,3 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
-

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.46.0"
+version = "1.47.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,12 +12,12 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.6.0", default-features = false } 
+pgp = { version = "0.7.0", default-features = false } 
 hex = "0.4.0"
 sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "=2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
 async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
@@ -25,7 +25,7 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.4.0"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.1", features = ["unstable"] }
+async-std = { version = "1.6.4", features = ["unstable"] }
 base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -34,22 +34,22 @@ serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
 kamadak-exif = "0.5"
-lazy_static = "1.4.0"
+once_cell = "1.4.1"
 regex = "1.1.6"
-rusqlite = { version = "0.23", features = ["bundled"] }
-r2d2_sqlite = "0.16.0"
+rusqlite = { version = "0.24", features = ["bundled"] }
+r2d2_sqlite = "0.17.0"
 r2d2 = "0.8.5"
-strum = "0.18.0"
-strum_macros = "0.18.0"
+strum = "0.19.0"
+strum_macros = "0.19.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.9.0"
 quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-sanitize-filename = "0.2.1"
+sanitize-filename = "0.3.0"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.1"
+mailparse = "0.13.0"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
@@ -75,8 +75,9 @@ tempfile = "3.0"
 pretty_assertions = "0.6.1"
 pretty_env_logger = "0.4.0"
 proptest = "0.10"
-async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
-smol = "0.1.10"
+async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
+futures-lite = "1.7.0"
+criterion = "0.3"
 
 [workspace]
 members = [
@@ -95,6 +96,10 @@ path = "examples/repl/main.rs"
 required-features = ["repl"]
 
 
+[[bench]]
+name = "create_account"
+harness = false
+
 [features]
 default = []
 internals = []
@@ -102,6 +107,3 @@ repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 
-[patch.crates-io]
-polling = { git = "https://github.com/oblique/polling", branch = "master"}
-async-std = { git = "https://github.com/async-rs/async-std", branch = "master"}

benches/create_account.rs

@@ -0,0 +1,26 @@
+use async_std::path::PathBuf;
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::accounts::Accounts;
+use tempfile::tempdir;
+
+async fn create_accounts(n: u32) {
+    let dir = tempdir().unwrap();
+    let p: PathBuf = dir.path().join("accounts").into();
+
+    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+    for expected_id in 2..n {
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, expected_id);
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("create 1 account", |b| {
+        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

ci_scripts/remote_python_packaging.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"

ci_scripts/remote_tests_python.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
@@ -30,9 +28,6 @@ ssh $SSHTARGET <<_HERE
 
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL

ci_scripts/remote_tests_rust.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 set -e
@@ -24,9 +22,6 @@ ssh $SSHTARGET <<_HERE
     shopt -s huponexit
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=x86_64-unknown-linux-gnu
     export RUSTC_WRAPPER=sccache
 

ci_scripts/run-python-test.sh

@@ -4,6 +4,7 @@
 # and tox/pytest. 
 
 set -e -x
+shopt -s huponexit
 
 # for core-building and python install step
 export DCC_RS_TARGET=debug

ci_scripts/run-rust-test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -ex
+shopt -s huponexit
 
 #export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.46.0"
+version = "1.47.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/Doxyfile

@@ -236,12 +236,6 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
-# This tag can be used to specify a number of word-keyword mappings (TCL only).
-# A mapping has the form "name=value". For example adding "class=itcl::class"
-# will allow you to use the command class in the itcl::class meaning.
-
-TCL_SUBST              =
-
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all

deltachat-ffi/deltachat.h

@@ -218,7 +218,6 @@ dc_context_t*   dc_context_new               (const char* os_name, const char* d
  * @param context The context object as created by dc_context_new(),
  *     dc_accounts_get_account() or dc_accounts_get_selected_account().
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_context_unref             (dc_context_t* context);
 
@@ -350,6 +349,10 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    https://github.com/cracker0dks/basicwebrtc which some UIs have native support for.
  *                    The type `jitsi:` may be handled by external apps.
  *                    If no type is prefixed, the videochat is handled completely in a browser.
+ * - `bot`          = Set to "1" if this is a bot. E.g. prevents adding the "Device messages" and "Saved messages" chats.
+ * - `fetch_existing` = 1=fetch most recent existing messages on configure (default),
+ *                    0=do not fetch existing messages on configure.
+ *                    In both cases, existing recipients are added to the contact database.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -432,7 +435,7 @@ int             dc_set_config_from_qr   (dc_context_t* context, const char* qr);
  * @param context The context object.
  * @return String which must be released using dc_str_unref() after usage.  Never returns NULL.
  */
-char*           dc_get_info                  (dc_context_t* context);
+char*           dc_get_info                  (const dc_context_t* context);
 
 
 /**
@@ -506,7 +509,6 @@ char*           dc_get_oauth2_url            (dc_context_t* context, const char*
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @return None.
  *
  * There is no need to call dc_configure() on every program start,
  * the configuration result is saved in the database
@@ -546,7 +548,6 @@ int             dc_is_configured   (const dc_context_t* context);
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
- * @return None
  */
 void            dc_start_io     (dc_context_t* context);
 
@@ -570,7 +571,6 @@ int             dc_is_io_running(const dc_context_t* context);
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
- * @return None
  */
 void            dc_stop_io(dc_context_t* context);
 
@@ -595,7 +595,6 @@ void            dc_stop_io(dc_context_t* context);
  *
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
- * @return None.
  */
 void            dc_maybe_network             (dc_context_t* context);
 
@@ -936,7 +935,6 @@ uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
  *     NULL deletes the existing draft, if any, without sending it.
  *     Currently, also non-text-messages
  *     will delete the existing drafts.
- * @return None.
  */
 void            dc_set_draft                 (dc_context_t* context, uint32_t chat_id, dc_msg_t* msg);
 
@@ -990,25 +988,6 @@ void            dc_set_draft                 (dc_context_t* context, uint32_t ch
  */
 uint32_t        dc_add_device_msg            (dc_context_t* context, const char* label, dc_msg_t* msg);
 
-
-/**
- * Init device-messages and saved-messages chat.
- * This function adds the device-chat and saved-messages chat
- * and adds one or more welcome or update-messages.
- * The ui can add messages on its own using dc_add_device_msg() -
- * for ordering, either before or after or even without calling this function.
- *
- * Chat and message creation is done only once.
- * So if the user has manually deleted things, they won't be re-created
- * (however, not seen device messages are added and may re-create the device-chat).
- *
- * @memberof dc_context_t
- * @param context The context object.
- * @return None.
- */
-void            dc_update_device_chats       (dc_context_t* context);
-
-
 /**
  * Check if a device-message with a given label was ever added.
  * Device-messages can be added dc_add_device_msg().
@@ -1090,7 +1069,8 @@ int             dc_get_fresh_msg_cnt         (dc_context_t* context, uint32_t ch
 /**
  * Estimate the number of messages that will be deleted
  * by the dc_set_config()-options `delete_device_after` or `delete_server_after`.
- * This is typically used to show the estimated impact to the user before actually enabling ephemeral messages.
+ * This is typically used to show the estimated impact to the user
+ * before actually enabling deletion of old messages.
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
@@ -1121,27 +1101,16 @@ dc_array_t*     dc_get_fresh_msgs            (dc_context_t* context);
  * but are still waiting for being marked as "seen" using dc_markseen_msgs()
  * (IMAP/MDNs is not done for noticed messages).
  *
- * Calling this function usually results in the event #DC_EVENT_MSGS_CHANGED.
- * See also dc_marknoticed_all_chats(), dc_marknoticed_contact() and dc_markseen_msgs().
+ * Calling this function usually results in the event #DC_EVENT_MSGS_NOTICED.
+ * See also dc_marknoticed_contact() and dc_markseen_msgs().
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID of which all messages should be marked as being noticed.
- * @return None.
  */
 void            dc_marknoticed_chat          (dc_context_t* context, uint32_t chat_id);
 
 
-/**
- * Same as dc_marknoticed_chat() but for _all_ chats.
- *
- * @memberof dc_context_t
- * @param context The context object as returned from dc_context_new().
- * @return None.
- */
-void            dc_marknoticed_all_chats     (dc_context_t* context);
-
-
 /**
  * Returns all message IDs of the given types in a chat.
  * Typically used to show a gallery.
@@ -1187,6 +1156,24 @@ dc_array_t*     dc_get_chat_media            (dc_context_t* context, uint32_t ch
 uint32_t        dc_get_next_media            (dc_context_t* context, uint32_t msg_id, int dir, int msg_type, int msg_type2, int msg_type3);
 
 
+/**
+ * Enable or disable protection against active attacks.
+ * To enable protection, it is needed that all members are verified;
+ * if this condition is met, end-to-end-encryption is always enabled
+ * and only the verified keys are used.
+ *
+ * Sends out #DC_EVENT_CHAT_MODIFIED on changes
+ * and #DC_EVENT_MSGS_CHANGED if a status message was sent.
+ *
+ * @memberof dc_context_t
+ * @param context The context object as returned from dc_context_new().
+ * @param chat_id The ID of the chat to change the protection for.
+ * @param protect 1=protect chat, 0=unprotect chat
+ * @return 1=success, 0=error, eg. some members may be unverified
+ */
+int             dc_set_chat_protection       (dc_context_t* context, uint32_t chat_id, int protect);
+
+
 /**
  * Set chat visibility to pinned, archived or normal.
  *
@@ -1197,7 +1184,6 @@ uint32_t        dc_get_next_media            (dc_context_t* context, uint32_t ms
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The ID of the chat to change the visibility for.
  * @param visibility one of @ref DC_CHAT_VISIBILITY
- * @return None.
  */
 void            dc_set_chat_visibility       (dc_context_t* context, uint32_t chat_id, int visibility);
 
@@ -1226,7 +1212,6 @@ void            dc_set_chat_visibility       (dc_context_t* context, uint32_t ch
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The ID of the chat to delete.
- * @return None.
  */
 void            dc_delete_chat               (dc_context_t* context, uint32_t chat_id);
 
@@ -1252,6 +1237,8 @@ dc_array_t*     dc_get_chat_contacts         (dc_context_t* context, uint32_t ch
 
 /**
  * Get the chat's ephemeral message timer.
+ * The ephemeral message timer is set by dc_set_chat_ephemeral_timer()
+ * on this or any other device participating in the chat.
  *
  * @memberof dc_context_t
  * @param context The context object.
@@ -1316,15 +1303,15 @@ dc_chat_t*      dc_get_chat                  (dc_context_t* context, uint32_t ch
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @param verified If set to 1 the function creates a secure verified group.
- *     Only secure-verified members are allowed in these groups
+ * @param protect If set to 1 the function creates group with protection initially enabled.
+ *     Only verified members are allowed in these groups
  *     and end-to-end-encryption is always enabled.
  * @param name The name of the group chat to create.
  *     The name may be changed later using dc_set_chat_name().
  *     To find out the name of a group later, see dc_chat_get_name()
  * @return The chat ID of the new group chat, 0 on errors.
  */
-uint32_t        dc_create_group_chat         (dc_context_t* context, int verified, const char* name);
+uint32_t        dc_create_group_chat         (dc_context_t* context, int protect, const char* name);
 
 
 /**
@@ -1346,7 +1333,7 @@ int             dc_is_contact_in_chat        (dc_context_t* context, uint32_t ch
  * If the group is already _promoted_ (any message was sent to the group),
  * all group members are informed by a special status message that is sent automatically by this function.
  *
- * If the group is a verified group, only verified contacts can be added to the group.
+ * If the group has group protection enabled, only verified contacts can be added to the group.
  *
  * Sends out #DC_EVENT_CHAT_MODIFIED and #DC_EVENT_MSGS_CHANGED if a status message was sent.
  *
@@ -1395,8 +1382,11 @@ int             dc_set_chat_name             (dc_context_t* context, uint32_t ch
 /**
  * Set the chat's ephemeral message timer.
  *
- * This timer is applied to all messages in a chat and starts when the
- * message is read. The setting is synchronized to all clients
+ * This timer is applied to all messages in a chat and starts when the message is read.
+ * For outgoing messages, the timer starts once the message is sent,
+ * for incoming messages, the timer starts once dc_markseen_msgs() is called.
+ *
+ * The setting is synchronized to all clients
  * participating in a chat.
  *
  * @memberof dc_context_t
@@ -1488,7 +1478,6 @@ char*           dc_get_mime_headers          (dc_context_t* context, uint32_t ms
  * @param context The context object
  * @param msg_ids an array of uint32_t containing all message IDs that should be deleted
  * @param msg_cnt The number of messages IDs in the msg_ids array
- * @return None.
  */
 void            dc_delete_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
@@ -1501,22 +1490,22 @@ void            dc_delete_msgs               (dc_context_t* context, const uint3
  * @param msg_ids An array of uint32_t containing all message IDs that should be forwarded
  * @param msg_cnt The number of messages IDs in the msg_ids array
  * @param chat_id The destination chat ID.
- * @return None.
  */
 void            dc_forward_msgs              (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt, uint32_t chat_id);
 
 
 /**
- * Mark all messages sent by the given contact
- * as _noticed_.  See also dc_marknoticed_chat() and
- * dc_markseen_msgs()
+ * Mark all messages sent by the given contact as _noticed_.
+ * This function is typically used to ignore a user in the deaddrop temporarily ("Not now" button).
  *
- * Calling this function usually results in the event #DC_EVENT_MSGS_CHANGED.
+ * The contact is expected to belong to the deaddrop;
+ * only one #DC_EVENT_MSGS_NOTICED with chat_id=DC_CHAT_ID_DEADDROP may be emitted.
+ *
+ * See also dc_marknoticed_chat() and dc_markseen_msgs()
  *
  * @memberof dc_context_t
  * @param context The context object.
  * @param contact_id The contact ID of which all messages should be marked as noticed.
- * @return None.
  */
 void            dc_marknoticed_contact       (dc_context_t* context, uint32_t contact_id);
 
@@ -1527,30 +1516,19 @@ void            dc_marknoticed_contact       (dc_context_t* context, uint32_t co
  * message is only marked as NOTICED and no IMAP/MDNs is done.  See also
  * dc_marknoticed_chat() and dc_marknoticed_contact()
  *
+ * Moreover, if messages belong to a chat with ephemeral messages enabled,
+ * the ephemeral timer is started for these messages.
+ *
+ * One #DC_EVENT_MSGS_NOTICED event is emitted per modified chat.
+ *
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_ids An array of uint32_t containing all the messages IDs that should be marked as seen.
  * @param msg_cnt The number of message IDs in msg_ids.
- * @return None.
  */
 void            dc_markseen_msgs             (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
 
-/**
- * Star/unstar messages by setting the last parameter to 0 (unstar) or 1 (star).
- * Starred messages are collected in a virtual chat that can be shown using
- * dc_get_chat_msgs() using the chat_id DC_CHAT_ID_STARRED.
- *
- * @memberof dc_context_t
- * @param context The context object.
- * @param msg_ids An array of uint32_t message IDs defining the messages to star or unstar
- * @param msg_cnt The number of IDs in msg_ids
- * @param star 0=unstar the messages in msg_ids, 1=star them
- * @return None.
- */
-void            dc_star_msgs                 (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt, int star);
-
-
 /**
  * Get a single message object of the type dc_msg_t.
  * For a list of messages in a chat, see dc_get_chat_msgs()
@@ -1701,7 +1679,6 @@ dc_array_t*     dc_get_blocked_contacts      (dc_context_t* context);
  * @param context The context object.
  * @param contact_id The ID of the contact to block or unblock.
  * @param block 1=block contact, 0=unblock contact
- * @return None.
  */
 void            dc_block_contact             (dc_context_t* context, uint32_t contact_id, int block);
 
@@ -1797,7 +1774,6 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
  * @param param1 Meaning depends on the DC_IMEX_* constants. If this parameter is a directory, it should not end with
  *     a slash (otherwise you'll get double slashes when receiving #DC_EVENT_IMEX_FILE_WRITTEN). Set to NULL if not used.
  * @param param2 Meaning depends on the DC_IMEX_* constants. Set to NULL if not used.
- * @return None.
  */
 void            dc_imex                      (dc_context_t* context, int what, const char* param1, const char* param2);
 
@@ -1940,7 +1916,6 @@ int             dc_continue_key_transfer     (dc_context_t* context, uint32_t ms
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @return None.
  */
 void            dc_stop_ongoing_process      (dc_context_t* context);
 
@@ -2002,7 +1977,7 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
  * @param context The context object.
  * @param chat_id If set to a group-chat-id,
  *     the Verified-Group-Invite protocol is offered in the QR code;
- *     works for verified groups as well as for normal groups.
+ *     works for protected groups as well as for normal groups.
  *     If set to 0, the Setup-Contact protocol is offered in the QR code.
  *     See https://countermitm.readthedocs.io/en/latest/new.html
  *     for details about both protocols.
@@ -2032,7 +2007,7 @@ char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t ch
  *   When the protocol has finished, an info-message is added to that chat.
  * - If the given QR code starts the Verified-Group-Invite protocol,
  *   the function waits until the protocol has finished.
- *   This is because the verified group is not opportunistic
+ *   This is because the protected group is not opportunistic
  *   and can be created only when the contacts have verified each other.
  *
  * See https://countermitm.readthedocs.io/en/latest/new.html
@@ -2044,8 +2019,8 @@ char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t ch
  *     to dc_check_qr().
  * @return Chat-id of the joined chat, the UI may redirect to the this chat.
  *     If the out-of-band verification failed or was aborted, 0 is returned.
- *     A returned chat-id does not guarantee that the chat or the belonging contact is verified.
- *     If needed, this be checked with dc_chat_is_verified() and dc_contact_is_verified(),
+ *     A returned chat-id does not guarantee that the chat is protected or the belonging contact is verified.
+ *     If needed, this be checked with dc_chat_is_protected() and dc_contact_is_verified(),
  *     however, in practise, the UI will just listen to #DC_EVENT_CONTACTS_CHANGED unconditionally.
  */
 uint32_t        dc_join_securejoin           (dc_context_t* context, const char* qr);
@@ -2069,7 +2044,6 @@ uint32_t        dc_join_securejoin           (dc_context_t* context, const char*
  * @param chat_id Chat id to enable location streaming for.
  * @param seconds >0: enable location streaming for the given number of seconds;
  *     0: disable location streaming.
- * @return None.
  */
 void        dc_send_locations_to_chat       (dc_context_t* context, uint32_t chat_id, int seconds);
 
@@ -2192,7 +2166,6 @@ dc_array_t* dc_get_locations                (dc_context_t* context, uint32_t cha
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @return None.
  */
 void        dc_delete_all_locations         (dc_context_t* context);
 
@@ -2203,11 +2176,11 @@ void        dc_delete_all_locations         (dc_context_t* context);
  *   MUST NOT be released by the standard free() function;
  *   always use dc_str_unref() for this purpose.
  * - dc_str_unref() MUST NOT be called for strings not returned by deltachat-core.
- * - dc_str_unref() MUST NOT be called for other objectes returned by deltachat-core.
+ * - dc_str_unref() MUST NOT be called for other objects returned by deltachat-core.
  *
  * @memberof dc_context_t
  * @param str The string to release.
- * @return None.
+ *     If NULL is given, nothing is done.
  */
 void dc_str_unref (char* str);
 
@@ -2394,7 +2367,6 @@ int            dc_accounts_select_account       (dc_accounts_t* accounts, uint32
  *
  * @memberof dc_accounts_t
  * @param accounts Account manager as created by dc_accounts_new().
- * @return None.
  */
 void           dc_accounts_start_io             (dc_accounts_t* accounts);
 
@@ -2462,7 +2434,6 @@ dc_accounts_event_emitter_t* dc_accounts_get_event_emitter (dc_accounts_t* accou
  * @param array The array object to free,
  *     created eg. by dc_get_chatlist(), dc_get_contacts() and so on.
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void             dc_array_unref              (dc_array_t* array);
 
@@ -2661,7 +2632,6 @@ int              dc_array_search_id          (const dc_array_t* array, uint32_t
  * @memberof dc_chatlist_t
  * @param chatlist The chatlist object to free, created eg. by dc_get_chatlist(), dc_search_msgs().
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void             dc_chatlist_unref           (dc_chatlist_t* chatlist);
 
@@ -2748,7 +2718,7 @@ dc_lot_t*        dc_chatlist_get_summary     (const dc_chatlist_t* chatlist, siz
  * @memberof dc_context_t
  * @param context The context object.
  * @param chat_id Chat to get a summary for.
- * @param msg_id Messasge to get a summary for.
+ * @param msg_id Message to get a summary for.
  * @return The summary as an dc_lot_t object, see dc_chatlist_get_summary() for details.
  *     Must be freed using dc_lot_unref().  NULL is never returned.
  */
@@ -2795,8 +2765,6 @@ char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat
 
 #define         DC_CHAT_ID_DEADDROP          1 // virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 #define         DC_CHAT_ID_TRASH             3 // messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
-#define         DC_CHAT_ID_MSGS_IN_CREATION  4 // a message is just in creation but not yet assigned to a chat (eg. we may need the message ID to set up blobs; this avoids unready message to be sent and shown)
-#define         DC_CHAT_ID_STARRED           5 // virtual chat showing all messages flagged with msgs.starred=2
 #define         DC_CHAT_ID_ARCHIVED_LINK     6 // only an indicator in a chatlist
 #define         DC_CHAT_ID_ALLDONE_HINT      7 // only an indicator in a chatlist
 #define         DC_CHAT_ID_LAST_SPECIAL      9 // larger chat IDs are "real" chats, their messages are "real" messages.
@@ -2805,7 +2773,6 @@ char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat
 #define         DC_CHAT_TYPE_UNDEFINED       0
 #define         DC_CHAT_TYPE_SINGLE          100
 #define         DC_CHAT_TYPE_GROUP           120
-#define         DC_CHAT_TYPE_VERIFIED_GROUP  130
 
 
 /**
@@ -2814,7 +2781,6 @@ char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat
  * @memberof dc_chat_t
  * @param chat Chat object are returned eg. by dc_get_chat().
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_chat_unref                (dc_chat_t* chat);
 
@@ -2824,7 +2790,6 @@ void            dc_chat_unref                (dc_chat_t* chat);
  *
  * Special IDs:
  * - DC_CHAT_ID_DEADDROP         (1) - Virtual chat containing messages which senders are not confirmed by the user.
- * - DC_CHAT_ID_STARRED          (5) - Virtual chat containing all starred messages-
  * - DC_CHAT_ID_ARCHIVED_LINK    (6) - A link at the end of the chatlist, if present the UI should show the button "Archived chats"-
  *
  * "Normal" chat IDs are larger than these special IDs (larger than DC_CHAT_ID_LAST_SPECIAL).
@@ -2848,9 +2813,6 @@ uint32_t        dc_chat_get_id               (const dc_chat_t* chat);
  * - DC_CHAT_TYPE_GROUP  (120) - a group chat, chats_contacts contain all group
  *   members, incl. DC_CONTACT_ID_SELF
  *
- * - DC_CHAT_TYPE_VERIFIED_GROUP  (130) - a verified group chat. In verified groups,
- *   all members are verified and encryption is always active and cannot be disabled.
- *
  * @memberof dc_chat_t
  * @param chat The chat object.
  * @return Chat type.
@@ -2980,15 +2942,16 @@ int             dc_chat_can_send              (const dc_chat_t* chat);
 
 
 /**
- * Check if a chat is verified.  Verified chats contain only verified members
- * and encryption is alwasy enabled.  Verified chats are created using
- * dc_create_group_chat() by setting the 'verified' parameter to true.
+ * Check if a chat is protected.
+ * Protected chats contain only verified members and encryption is always enabled.
+ * Protected chats are created using dc_create_group_chat() by setting the 'protect' parameter to 1.
+ * The status can be changed using dc_set_chat_protection().
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
- * @return 1=chat verified, 0=chat is not verified
+ * @return 1=chat protected, 0=chat is not protected
  */
-int             dc_chat_is_verified          (const dc_chat_t* chat);
+int             dc_chat_is_protected         (const dc_chat_t* chat);
 
 
 /**
@@ -3075,7 +3038,6 @@ dc_msg_t*       dc_msg_new                    (dc_context_t* context, int viewty
  * @memberof dc_msg_t
  * @param msg The message object to free.
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_msg_unref                  (dc_msg_t* msg);
 
@@ -3354,6 +3316,7 @@ int             dc_msg_get_showpadlock        (const dc_msg_t* msg);
 
 /**
  * Get ephemeral timer duration for message.
+ * This is the value of dc_get_chat_ephemeral_timer() in the moment the message was sent.
  *
  * To check if the timer is started and calculate remaining time,
  * use dc_msg_get_ephemeral_timestamp().
@@ -3373,7 +3336,8 @@ uint32_t        dc_msg_get_ephemeral_timer    (const dc_msg_t* msg);
  *
  * @memberof dc_msg_t
  * @param msg The message object.
- * @return Time of message removal, 0 if the timer is not started.
+ * @return Time of message removal, 0 if the timer is not yet started
+ *     (the timer starts on sending messages or when dc_markseen_msgs() is called)
  */
 int64_t          dc_msg_get_ephemeral_timestamp (const dc_msg_t* msg);
 
@@ -3459,21 +3423,6 @@ int             dc_msg_has_location           (const dc_msg_t* msg);
 int             dc_msg_is_sent                (const dc_msg_t* msg);
 
 
-/**
- * Check if a message is starred.  Starred messages are "favorites" marked by the user
- * with a "star" or something like that.  Starred messages can typically be shown
- * easily and are not deleted automatically.
- *
- * To star one or more messages, use dc_star_msgs(), to get a list of starred messages,
- * use dc_get_chat_msgs() using DC_CHAT_ID_STARRED as the chat_id.
- *
- * @memberof dc_msg_t
- * @param msg The message object.
- * @return 1=message is starred, 0=message not starred.
- */
-int             dc_msg_is_starred             (const dc_msg_t* msg);
-
-
 /**
  * Check if the message is a forwarded message.
  *
@@ -3496,7 +3445,8 @@ int             dc_msg_is_forwarded           (const dc_msg_t* msg);
 /**
  * Check if the message is an informational message, created by the
  * device or by another users. Such messages are not "typed" by the user but
- * created due to other actions, eg. dc_set_chat_name(), dc_set_chat_profile_image()
+ * created due to other actions,
+ * eg. dc_set_chat_name(), dc_set_chat_profile_image(), dc_set_chat_protection()
  * or dc_add_contact_to_chat().
  *
  * These messages are typically shown in the center of the chat view,
@@ -3512,6 +3462,32 @@ int             dc_msg_is_forwarded           (const dc_msg_t* msg);
 int             dc_msg_is_info                (const dc_msg_t* msg);
 
 
+/**
+ * Get the type of an informational message.
+ * If dc_msg_is_info() returns 1, this function returns the type of the informational message.
+ * UIs can display eg. an icon based upon the type.
+ *
+ * Currently, the following types are defined:
+ * - DC_INFO_PROTECTION_ENABLED (11) - Info-message for "Chat is now protected"
+ * - DC_INFO_PROTECTION_DISABLED (12) - Info-message for "Chat is no longer protected"
+ *
+ * Even when you display an icon,
+ * you should still display the text of the informational message using dc_msg_get_text()
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return One of the DC_INFO* constants.
+ *     0 or other values indicate unspecified types
+ *     or that the message is not an info-message.
+ */
+int             dc_msg_get_info_type          (const dc_msg_t* msg);
+
+
+// DC_INFO* uses the same values as SystemMessage in rust-land
+#define         DC_INFO_PROTECTION_ENABLED     11
+#define         DC_INFO_PROTECTION_DISABLED    12
+
+
 /**
  * Check if a message is still in creation.  A message is in creation between
  * the calls to dc_prepare_msg() and dc_send_msg().
@@ -3576,6 +3552,27 @@ char*           dc_msg_get_setupcodebegin     (const dc_msg_t* msg);
  */
 char* dc_msg_get_videochat_url (const dc_msg_t* msg);
 
+/**
+ * Gets the error status of the message.
+ * If there is no error associated with the message, NULL is returned.
+ *
+ * A message can have an associated error status if something went wrong when sending or
+ * receiving message itself.  The error status is free-form text and should not be further parsed,
+ * rather it's presence is meant to indicate *something* went wrong with the message and the
+ * text of the error is detailed information on what.
+ * 
+ * Some common reasons error can be associated with messages are:
+ * * Lack of valid signature on an e2ee message, usually for received messages.
+ * * Failure to decrypt an e2ee message, usually for received messages.
+ * * When a message could not be delivered to one or more recipients the non-delivery
+ *   notification text can be stored in the error status.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return Error or NULL. The result must be released using dc_str_unref().
+ */
+char*           dc_msg_get_error               (const dc_msg_t* msg);
+
 
 /**
  * Get type of videochat.
@@ -3619,7 +3616,6 @@ int dc_msg_get_videochat_type (const dc_msg_t* msg);
  * @memberof dc_msg_t
  * @param msg The message object.
  * @param text Message text.
- * @return None.
  */
 void            dc_msg_set_text               (dc_msg_t* msg, const char* text);
 
@@ -3635,7 +3631,6 @@ void            dc_msg_set_text               (dc_msg_t* msg, const char* text);
  * @param file If the message object is used in dc_send_msg() later,
  *     this must be the full path of the image file to send.
  * @param filemime Mime type of the file. NULL if you don't know or don't care.
- * @return None.
  */
 void            dc_msg_set_file               (dc_msg_t* msg, const char* file, const char* filemime);
 
@@ -3649,7 +3644,6 @@ void            dc_msg_set_file               (dc_msg_t* msg, const char* file,
  * @param msg The message object.
  * @param width Width in pixels, if known. 0 if you don't know or don't care.
  * @param height Height in pixels, if known. 0 if you don't know or don't care.
- * @return None.
  */
 void            dc_msg_set_dimension          (dc_msg_t* msg, int width, int height);
 
@@ -3662,7 +3656,6 @@ void            dc_msg_set_dimension          (dc_msg_t* msg, int width, int hei
  * @memberof dc_msg_t
  * @param msg The message object.
  * @param duration Length in milliseconds. 0 if you don't know or don't care.
- * @return None.
  */
 void            dc_msg_set_duration           (dc_msg_t* msg, int duration);
 
@@ -3682,7 +3675,6 @@ void            dc_msg_set_duration           (dc_msg_t* msg, int duration);
  * @param msg The message object.
  * @param latitude North-south position of the location.
  * @param longitude East-west position of the location.
- * @return None.
  */
 void            dc_msg_set_location           (dc_msg_t* msg, double latitude, double longitude);
 
@@ -3707,11 +3699,67 @@ void            dc_msg_set_location           (dc_msg_t* msg, double latitude, d
  * @param width The new width to store in the message object. 0 if you do not want to change width and height.
  * @param height The new height to store in the message object. 0 if you do not want to change width and height.
  * @param duration The new duration to store in the message object. 0 if you do not want to change it.
- * @return None.
  */
 void            dc_msg_latefiling_mediasize   (dc_msg_t* msg, int width, int height, int duration);
 
 
+/**
+ * Set the message replying to.
+ * This allows optionally to reply to an explicit message
+ * instead of replying implicitly to the end of the chat.
+ *
+ * dc_msg_set_quote() copies some basic data from the quoted message object
+ * so that dc_msg_get_quoted_text() will always work.
+ * dc_msg_get_quoted_msg() gets back the quoted message only if it is _not_ deleted.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object to set the reply to.
+ * @param quote The quote to set for msg.
+ */
+void             dc_msg_set_quote             (dc_msg_t* msg, const dc_msg_t* quote);
+
+
+/**
+ * Get quoted text, if any.
+ * You can use this function also check if there is a quote for a message.
+ *
+ * The text is a summary of the original text,
+ * similar to what is shown in the chatlist.
+ *
+ * If available, you can get the whole quoted message object using dc_msg_get_quoted_msg().
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return The quoted text or NULL if there is no quote.
+ *     Returned strings must be released using dc_str_unref().
+ */
+char*           dc_msg_get_quoted_text        (const dc_msg_t* msg);
+
+
+/**
+ * Get quoted message, if available.
+ * UIs might use this information to offer "jumping back" to the quoted message
+ * or to enrich displaying the quote.
+ *
+ * If this function returns NULL,
+ * this does not mean there is no quote for the message -
+ * it might also mean that a quote exist but the quoted message is deleted meanwhile.
+ * Therefore, do not use this function to check if there is a quote for a message.
+ * To check if a message has a quote, use dc_msg_get_quoted_text().
+ *
+ * To display the quote in the chat, use dc_msg_get_quoted_text() as a primary source,
+ * however, one might add information from the message object (eg. an image).
+ *
+ * It is not guaranteed that the message belong to the same chat.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return The quoted message or NULL.
+ *     Must be freed using dc_msg_unref() after usage.
+ */
+dc_msg_t*       dc_msg_get_quoted_msg         (const dc_msg_t* msg);
+
+
 /**
  * @class dc_contact_t
  *
@@ -3745,7 +3793,6 @@ void            dc_msg_latefiling_mediasize   (dc_msg_t* msg, int width, int hei
  * @memberof dc_contact_t
  * @param contact The contact object as created eg. by dc_get_contact().
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_contact_unref             (dc_contact_t* contact);
 
@@ -3985,7 +4032,6 @@ void            dc_provider_unref                     (dc_provider_t* provider);
  * @memberof dc_lot_t
  * @param lot The object to free.
  *     If NULL is given, nothing is done.
- * @return None.
  */
 void            dc_lot_unref             (dc_lot_t* lot);
 
@@ -4299,7 +4345,6 @@ dc_event_t* dc_get_next_event(dc_event_emitter_t* emitter);
  * @memberof dc_event_emitter_t
  * @param emitter Event emitter object as returned from dc_get_event_emitter().
  *     If NULL is given, nothing is done and an error is logged.
- * @return None.
  */
 void  dc_event_emitter_unref(dc_event_emitter_t* emitter);
 
@@ -4333,7 +4378,6 @@ dc_event_t* dc_accounts_get_next_event (dc_accounts_event_emitter_t* emitter);
  * @memberof dc_accounts_event_emitter_t
  * @param emitter Event emitter object as returned from dc_accounts_get_event_emitter().
  *     If NULL is given, nothing is done and an error is logged.
- * @return None.
  */
 void dc_accounts_event_emitter_unref(dc_accounts_event_emitter_t* emitter);
 
@@ -4395,7 +4439,7 @@ int dc_event_get_data2_int(dc_event_t* event);
  *
  * @memberof dc_event_t
  * @param event Event object as returned from dc_get_next_event().
- * @return "data2" as a string,
+ * @return "data2" as a string or NULL.
  *     the meaning depends on the event type associated with this event.
  *     Once you're done with the string, you have to unref it using dc_unref_str().
  */
@@ -4420,7 +4464,6 @@ uint32_t dc_event_get_account_id(dc_event_t* event);
  *
  * @memberof dc_event_t
  * @param event Event object as returned from dc_get_next_event().
- * @return None.
  */
 void dc_event_unref(dc_event_t* event);
 
@@ -4596,6 +4639,21 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_INCOMING_MSG             2005
 
 
+/**
+ * Messages were marked noticed or seen.
+ * The ui may update badge counters or stop showing a chatlist-item with a bold font.
+ *
+ * This event is emitted eg. when calling dc_markseen_msgs(), dc_marknoticed_chat() or dc_marknoticed_contact()
+ * or when a chat is answered on another device.
+ * Do not try to derive the state of an item from just the fact you received the event;
+ * use eg. dc_msg_get_state() or dc_get_fresh_msg_cnt() for this purpose.
+ *
+ * @param data1 (int) chat_id
+ * @param data2 0
+ */
+#define DC_EVENT_MSGS_NOTICED             2008
+
+
 /**
  * A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
  * DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
@@ -4647,7 +4705,7 @@ void dc_event_unref(dc_event_t* event);
 /**
  * Contact(s) created, renamed, verified, blocked or deleted.
  *
- * @param data1 (int) If not 0, this is the contact_id of an added contact that should be selected.
+ * @param data1 (int) contact_id of the changed contact or 0 on batch-changes or deletion.
  * @param data2 0
  */
 #define DC_EVENT_CONTACTS_CHANGED         2030
@@ -4897,7 +4955,6 @@ void dc_event_unref(dc_event_t* event);
 #define DC_STR_CONTACT_NOT_VERIFIED       36
 #define DC_STR_CONTACT_SETUP_CHANGED      37
 #define DC_STR_ARCHIVEDCHATS              40
-#define DC_STR_STARREDMSGS                41
 #define DC_STR_AC_SETUP_MSG_SUBJECT       42
 #define DC_STR_AC_SETUP_MSG_BODY          43
 #define DC_STR_CANNOT_LOGIN               60
@@ -4925,8 +4982,12 @@ void dc_event_unref(dc_event_t* event);
 #define DC_STR_VIDEOCHAT_INVITATION       82
 #define DC_STR_VIDEOCHAT_INVITE_MSG_BODY  83
 #define DC_STR_CONFIGURATION_FAILED       84
-
-#define DC_STR_COUNT                      84
+#define DC_STR_BAD_TIME_MSG_BODY          85
+#define DC_STR_UPDATE_REMINDER_MSG_BODY   86
+#define DC_STR_ERROR_NO_NETWORK           87
+#define DC_STR_PROTECTION_ENABLED         88
+#define DC_STR_PROTECTION_DISABLED        89
+#define DC_STR_REPLY_NOUN                 90 /* eg. "Reply", used in summaries, a noun, not a verb (not: "to reply") */
 
 /*
  * @}

deltachat-ffi/src/lib.rs

@@ -25,7 +25,7 @@ use async_std::task::{block_on, spawn};
 use num_traits::{FromPrimitive, ToPrimitive};
 
 use deltachat::accounts::Accounts;
-use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
+use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, Origin};
 use deltachat::context::Context;
@@ -218,7 +218,7 @@ pub unsafe extern "C" fn dc_set_config_from_qr(
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c_char {
+pub unsafe extern "C" fn dc_get_info(context: *const dc_context_t) -> *mut libc::c_char {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_get_info()");
         return "".strdup();
@@ -316,7 +316,6 @@ pub unsafe extern "C" fn dc_get_id(context: *mut dc_context_t) -> libc::c_int {
     ctx.get_id() as libc::c_int
 }
 
-#[no_mangle]
 pub type dc_event_t = Event;
 
 #[no_mangle]
@@ -363,6 +362,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ErrorSelfNotInGroup(_) => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
+        | EventType::MsgsNoticed(chat_id)
         | EventType::MsgDelivered { chat_id, .. }
         | EventType::MsgFailed { chat_id, .. }
         | EventType::MsgRead { chat_id, .. }
@@ -408,6 +408,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ConfigureProgress { .. }
         | EventType::ImexProgress(_)
         | EventType::ImexFileWritten(_)
+        | EventType::MsgsNoticed(_)
         | EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
@@ -447,6 +448,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         }
         EventType::MsgsChanged { .. }
         | EventType::IncomingMsg { .. }
+        | EventType::MsgsNoticed(_)
         | EventType::MsgDelivered { .. }
         | EventType::MsgFailed { .. }
         | EventType::MsgRead { .. }
@@ -481,7 +483,6 @@ pub unsafe extern "C" fn dc_event_get_account_id(event: *mut dc_event_t) -> u32
     (*event).id
 }
 
-#[no_mangle]
 pub type dc_event_emitter_t = EventEmitter;
 
 #[no_mangle]
@@ -499,7 +500,7 @@ pub unsafe extern "C" fn dc_get_event_emitter(
 #[no_mangle]
 pub unsafe extern "C" fn dc_event_emitter_unref(emitter: *mut dc_event_emitter_t) {
     if emitter.is_null() {
-        eprintln!("ignoring careless call to dc_event_mitter_unref()");
+        eprintln!("ignoring careless call to dc_event_emitter_unref()");
         return;
     }
 
@@ -509,6 +510,7 @@ pub unsafe extern "C" fn dc_event_emitter_unref(emitter: *mut dc_event_emitter_t
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *mut dc_event_t {
     if events.is_null() {
+        eprintln!("ignoring careless call to dc_get_next_event()");
         return ptr::null_mut();
     }
     let events = &*events;
@@ -805,21 +807,6 @@ pub unsafe extern "C" fn dc_add_device_msg(
     .to_u32()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_update_device_chats(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_update_device_chats()");
-        return;
-    }
-    let ctx = &mut *context;
-
-    block_on(async move {
-        ctx.update_device_chats()
-            .await
-            .unwrap_or_log_default(&ctx, "Failed to add device message")
-    })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_was_device_msg_ever_added(
     context: *mut dc_context_t,
@@ -972,22 +959,6 @@ pub unsafe extern "C" fn dc_marknoticed_chat(context: *mut dc_context_t, chat_id
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_marknoticed_all_chats(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_marknoticed_all_chats()");
-        return;
-    }
-    let ctx = &*context;
-
-    block_on(async move {
-        chat::marknoticed_all_chats(&ctx)
-            .await
-            .log_err(ctx, "Failed marknoticed all chats")
-            .unwrap_or(())
-    })
-}
-
 fn from_prim<S, T>(s: S) -> Option<T>
 where
     T: FromPrimitive,
@@ -1071,6 +1042,32 @@ pub unsafe extern "C" fn dc_get_next_media(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_set_chat_protection(
+    context: *mut dc_context_t,
+    chat_id: u32,
+    protect: libc::c_int,
+) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_set_chat_protection()");
+        return 0;
+    }
+    let ctx = &*context;
+    let protect = if let Some(s) = ProtectionStatus::from_i32(protect) {
+        s
+    } else {
+        warn!(ctx, "bad protect-value for dc_set_chat_protection()");
+        return 0;
+    };
+
+    block_on(async move {
+        match ChatId::new(chat_id).set_protection(&ctx, protect).await {
+            Ok(()) => 1,
+            Err(_) => 0,
+        }
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_chat_visibility(
     context: *mut dc_context_t,
@@ -1184,7 +1181,7 @@ pub unsafe extern "C" fn dc_get_chat(context: *mut dc_context_t, chat_id: u32) -
 #[no_mangle]
 pub unsafe extern "C" fn dc_create_group_chat(
     context: *mut dc_context_t,
-    verified: libc::c_int,
+    protect: libc::c_int,
     name: *const libc::c_char,
 ) -> u32 {
     if context.is_null() || name.is_null() {
@@ -1192,14 +1189,15 @@ pub unsafe extern "C" fn dc_create_group_chat(
         return 0;
     }
     let ctx = &*context;
-    let verified = if let Some(s) = contact::VerifiedStatus::from_i32(verified) {
+    let protect = if let Some(s) = ProtectionStatus::from_i32(protect) {
         s
     } else {
+        warn!(ctx, "bad protect-value for dc_create_group_chat()");
         return 0;
     };
 
     block_on(async move {
-        chat::create_group_chat(&ctx, verified, to_string_lossy(name))
+        chat::create_group_chat(&ctx, protect, to_string_lossy(name))
             .await
             .log_err(ctx, "Failed to create group chat")
             .map(|id| id.to_u32())
@@ -1478,23 +1476,6 @@ pub unsafe extern "C" fn dc_markseen_msgs(
     block_on(message::markseen_msgs(&ctx, msg_ids));
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_star_msgs(
-    context: *mut dc_context_t,
-    msg_ids: *const u32,
-    msg_cnt: libc::c_int,
-    star: libc::c_int,
-) {
-    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
-        eprintln!("ignoring careless call to dc_star_msgs()");
-        return;
-    }
-    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
-    let ctx = &*context;
-
-    block_on(message::star_msgs(&ctx, msg_ids, star == 1));
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) -> *mut dc_msg_t {
     if context.is_null() {
@@ -1819,7 +1800,7 @@ pub unsafe extern "C" fn dc_continue_key_transfer(
         {
             Ok(()) => 1,
             Err(err) => {
-                error!(&ctx, "dc_continue_key_transfer: {}", err);
+                warn!(&ctx, "dc_continue_key_transfer: {}", err);
                 0
             }
         }
@@ -1989,7 +1970,6 @@ pub unsafe extern "C" fn dc_delete_all_locations(context: *mut dc_context_t) {
 
 // dc_array_t
 
-#[no_mangle]
 pub type dc_array_t = dc_array::dc_array_t;
 
 #[no_mangle]
@@ -2172,7 +2152,6 @@ pub struct ChatlistWrapper {
     list: chatlist::Chatlist,
 }
 
-#[no_mangle]
 pub type dc_chatlist_t = ChatlistWrapper;
 
 #[no_mangle]
@@ -2296,7 +2275,6 @@ pub struct ChatWrapper {
     chat: chat::Chat,
 }
 
-#[no_mangle]
 pub type dc_chat_t = ChatWrapper;
 
 #[no_mangle]
@@ -2423,13 +2401,13 @@ pub unsafe extern "C" fn dc_chat_can_send(chat: *mut dc_chat_t) -> libc::c_int {
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_chat_is_verified(chat: *mut dc_chat_t) -> libc::c_int {
+pub unsafe extern "C" fn dc_chat_is_protected(chat: *mut dc_chat_t) -> libc::c_int {
     if chat.is_null() {
-        eprintln!("ignoring careless call to dc_chat_is_verified()");
+        eprintln!("ignoring careless call to dc_chat_is_protected()");
         return 0;
     }
     let ffi_chat = &*chat;
-    ffi_chat.chat.is_verified() as libc::c_int
+    ffi_chat.chat.is_protected() as libc::c_int
 }
 
 #[no_mangle]
@@ -2522,7 +2500,6 @@ pub struct MessageWrapper {
     message: message::Message,
 }
 
-#[no_mangle]
 pub type dc_msg_t = MessageWrapper;
 
 #[no_mangle]
@@ -2832,16 +2809,6 @@ pub unsafe extern "C" fn dc_msg_is_sent(msg: *mut dc_msg_t) -> libc::c_int {
     ffi_msg.message.is_sent().into()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_is_starred(msg: *mut dc_msg_t) -> libc::c_int {
-    if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_is_starred()");
-        return 0;
-    }
-    let ffi_msg = &*msg;
-    ffi_msg.message.is_starred().into()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_is_forwarded(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2862,6 +2829,16 @@ pub unsafe extern "C" fn dc_msg_is_info(msg: *mut dc_msg_t) -> libc::c_int {
     ffi_msg.message.is_info().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_info_type(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_info_type()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_info_type() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_is_increation(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -3007,6 +2984,74 @@ pub unsafe extern "C" fn dc_msg_latefiling_mediasize(
     });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_error(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_error()");
+        return ptr::null_mut();
+    }
+    let ffi_msg = &*msg;
+    match ffi_msg.message.error() {
+        Some(error) => error.strdup(),
+        None => ptr::null_mut(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_quote(msg: *mut dc_msg_t, quote: *const dc_msg_t) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_quote()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    let ffi_quote = &*quote;
+
+    block_on(async move {
+        ffi_msg
+            .message
+            .set_quote(&*ffi_msg.context, &ffi_quote.message)
+            .await
+            .log_err(&*ffi_msg.context, "failed to set quote")
+            .ok();
+    });
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_quoted_text(msg: *const dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_quoted_text()");
+        return ptr::null_mut();
+    }
+    let ffi_msg: &MessageWrapper = &*msg;
+    ffi_msg
+        .message
+        .quoted_text()
+        .map_or_else(ptr::null_mut, |s| s.strdup())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_quoted_msg(msg: *const dc_msg_t) -> *mut dc_msg_t {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_get_quoted_msg()");
+        return ptr::null_mut();
+    }
+    let ffi_msg: &MessageWrapper = &*msg;
+    let context = &*ffi_msg.context;
+    let res = block_on(async move {
+        ffi_msg
+            .message
+            .quoted_message(context)
+            .await
+            .log_err(context, "failed to get quoted message")
+            .unwrap_or(None)
+    });
+
+    match res {
+        Some(message) => Box::into_raw(Box::new(MessageWrapper { context, message })),
+        None => ptr::null_mut(),
+    }
+}
+
 // dc_contact_t
 
 /// FFI struct for [dc_contact_t]
@@ -3021,7 +3066,6 @@ pub struct ContactWrapper {
     contact: contact::Contact,
 }
 
-#[no_mangle]
 pub type dc_contact_t = ContactWrapper;
 
 #[no_mangle]
@@ -3154,7 +3198,6 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
 
 // dc_lot_t
 
-#[no_mangle]
 pub type dc_lot_t = lot::Lot;
 
 #[no_mangle]
@@ -3263,7 +3306,8 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
 
     fn log_err(self, ctx: &Context, message: &str) -> Result<T, E> {
         self.map_err(|err| {
-            warn!(ctx, "{}: {}", message, err);
+            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+            warn!(ctx, "{}: {:#}", message, err);
             err
         })
     }
@@ -3295,7 +3339,6 @@ fn convert_and_prune_message_ids(msg_ids: *const u32, msg_cnt: libc::c_int) -> V
 
 // dc_provider_t
 
-#[no_mangle]
 pub type dc_provider_t = provider::Provider;
 
 #[no_mangle]
@@ -3386,7 +3429,8 @@ pub unsafe extern "C" fn dc_accounts_new(
     match accs {
         Ok(accs) => Box::into_raw(Box::new(accs)),
         Err(err) => {
-            eprintln!("failed to create accounts: {}", err);
+            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+            eprintln!("failed to create accounts: {:#}", err);
             ptr::null_mut()
         }
     }
@@ -3447,7 +3491,7 @@ pub unsafe extern "C" fn dc_accounts_select_account(
     let accounts = &*accounts;
     block_on(accounts.select_account(id))
         .map(|_| 1)
-        .unwrap_or_else(|_| 0)
+        .unwrap_or(0)
 }
 
 #[no_mangle]
@@ -3459,7 +3503,7 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -
 
     let accounts = &*accounts;
 
-    block_on(accounts.add_account()).unwrap_or_else(|_| 0)
+    block_on(accounts.add_account()).unwrap_or(0)
 }
 
 #[no_mangle]
@@ -3561,7 +3605,6 @@ pub unsafe extern "C" fn dc_accounts_maybe_network(accounts: *mut dc_accounts_t)
     block_on(accounts.maybe_network());
 }
 
-#[no_mangle]
 pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
 
 #[no_mangle]
@@ -3595,9 +3638,10 @@ pub unsafe extern "C" fn dc_accounts_get_next_event(
     emitter: *mut dc_accounts_event_emitter_t,
 ) -> *mut dc_event_t {
     if emitter.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_get_next_event()");
         return ptr::null_mut();
     }
-    let emitter = &*emitter;
+    let emitter = &mut *emitter;
 
     emitter
         .recv_sync()

examples/repl/cmdline.rs

@@ -4,7 +4,7 @@ use std::str::FromStr;
 
 use anyhow::{bail, ensure};
 use async_std::path::Path;
-use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility};
+use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility, ProtectionStatus};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
@@ -185,7 +185,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -193,7 +193,6 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.is_starred() { "★" } else { "" },
         if msg.get_from_id() == 1 as libc::c_uint {
             ""
         } else if msg.get_state() == MessageState::InSeen {
@@ -358,7 +357,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  createchat <contact-id>\n\
                  createchatbymsg <msg-id>\n\
                  creategroup <name>\n\
-                 createverified <name>\n\
+                 createprotected <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
                  groupname <name>\n\
@@ -380,6 +379,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  unarchive <chat-id>\n\
                  pin <chat-id>\n\
                  unpin <chat-id>\n\
+                 protect <chat-id>\n\
+                 unprotect <chat-id>\n\
                  delchat <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
@@ -387,8 +388,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
-                 star <msg-id>\n\
-                 unstar <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
                  listcontacts [<query>]\n\
@@ -526,7 +525,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}",
+                        "{}#{}: {} [{} fresh] {}{}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
@@ -536,6 +535,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             ChatVisibility::Archived => "📦",
                             ChatVisibility::Pinned => "📌",
                         },
+                        if chat.is_protected() { "🛡️" } else { "" },
                     );
                     let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
@@ -610,7 +610,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 format!("{} member(s)", members.len())
             };
             println!(
-                "{}#{}: {} [{}]{}{}",
+                "{}#{}: {} [{}]{}{} {}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
                 sel_chat.get_name(),
@@ -627,6 +627,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     },
                     _ => "".to_string(),
                 },
+                if sel_chat.is_protected() {
+                    "🛡️"
+                } else {
+                    ""
+                },
             );
             log_msglist(&context, &msglist).await?;
             if let Some(draft) = sel_chat.get_id().get_draft(&context).await? {
@@ -657,15 +662,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
-                chat::create_group_chat(&context, VerifiedStatus::Unverified, arg1).await?;
+                chat::create_group_chat(&context, ProtectionStatus::Unprotected, arg1).await?;
 
             println!("Group#{} created successfully.", chat_id);
         }
-        "createverified" => {
+        "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id = chat::create_group_chat(&context, VerifiedStatus::Verified, arg1).await?;
+            let chat_id =
+                chat::create_group_chat(&context, ProtectionStatus::Protected, arg1).await?;
 
-            println!("VerifiedGroup#{} created successfully.", chat_id);
+            println!("Group#{} created and protected successfully.", chat_id);
         }
         "addmember" => {
             ensure!(sel_chat.is_some(), "No chat selected");
@@ -875,9 +881,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg.set_text(Some(arg1.to_string()));
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
-        "updatedevicechats" => {
-            context.update_device_chats().await?;
-        }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
@@ -909,7 +912,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                         "archive" => ChatVisibility::Archived,
                         "unarchive" | "unpin" => ChatVisibility::Normal,
                         "pin" => ChatVisibility::Pinned,
-                        _ => panic!("Unexpected command (This should never happen)"),
+                        _ => unreachable!("arg0={:?}", arg0),
+                    },
+                )
+                .await?;
+        }
+        "protect" | "unprotect" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id
+                .set_protection(
+                    &context,
+                    match arg0 {
+                        "protect" => ProtectionStatus::Protected,
+                        "unprotect" => ProtectionStatus::Unprotected,
+                        _ => unreachable!("arg0={:?}", arg0),
                     },
                 )
                 .await?;
@@ -948,12 +965,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             message::markseen_msgs(&context, msg_ids).await;
         }
-        "star" | "unstar" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = vec![MsgId::new(0); 1];
-            msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::star_msgs(&context, msg_ids, arg0 == "star").await;
-        }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];

python/src/deltachat/__init__.py

@@ -77,6 +77,7 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
+        ac.set_config("bot", "1")
         configtracker = ac.configure()
         configtracker.wait_finish()
 

python/src/deltachat/account.py

@@ -263,6 +263,17 @@ class Account(object):
         """
         return Contact(self, contact_id)
 
+    def get_blocked_contacts(self):
+        """ return a list of all blocked contacts.
+
+        :returns: list of :class:`deltachat.contact.Contact` objects.
+        """
+        dc_array = ffi.gc(
+            lib.dc_get_blocked_contacts(self._dc_context),
+            lib.dc_array_unref
+        )
+        return list(iter_array(dc_array, lambda x: Contact(self, x)))
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -336,6 +347,9 @@ class Account(object):
     def get_deaddrop_chat(self):
         return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
+    def get_device_chat(self):
+        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
+
     def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
@@ -546,6 +560,9 @@ class Account(object):
         You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
         account is started.
 
+        If you are using this from a test, you may want to call
+        wait_all_initial_fetches() afterwards.
+
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 

python/src/deltachat/chat.py

@@ -57,10 +57,7 @@ class Chat(object):
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
-        return lib.dc_chat_get_type(self._dc_chat) in (
-            const.DC_CHAT_TYPE_GROUP,
-            const.DC_CHAT_TYPE_VERIFIED_GROUP
-        )
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
@@ -85,12 +82,20 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def is_verified(self):
-        """ return True if this chat is a verified group.
+    def can_send(self):
+        """Check if messages can be sent to a give chat.
+        This is not true eg. for the deaddrop or for the device-talk
 
-        :returns: True if chat is verified, False otherwise.
+        :returns: True if the chat is writable, False otherwise
         """
-        return lib.dc_chat_is_verified(self._dc_chat)
+        return lib.dc_chat_can_send(self._dc_chat)
+
+    def is_protected(self):
+        """ return True if this chat is a protected chat.
+
+        :returns: True if chat is protected, False otherwise.
+        """
+        return lib.dc_chat_is_protected(self._dc_chat)
 
     def get_name(self):
         """ return name of this chat.

python/src/deltachat/contact.py

@@ -52,9 +52,17 @@ class Contact(object):
         return lib.dc_contact_is_blocked(self._dc_contact)
 
     def set_blocked(self, block=True):
-        """ Block or unblock a contact. """
+        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
         return lib.dc_block_contact(self.account._dc_context, self.id, block)
 
+    def block(self):
+        """ Block this contact. Message will not be seen/retrieved from this contact. """
+        return lib.dc_block_contact(self.account._dc_context, self.id, True)
+
+    def unblock(self):
+        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
+        return lib.dc_block_contact(self.account._dc_context, self.id, False)
+
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)

python/src/deltachat/direct_imap.py

@@ -24,12 +24,13 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    imap = DirectImap(account)
-    if imap.select_config_folder("mvbox"):
+    if not hasattr(account, "direct_imap"):
+        imap = DirectImap(account)
+        if imap.select_config_folder("mvbox"):
+            imap.delete(ALL, expunge=True)
+        assert imap.select_config_folder("inbox")
         imap.delete(ALL, expunge=True)
-    assert imap.select_config_folder("inbox")
-    imap.delete(ALL, expunge=True)
-    setattr(account, "direct_imap", imap)
+        setattr(account, "direct_imap", imap)
 
 
 @deltachat.global_hookimpl

python/src/deltachat/events.py

@@ -1,6 +1,7 @@
 import threading
 import time
 import re
+import os
 from queue import Queue, Empty
 
 import deltachat
@@ -48,6 +49,15 @@ class FFIEventLogger:
         if self.logid:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+
+        if os.name == "posix":
+            WARN = '\033[93m'
+            ERROR = '\033[91m'
+            ENDC = '\033[0m'
+            if message.startswith("DC_EVENT_WARNING"):
+                s = WARN + s + ENDC
+            if message.startswith("DC_EVENT_ERROR"):
+                s = ERROR + s + ENDC
         with self._loglock:
             print(s, flush=True)
 
@@ -111,6 +121,15 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
+    def wait_all_initial_fetches(self):
+        """Has to be called after start_io() to wait for fetch_existing_msgs to run
+        so that new messages are not mistaken for old ones:
+        - ac1 and ac2 are created
+        - ac1 sends a message to ac2
+        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
+        - therefore no DC_EVENT_INCOMING_MSG is sent"""
+        self.get_info_contains("Done fetching existing messages")
+
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")

python/src/deltachat/message.py

@@ -175,6 +175,27 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
+    @property
+    def quoted_text(self):
+        """Text inside the quote
+
+        :returns: Quoted text"""
+        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
+
+    @property
+    def quote(self):
+        """Quote getter
+
+        :returns: Quoted message, if found in the database"""
+        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
+        if msg:
+            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
+
+    @quote.setter
+    def quote(self, quoted_message):
+        """Quote setter"""
+        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 

python/src/deltachat/testplugin.py

@@ -342,19 +342,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
+            if hasattr(account, "direct_imap"):
+                # Attach the existing direct_imap. If we did not do this, a new one would be created and
+                # delete existing messages (see dc_account_extra_configure(configure))
+                ac.direct_imap = account.direct_imap
             ac._configtracker = ac.configure()
             return ac
 
         def wait_configure_and_start_io(self):
+            started_accounts = []
             for acc in self._accounts:
                 if hasattr(acc, "_configtracker"):
                     acc._configtracker.wait_finish()
+                    acc._evtracker.consume_events()
+                    acc.get_device_chat().mark_noticed()
                     del acc._configtracker
                 acc.set_config("bcc_self", "0")
                 if acc.is_configured() and not acc.is_started():
                     acc.start_io()
+                    started_accounts.append(acc)
                 print("{}: {} account was successfully setup".format(
                     acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in started_accounts:
+                acc._evtracker.wait_all_initial_fetches()
 
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__

python/tests/test_account.py

@@ -129,6 +129,20 @@ class TestOfflineContact:
         assert not contact1.is_blocked()
         assert not contact1.is_verified()
 
+    def test_get_blocked(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact1 = ac1.create_contact("some1@example.org", name="some1")
+        contact2 = ac1.create_contact("some2@example.org", name="some2")
+        ac1.create_contact("some3@example.org", name="some3")
+        assert ac1.get_blocked_contacts() == []
+        contact1.block()
+        assert ac1.get_blocked_contacts() == [contact1]
+        contact2.block()
+        blocked = ac1.get_blocked_contacts()
+        assert len(blocked) == 2 and contact1 in blocked and contact2 in blocked
+        contact2.unblock()
+        assert ac1.get_blocked_contacts() == [contact1]
+
     def test_create_self_contact(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact(ac1.get_config("addr"))
@@ -166,6 +180,16 @@ class TestOfflineContact:
         with pytest.raises(ValueError):
             ac1.create_chat(ac3)
 
+    def test_contact_rename(self, acfactory):
+        ac1 = acfactory.get_configured_offline_account()
+        contact = ac1.create_contact("some1@example.com", name="some1")
+        chat = ac1.create_chat(contact)
+        assert chat.get_name() == "some1"
+        ac1.create_contact("some1@example.com", name="renamed")
+        ev = ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED")
+        assert ev.data1 == chat.id
+        assert chat.get_name() == "renamed"
+
 
 class TestOfflineChat:
     @pytest.fixture
@@ -466,6 +490,21 @@ class TestOfflineChat:
         assert not res.is_ask_verifygroup()
         assert res.contact_id == 10
 
+    def test_quote(self, chat1):
+        """Offline quoting test"""
+        msg = Message.new_empty(chat1.account, "text")
+        msg.set_text("Multi\nline\nmessage")
+        assert msg.quoted_text is None
+
+        # Prepare message to assign it a Message-Id.
+        # Messages without Message-Id cannot be quoted.
+        msg = chat1.prepare_message(msg)
+
+        reply_msg = Message.new_empty(chat1.account, "text")
+        reply_msg.set_text("reply")
+        reply_msg.quote = msg
+        assert reply_msg.quoted_text == "Multi\nline\nmessage"
+
     def test_group_chat_many_members_add_remove(self, ac1, lp):
         lp.sec("ac1: creating group chat with 10 other members")
         chat = ac1.create_group_chat(name="title1")
@@ -872,7 +911,13 @@ class TestOnlineAccount:
         lp.sec("mark messages as seen on ac2, wait for changes on ac1")
         ac2.direct_imap.idle_start()
         ac1.direct_imap.idle_start()
+
         ac2.mark_seen_messages([msg2, msg4])
+        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+        assert msg2.chat.id == msg4.chat.id
+        assert ev.data1 == msg2.chat.id
+        assert ev.data2 == 0
+
         ac2.direct_imap.idle_check(terminate=True)
         lp.step("1")
         for i in range(2):
@@ -1001,7 +1046,68 @@ class TestOnlineAccount:
         assert msg_in.text == text2
         assert ac1.get_config("addr") in [x.addr for x in msg_in.chat.get_contacts()]
 
+    def test_no_draft_if_cant_send(self, acfactory):
+        """Tests that no quote can be set if the user can't send to this chat"""
+        ac1 = acfactory.get_one_online_account()
+        device_chat = ac1.get_device_chat()
+        msg = Message.new_empty(ac1, "text")
+        device_chat.set_draft(msg)
+
+        assert not device_chat.can_send()
+        assert device_chat.get_draft() is None
+
+    def test_prefer_encrypt(self, acfactory, lp):
+        """Test quorum rule for encryption preference in 1:1 and group chat."""
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+        ac1.set_config("e2ee_enabled", "0")
+        ac2.set_config("e2ee_enabled", "1")
+        ac3.set_config("e2ee_enabled", "0")
+
+        # Make sure we do not send a copy to ourselves. This is to
+        # test that we count own preference even when we are not in
+        # the recipient list.
+        ac1.set_config("bcc_self", "0")
+        ac2.set_config("bcc_self", "0")
+        ac3.set_config("bcc_self", "0")
+
+        acfactory.introduce_each_other([ac1, ac2, ac3])
+
+        lp.sec("ac1: sending message to ac2")
+        chat1 = ac1.create_chat(ac2)
+        msg1 = chat1.send_text("message1")
+        assert not msg1.is_encrypted()
+        ac2._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac2: sending message to ac1")
+        chat2 = ac2.create_chat(ac1)
+        msg2 = chat2.send_text("message2")
+        assert not msg2.is_encrypted()
+        ac1._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac1: sending message to group chat with ac2 and ac3")
+        group = ac1.create_group_chat("hello")
+        group.add_contact(ac2)
+        group.add_contact(ac3)
+        msg3 = group.send_text("message3")
+        assert not msg3.is_encrypted()
+        ac2._evtracker.wait_next_incoming_message()
+        ac3._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac3: start preferring encryption and inform ac1")
+        ac3.set_config("e2ee_enabled", "1")
+        chat3 = ac3.create_chat(ac1)
+        msg4 = chat3.send_text("message4")
+        # ac1 still does not prefer encryption
+        assert not msg4.is_encrypted()
+        ac1._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac1: sending another message to group chat with ac2 and ac3")
+        msg5 = group.send_text("message5")
+        # Majority prefers encryption now
+        assert msg5.is_encrypted()
+
     def test_reply_encrypted(self, acfactory, lp):
+        """Test that replies to encrypted messages are encrypted."""
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         lp.sec("ac1: create chat with ac2")
@@ -1029,26 +1135,26 @@ class TestOnlineAccount:
         print("ac2: e2ee_enabled={}".format(ac2.get_config("e2ee_enabled")))
         ac1.set_config("e2ee_enabled", "0")
 
-        # Set unprepared and unencrypted draft to test that it is not
-        # taken into account when determining whether last message is
-        # encrypted.
-        msg_draft = Message.new_empty(ac1, "text")
-        msg_draft.set_text("message2 -- should be encrypted")
-        chat.set_draft(msg_draft)
+        for quoted_msg in msg1, msg3:
+            # Save the draft with a quote.
+            # It should be encrypted if quoted message is encrypted.
+            msg_draft = Message.new_empty(ac1, "text")
+            msg_draft.set_text("message reply")
+            msg_draft.quote = quoted_msg
+            chat.set_draft(msg_draft)
 
-        # Get the draft, prepare and send it.
-        msg_draft = chat.get_draft()
-        msg_out = chat.prepare_message(msg_draft)
-        chat.send_prepared(msg_out)
+            # Get the draft, prepare and send it.
+            msg_draft = chat.get_draft()
+            msg_out = chat.prepare_message(msg_draft)
+            chat.send_prepared(msg_out)
 
-        chat.set_draft(None)
-        assert chat.get_draft() is None
+            chat.set_draft(None)
+            assert chat.get_draft() is None
 
-        lp.sec("wait for ac2 to receive message")
-        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
-        msg_in = ac2.get_message_by_id(ev.data2)
-        assert msg_in.text == "message2 -- should be encrypted"
-        assert msg_in.is_encrypted()
+            msg_in = ac2._evtracker.wait_next_incoming_message()
+            assert msg_in.text == "message reply"
+            assert msg_in.quoted_text == quoted_msg.text
+            assert msg_in.is_encrypted() == quoted_msg.is_encrypted()
 
     def test_saved_mime_on_received_message(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1261,7 +1367,7 @@ class TestOnlineAccount:
         ac1, ac2 = acfactory.get_two_online_accounts()
         lp.sec("ac1: create verified-group QR, ac2 scans and joins")
         chat1 = ac1.create_group_chat("hello", verified=True)
-        assert chat1.is_verified()
+        assert chat1.is_protected()
         qr = chat1.get_join_qr()
         lp.sec("ac2: start QR-code based join-group protocol")
         chat2 = ac2.qr_join_chat(qr)
@@ -1280,7 +1386,7 @@ class TestOnlineAccount:
         lp.sec("ac2: read message and check it's verified chat")
         msg = ac2._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
-        assert msg.chat.is_verified()
+        assert msg.chat.is_protected()
         assert msg.is_encrypted()
 
         lp.sec("ac2: send message and let ac1 read it")
@@ -1417,6 +1523,8 @@ class TestOnlineAccount:
         contact = ac1.create_contact(ac2)
         contact.set_blocked()
         assert contact.is_blocked()
+        ev = ac1._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        assert ev.data1 == contact.id
 
         lp.sec("ac2 sends a message to ac1 that does not arrive because it is blocked")
         ac2.create_chat(ac1).send_text("This will not arrive!")
@@ -1560,6 +1668,7 @@ class TestOnlineAccount:
         chat41 = ac4.create_chat(ac1)
         chat42 = ac4.create_chat(ac2)
         ac4.start_io()
+        ac4._evtracker.wait_all_initial_fetches()
 
         lp.sec("ac1: creating group chat with 2 other members")
         chat = ac1.create_group_chat("title", contacts=[ac2, ac3])
@@ -1727,6 +1836,37 @@ class TestOnlineAccount:
 
         assert len(imap2.get_all_messages()) == 1
 
+    def test_configure_error_msgs(self, acfactory):
+        ac1, configdict = acfactory.get_online_config()
+        ac1.update_config(configdict)
+        ac1.set_config("mail_pw", "abc")  # Wrong mail pw
+        ac1.configure()
+        while True:
+            ev = ac1._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+            if ev.data1 == 0:
+                break
+        # Password is wrong so it definitely has to say something about "password"
+        assert "password" in ev.data2
+
+        ac2, configdict = acfactory.get_online_config()
+        ac2.update_config(configdict)
+        ac2.set_config("addr", "abc@def.invalid")  # mail server can't be reached
+        ac2.configure()
+        while True:
+            ev = ac2._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+            if ev.data1 == 0:
+                break
+        # Can't connect so it probably should say something about "internet"
+        # again, should not repeat itself
+        # If this fails then probably `e.msg.to_lowercase().contains("could not resolve")`
+        # in configure/mod.rs returned false because the error message was changed
+        # (i.e. did not contain "could not resolve" anymore)
+        assert (ev.data2.count("internet") + ev.data2.count("network")) == 1
+        # Should mention that it can't connect:
+        assert ev.data2.count("connect") == 1
+        # The users do not know what "configuration" is
+        assert "configuration" not in ev.data2.lower()
+
     def test_name_changes(self, acfactory):
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac1.set_config("displayname", "Account 1")
@@ -1750,6 +1890,8 @@ class TestOnlineAccount:
         # Explicitly rename contact on ac2 to "Renamed"
         ac2.create_contact(contact, name="Renamed")
         assert contact.name == "Renamed"
+        ev = ac2._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        assert ev.data1 == contact.id
 
         # ac1 also renames itself into "Renamed"
         assert update_name() == "Renamed"
@@ -1767,6 +1909,82 @@ class TestOnlineAccount:
             # No renames should happen after explicit rename
             assert updated_name == "Renamed"
 
+    def test_group_quote(self, acfactory, lp):
+        """Test quoting in a group with a new member who have not seen the quoted message."""
+        ac1, ac2, ac3 = accounts = acfactory.get_many_online_accounts(3)
+        acfactory.introduce_each_other(accounts)
+        chat = ac1.create_group_chat(name="quote group")
+        chat.add_contact(ac2)
+
+        lp.sec("ac1: sending message")
+        out_msg = chat.send_text("hello")
+
+        lp.sec("ac2: receiving message")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        chat.add_contact(ac3)
+        ac2._evtracker.wait_next_incoming_message()
+        ac3._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac2: sending reply with a quote")
+        reply_msg = Message.new_empty(msg.chat.account, "text")
+        reply_msg.set_text("reply")
+        reply_msg.quote = msg
+        reply_msg = msg.chat.prepare_message(reply_msg)
+        assert reply_msg.quoted_text == "hello"
+        msg.chat.send_prepared(reply_msg)
+
+        lp.sec("ac3: receiving reply")
+        received_reply = ac3._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "reply"
+        assert received_reply.quoted_text == "hello"
+        # ac3 was not in the group and has not received quoted message
+        assert received_reply.quote is None
+
+        lp.sec("ac1: receiving reply")
+        received_reply = ac1._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "reply"
+        assert received_reply.quoted_text == "hello"
+        assert received_reply.quote.id == out_msg.id
+
+    @pytest.mark.parametrize("mvbox_move", [False, True])
+    def test_add_all_recipients_as_contacts(self, acfactory, lp, mvbox_move):
+        """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
+        This way, we can already offer them some email addresses they can write to.
+
+        Also test that existing emails are fetched during onboarding.
+
+        Lastly, tests that bcc_self messages moved to the mvbox are marked as read."""
+        ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure_and_start_io()
+
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("send out message with bcc to ourselves")
+        if mvbox_move:
+            ac1.direct_imap.select_config_folder("mvbox")
+        ac1.direct_imap.idle_start()
+        ac1.set_config("bcc_self", "1")
+        chat.send_text("message text")
+
+        # now wait until the bcc_self message arrives
+        # Also test that bcc_self messages moved to the mvbox are marked as read.
+        assert ac1.direct_imap.idle_wait_for_seen()
+
+        ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone._configtracker.wait_finish()
+        ac1_clone.start_io()
+
+        ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        ac2_addr = ac2.get_config("addr")
+        assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
+
+        msg = ac1_clone._evtracker.wait_next_messages_changed()
+        assert msg.text == "message text"
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):

spec.md

@@ -32,7 +32,7 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
+by the [Protected Headers](https://tools.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages

src/accounts.rs

@@ -1,10 +1,8 @@
 use std::collections::BTreeMap;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::task::{Context as TaskContext, Poll};
 
 use async_std::fs;
 use async_std::path::PathBuf;
+use async_std::prelude::*;
 use async_std::sync::{Arc, RwLock};
 use uuid::Uuid;
 
@@ -225,61 +223,42 @@ impl Accounts {
 
     /// Unified event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
-        let emitters = self
+        let emitters: Vec<_> = self
             .accounts
             .read()
             .await
             .iter()
-            .map(|(id, a)| EmitterWrapper {
-                id: *id,
-                emitter: a.get_event_emitter(),
-                done: AtomicBool::new(false),
-            })
+            .map(|(_id, a)| a.get_event_emitter())
             .collect();
 
-        EventEmitter(emitters)
+        EventEmitter(futures::stream::select_all(emitters))
     }
 }
 
+#[derive(Debug)]
+pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
+
 impl EventEmitter {
-    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
-    pub fn recv_sync(&self) -> Option<Event> {
+    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
+    pub fn recv_sync(&mut self) -> Option<Event> {
         async_std::task::block_on(self.recv())
     }
 
-    /// Async recv of an event. Return `None` if the `Sender` has been droped.
-    pub async fn recv(&self) -> Option<Event> {
-        futures::future::poll_fn(|cx| Pin::new(self).recv_poll(cx)).await
-    }
-
-    fn recv_poll(self: Pin<&Self>, _cx: &mut TaskContext<'_>) -> Poll<Option<Event>> {
-        for e in &*self.0 {
-            if e.done.load(Ordering::Acquire) {
-                // skip emitters that are already done
-                continue;
-            }
-
-            match e.emitter.try_recv() {
-                Ok(event) => return Poll::Ready(Some(event)),
-                Err(async_std::sync::TryRecvError::Disconnected) => {
-                    e.done.store(true, Ordering::Release);
-                }
-                Err(async_std::sync::TryRecvError::Empty) => {}
-            }
-        }
-
-        Poll::Pending
+    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
+    pub async fn recv(&mut self) -> Option<Event> {
+        self.0.next().await
     }
 }
 
-#[derive(Debug)]
-pub struct EventEmitter(Vec<EmitterWrapper>);
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
 
-#[derive(Debug)]
-struct EmitterWrapper {
-    id: u32,
-    emitter: crate::events::EventEmitter,
-    done: AtomicBool,
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
+    }
 }
 
 pub const CONFIG_NAME: &str = "accounts.toml";

src/chatlist.rs

@@ -362,9 +362,7 @@ impl Chatlist {
         let mut lastcontact = None;
 
         let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
-            if lastmsg.from_id != DC_CONTACT_ID_SELF
-                && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
-            {
+            if lastmsg.from_id != DC_CONTACT_ID_SELF && chat.typ == Chattype::Group {
                 lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
             }
 
@@ -440,13 +438,13 @@ mod tests {
     #[async_std::test]
     async fn test_try_load() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat")
+        let chat_id2 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "b chat")
             .await
             .unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat")
+        let chat_id3 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "c chat")
             .await
             .unwrap();
 
@@ -489,7 +487,7 @@ mod tests {
     async fn test_sort_self_talk_up_on_forward() {
         let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
-        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
@@ -546,7 +544,7 @@ mod tests {
     #[async_std::test]
     async fn test_get_summary_unwrap() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 

src/config.rs

@@ -69,6 +69,9 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
     MediaQuality,
 
+    #[strum(props(default = "1"))]
+    FetchExisting,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -121,9 +124,11 @@ pub enum Config {
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
 
-    #[strum(props(default = "0"))]
+    Bot,
+
     /// Whether we send a warning if the password is wrong (set to false when we send a warning
     /// because we do not want to send a second warning)
+    #[strum(props(default = "0"))]
     NotifyAboutWrongPw,
 
     /// address to webrtc instance to use for videochats

src/configure/mod.rs

@@ -8,11 +8,11 @@ mod server_params;
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
 use async_std::task;
+use itertools::Itertools;
+use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::config::Config;
-use crate::constants::*;
-use crate::context::Context;
 use crate::dc_tools::*;
 use crate::imap::Imap;
 use crate::login_param::{LoginParam, ServerLoginParam};
@@ -22,6 +22,8 @@ use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
 use crate::stock::StockMessage;
 use crate::{chat, e2ee, provider};
+use crate::{constants::*, job};
+use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
@@ -124,7 +126,8 @@ impl Context {
                     Some(
                         self.stock_string_repl_str(
                             StockMessage::ConfigurationFailed,
-                            err.to_string(),
+                            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                            format!("{:#}", err),
                         )
                         .await
                     )
@@ -160,6 +163,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         DC_LP_AUTH_NORMAL as i32
     };
 
+    //let ctx2 = ctx.clone();
+    //let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
+
     // Step 1: Load the parameters and check email-address and password
 
     if oauth2 {
@@ -211,28 +217,32 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 500);
 
-    let servers = expand_param_vector(
-        param_autoconfig.unwrap_or_else(|| {
-            vec![
-                ServerParams {
-                    protocol: Protocol::IMAP,
-                    hostname: param.imap.server.clone(),
-                    port: param.imap.port,
-                    socket: param.imap.security,
-                    username: param.imap.user.clone(),
-                },
-                ServerParams {
-                    protocol: Protocol::SMTP,
-                    hostname: param.smtp.server.clone(),
-                    port: param.smtp.port,
-                    socket: param.smtp.security,
-                    username: param.smtp.user.clone(),
-                },
-            ]
-        }),
-        &param.addr,
-        &param_domain,
-    );
+    let mut servers = param_autoconfig.unwrap_or_default();
+    if !servers
+        .iter()
+        .any(|server| server.protocol == Protocol::IMAP)
+    {
+        servers.push(ServerParams {
+            protocol: Protocol::IMAP,
+            hostname: param.imap.server.clone(),
+            port: param.imap.port,
+            socket: param.imap.security,
+            username: param.imap.user.clone(),
+        })
+    }
+    if !servers
+        .iter()
+        .any(|server| server.protocol == Protocol::SMTP)
+    {
+        servers.push(ServerParams {
+            protocol: Protocol::SMTP,
+            hostname: param.smtp.server.clone(),
+            port: param.smtp.port,
+            socket: param.smtp.security,
+            username: param.smtp.user.clone(),
+        })
+    }
+    let servers = expand_param_vector(servers, &param.addr, &param_domain);
 
     progress!(ctx, 550);
 
@@ -250,22 +260,28 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     let smtp_config_task = task::spawn(async move {
         let mut smtp_configured = false;
+        let mut errors = Vec::new();
         for smtp_server in smtp_servers {
             smtp_param.user = smtp_server.username.clone();
             smtp_param.server = smtp_server.hostname.clone();
             smtp_param.port = smtp_server.port;
             smtp_param.security = smtp_server.socket;
 
-            if try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp).await {
-                smtp_configured = true;
-                break;
+            match try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp)
+                .await
+            {
+                Ok(_) => {
+                    smtp_configured = true;
+                    break;
+                }
+                Err(e) => errors.push(e),
             }
         }
 
         if smtp_configured {
-            Some(smtp_param)
+            Ok(smtp_param)
         } else {
-            None
+            Err(errors)
         }
     });
 
@@ -281,15 +297,19 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         .filter(|params| params.protocol == Protocol::IMAP)
         .collect();
     let imap_servers_count = imap_servers.len();
+    let mut errors = Vec::new();
     for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
         param.imap.user = imap_server.username.clone();
         param.imap.server = imap_server.hostname.clone();
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
 
-        if try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
-            imap_configured = true;
-            break;
+        match try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
+            Ok(_) => {
+                imap_configured = true;
+                break;
+            }
+            Err(e) => errors.push(e),
         }
         progress!(
             ctx,
@@ -297,16 +317,19 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         );
     }
     if !imap_configured {
-        bail!("IMAP autoconfig did not succeed");
+        bail!(nicer_configuration_error(ctx, errors).await);
     }
 
     progress!(ctx, 850);
 
     // Wait for SMTP configuration
-    if let Some(smtp_param) = smtp_config_task.await {
-        param.smtp = smtp_param;
-    } else {
-        bail!("SMTP autoconfig did not succeed");
+    match smtp_config_task.await {
+        Ok(smtp_param) => {
+            param.smtp = smtp_param;
+        }
+        Err(errors) => {
+            bail!(nicer_configuration_error(ctx, errors).await);
+        }
     }
 
     progress!(ctx, 900);
@@ -334,7 +357,14 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     e2ee::ensure_secret_key_exists(ctx).await?;
     info!(ctx, "key generation completed");
 
+    job::add(
+        ctx,
+        job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
+    )
+    .await;
+
     progress!(ctx, 940);
+    //update_device_chats_handle.await?;
 
     Ok(())
 }
@@ -478,7 +508,7 @@ async fn try_imap_one_param(
     addr: &str,
     oauth2: bool,
     imap: &mut Imap,
-) -> bool {
+) -> Result<(), ConfigurationError> {
     let inf = format!(
         "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
         param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
@@ -487,10 +517,13 @@ async fn try_imap_one_param(
 
     if let Err(err) = imap.connect(context, param, addr, oauth2).await {
         info!(context, "failure: {}", err);
-        false
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
     } else {
         info!(context, "success: {}", inf);
-        true
+        Ok(())
     }
 }
 
@@ -500,7 +533,7 @@ async fn try_smtp_one_param(
     addr: &str,
     oauth2: bool,
     smtp: &mut Smtp,
-) -> bool {
+) -> Result<(), ConfigurationError> {
     let inf = format!(
         "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
         param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
@@ -509,27 +542,63 @@ async fn try_smtp_one_param(
 
     if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
         info!(context, "failure: {}", err);
-        false
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
     } else {
         info!(context, "success: {}", inf);
         smtp.disconnect().await;
-        true
+        Ok(())
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+#[error("Trying {config}…\nError: {msg}")]
+pub struct ConfigurationError {
+    config: String,
+    msg: String,
+}
+
+async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationError>) -> String {
+    let first_err = if let Some(f) = errors.first() {
+        f
+    } else {
+        // This means configuration failed but no errors have been captured. This should never
+        // happen, but if it does, the user will see classic "Error: no error".
+        return "no error".to_string();
+    };
+
+    if errors
+        .iter()
+        .all(|e| e.msg.to_lowercase().contains("could not resolve"))
+    {
+        return context
+            .stock_str(StockMessage::ErrorNoNetwork)
+            .await
+            .to_string();
+    }
+
+    if errors.iter().all(|e| e.msg == first_err.msg) {
+        return first_err.msg.to_string();
+    }
+
+    errors.iter().map(|e| e.to_string()).join("\n\n")
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("Invalid email address: {0:?}")]
     InvalidEmailAddress(String),
 
-    #[error("XML error at position {position}")]
+    #[error("XML error at position {position}: {error}")]
     InvalidXml {
         position: usize,
         #[source]
         error: quick_xml::Error,
     },
 
-    #[error("Failed to get URL")]
+    #[error("Failed to get URL: {0}")]
     ReadUrlError(#[from] self::read_url::Error),
 
     #[error("Number of redirection is exceeded")]

src/configure/read_url.rs

@@ -12,7 +12,7 @@ pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
     match surf::get(url).recv_string().await {
         Ok(res) => Ok(res),
         Err(err) => {
-            info!(context, "Can\'t read URL {}", url);
+            info!(context, "Can\'t read URL {}: {}", url, err);
 
             Err(Error::GetError(err))
         }

src/constants.rs

@@ -1,11 +1,9 @@
 //! # Constants
 use deltachat_derive::*;
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
-lazy_static! {
-    pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
-}
+pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION").to_string());
 
 #[derive(
     Debug,
@@ -108,14 +106,16 @@ pub const DC_GCL_ADD_SELF: usize = 0x02;
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 
+// warn about an outdated app after a given number of days.
+// as we use the "provider-db generation date" as reference (that might not be updated very often)
+// and as not all system get speedy updates,
+// do not use too small value that will annoy users checking for nonexistant updates.
+pub const DC_OUTDATED_WARNING_DAYS: i64 = 365;
+
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 pub const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
 pub const DC_CHAT_ID_TRASH: u32 = 3;
-/// a message is just in creation but not yet assigned to a chat (eg. we may need the message ID to set up blobs; this avoids unready message to be sent and shown)
-pub const DC_CHAT_ID_MSGS_IN_CREATION: u32 = 4;
-/// virtual chat showing all messages flagged with msgs.starred=2
-pub const DC_CHAT_ID_STARRED: u32 = 5;
 /// only an indicator in a chatlist
 pub const DC_CHAT_ID_ARCHIVED_LINK: u32 = 6;
 /// only an indicator in a chatlist
@@ -143,7 +143,6 @@ pub enum Chattype {
     Undefined = 0,
     Single = 100,
     Group = 120,
-    VerifiedGroup = 130,
 }
 
 impl Default for Chattype {
@@ -193,6 +192,9 @@ pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
 
+/// How many existing messages shall be fetched after configuration.
+pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
+
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 

src/contact.rs

@@ -3,7 +3,7 @@
 use async_std::path::PathBuf;
 use deltachat_derive::*;
 use itertools::Itertools;
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use regex::Regex;
 
 use crate::aheader::EncryptPreference;
@@ -16,7 +16,7 @@ use crate::error::{bail, ensure, format_err, Result};
 use crate::events::EventType;
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::message::{MessageState, MsgId};
+use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::*;
 use crate::peerstate::*;
@@ -247,13 +247,12 @@ impl Contact {
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, name, addr, Origin::ManuallyCreated).await?;
         let blocked = Contact::is_blocked_load(context, contact_id).await;
-        context.emit_event(EventType::ContactsChanged(
-            if sth_modified == Modifier::Created {
-                Some(contact_id)
-            } else {
-                None
-            },
-        ));
+        match sth_modified {
+            Modifier::None => {}
+            Modifier::Modified | Modifier::Created => {
+                context.emit_event(EventType::ContactsChanged(Some(contact_id)))
+            }
+        }
         if blocked {
             Contact::unblock(context, contact_id).await;
         }
@@ -261,10 +260,9 @@ impl Contact {
         Ok(contact_id)
     }
 
-    /// Mark all messages sent by the given contact
-    /// as *noticed*.  See also dc_marknoticed_chat() and dc_markseen_msgs()
-    ///
-    /// Calling this function usually results in the event `#DC_EVENT_MSGS_CHANGED`.
+    /// Mark messages from a contact as noticed.
+    /// The contact is expected to belong to the deaddrop,
+    /// therefore, DC_EVENT_MSGS_NOTICED(DC_CHAT_ID_DEADDROP) is emitted.
     pub async fn mark_noticed(context: &Context, id: u32) {
         if context
             .sql
@@ -275,10 +273,7 @@ impl Contact {
             .await
             .is_ok()
         {
-            context.emit_event(EventType::MsgsChanged {
-                chat_id: ChatId::new(0),
-                msg_id: MsgId::new(0),
-            });
+            context.emit_event(EventType::MsgsNoticed(ChatId::new(DC_CHAT_ID_DEADDROP)));
         }
     }
 
@@ -457,10 +452,20 @@ impl Contact {
                 if update_name {
                     // Update the contact name also if it is used as a group name.
                     // This is one of the few duplicated data, however, getting the chat list is easier this way.
-                    context.sql.execute(
-                    "UPDATE chats SET name=? WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    paramsv![new_name, Chattype::Single, row_id]
-                ).await.ok();
+                    let chat_id = context.sql.query_get_value::<i32>(
+                        context,
+                        "SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
+                        paramsv![Chattype::Single, row_id]
+                    ).await;
+                    if let Some(chat_id) = chat_id {
+                        match context.sql.execute("UPDATE chats SET name=? WHERE id=? AND name!=?1", paramsv![new_name, chat_id]).await {
+                            Err(err) => warn!(context, "Can't update chat name: {}", err),
+                            Ok(count) => if count > 0 {
+                                // Chat name updated
+                                context.emit_event(EventType::ChatModified(ChatId::new(chat_id as u32)));
+                            }
+                        }
+                    }
                 }
                 sth_modified = Modifier::Modified;
             }
@@ -1048,9 +1053,7 @@ pub fn addr_normalize(addr: &str) -> &str {
 }
 
 fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (String, String) {
-    lazy_static! {
-        static ref ADDR_WITH_NAME_REGEX: Regex = Regex::new("(.*)<(.*)>").unwrap();
-    }
+    static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
     if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
             if name.as_ref().is_empty() {
@@ -1091,12 +1094,12 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
             // However, I'm not sure about this point; it may be confusing if the user wants to add other people;
             // this would result in recreating the same group...)
             if context.sql.execute(
-                    "UPDATE chats SET blocked=? WHERE type=? AND id IN (SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    paramsv![new_blocking, 100, contact_id as i32],
-                ).await.is_ok() {
-                    Contact::mark_noticed(context, contact_id).await;
-                    context.emit_event(EventType::ContactsChanged(None));
-                }
+                "UPDATE chats SET blocked=? WHERE type=? AND id IN (SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
+                paramsv![new_blocking, 100, contact_id as i32]).await.is_ok()
+            {
+                Contact::mark_noticed(context, contact_id).await;
+                context.emit_event(EventType::ContactsChanged(Some(contact_id)));
+            }
         }
     }
 }

src/context.rs

@@ -136,10 +136,7 @@ impl Context {
         let ctx = Context {
             inner: Arc::new(inner),
         };
-        ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, false).await,
-            "Failed opening sqlite database"
-        );
+        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }

src/dc_receive_imf.rs

@@ -4,7 +4,7 @@ use sha2::{Digest, Sha256};
 
 use mailparse::SingleInfo;
 
-use crate::chat::{self, Chat, ChatId};
+use crate::chat::{self, Chat, ChatId, ProtectionStatus};
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
@@ -43,6 +43,17 @@ pub async fn dc_receive_imf(
     server_folder: impl AsRef<str>,
     server_uid: u32,
     seen: bool,
+) -> Result<()> {
+    dc_receive_imf_inner(context, imf_raw, server_folder, server_uid, seen, false).await
+}
+
+pub(crate) async fn dc_receive_imf_inner(
+    context: &Context,
+    imf_raw: &[u8],
+    server_folder: impl AsRef<str>,
+    server_uid: u32,
+    seen: bool,
+    fetching_existing_messages: bool,
 ) -> Result<()> {
     info!(
         context,
@@ -169,6 +180,7 @@ pub async fn dc_receive_imf(
             &mut insert_msg_id,
             &mut created_db_entries,
             &mut create_event_to_send,
+            fetching_existing_messages,
         )
         .await
         {
@@ -335,6 +347,7 @@ async fn add_parts(
     insert_msg_id: &mut MsgId,
     created_db_entries: &mut Vec<(ChatId, MsgId)>,
     create_event_to_send: &mut Option<CreateEvent>,
+    fetching_existing_messages: bool,
 ) -> Result<()> {
     let mut state: MessageState;
     let mut chat_id_blocked = Blocked::Not;
@@ -389,7 +402,7 @@ async fn add_parts(
     let to_id: u32;
 
     if incoming {
-        state = if seen {
+        state = if seen || fetching_existing_messages {
             MessageState::InSeen
         } else {
             MessageState::InFresh
@@ -532,6 +545,10 @@ async fn add_parts(
             && show_emails != ShowEmails::All
         {
             state = MessageState::InNoticed;
+        } else if fetching_existing_messages && Blocked::Deaddrop == chat_id_blocked {
+            // The fetched existing message should be shown in the chatlist-contact-request because
+            // a new user won't find the contact request in the menu
+            state = MessageState::InFresh;
         }
     } else {
         // Outgoing
@@ -628,6 +645,12 @@ async fn add_parts(
         }
     }
 
+    if fetching_existing_messages && mime_parser.decrypting_failed {
+        *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
+        // We are only gathering old messages on first start. We do not want to add loads of non-decryptable messages to the chats.
+        info!(context, "Dropping existing non-decipherable message.");
+    }
+
     // Extract ephemeral timer from the message.
     let mut ephemeral_timer = if let Some(value) = mime_parser.get(HeaderDef::EphemeralTimer) {
         match value.parse::<EphemeralTimer>() {
@@ -691,6 +714,43 @@ async fn add_parts(
         ephemeral_timer = EphemeralTimer::Disabled;
     }
 
+    // if a chat is protected, check additional properties
+    if !chat_id.is_special() {
+        let chat = Chat::load_from_db(context, *chat_id).await?;
+        let new_status = match mime_parser.is_system_message {
+            SystemMessage::ChatProtectionEnabled => Some(ProtectionStatus::Protected),
+            SystemMessage::ChatProtectionDisabled => Some(ProtectionStatus::Unprotected),
+            _ => None,
+        };
+
+        if chat.is_protected() || new_status.is_some() {
+            if let Err(err) =
+                check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
+            {
+                warn!(context, "verification problem: {}", err);
+                let s = format!("{}. See 'Info' for more details", err);
+                mime_parser.repl_msg_by_error(s);
+            } else {
+                // change chat protection only when verification check passes
+                if let Some(new_status) = new_status {
+                    if let Err(e) = chat_id.inner_set_protection(context, new_status).await {
+                        chat::add_info_msg(
+                            context,
+                            *chat_id,
+                            format!("Cannot set protection: {}", e),
+                        )
+                        .await;
+                        return Ok(()); // do not return an error as this would result in retrying the message
+                    }
+                    set_better_msg(
+                        mime_parser,
+                        context.stock_protection_msg(new_status, from_id).await,
+                    );
+                }
+            }
+        }
+    }
+
     // correct message_timestamp, it should not be used before,
     // however, we cannot do this earlier as we need from_id to be set
     let in_fresh = state == MessageState::InFresh;
@@ -815,7 +875,7 @@ async fn add_parts(
                     mime_headers,
                     mime_in_reply_to,
                     mime_references,
-                    part.error,
+                    part.error.take().unwrap_or_default(),
                     ephemeral_timer,
                     ephemeral_timestamp
                 ])?;
@@ -845,6 +905,11 @@ async fn add_parts(
         "Message has {} parts and is assigned to chat #{}.", icnt, chat_id,
     );
 
+    // new outgoing message from another device marks the chat as noticed.
+    if !incoming && !*hidden && !chat_id.is_special() {
+        chat::marknoticed_chat_if_older_than(context, chat_id, sort_timestamp).await?;
+    }
+
     // check event to send
     if chat_id.is_trash() || *hidden {
         *create_event_to_send = None;
@@ -1126,29 +1191,18 @@ async fn create_or_lookup_group(
     set_better_msg(mime_parser, &better_msg);
 
     // check, if we have a chat with this group ID
-    let (mut chat_id, chat_id_verified, _blocked) = chat::get_chat_id_by_grpid(context, &grpid)
+    let (mut chat_id, _, _blocked) = chat::get_chat_id_by_grpid(context, &grpid)
         .await
         .unwrap_or((ChatId::new(0), false, Blocked::Not));
-    if !chat_id.is_unset() {
-        if chat_id_verified {
-            if let Err(err) =
-                check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
-            {
-                warn!(context, "verification problem: {}", err);
-                let s = format!("{}. See 'Info' for more details", err);
-                mime_parser.repl_msg_by_error(s);
-            }
-        }
-        if !chat::is_contact_in_chat(context, chat_id, from_id as u32).await {
-            // The From-address is not part of this group.
-            // It could be a new user or a DSN from a mailer-daemon.
-            // in any case we do not want to recreate the member list
-            // but still show the message as part of the chat.
-            // After all, the sender has a reference/in-reply-to that
-            // points to this chat.
-            let s = context.stock_str(StockMessage::UnknownSenderForChat).await;
-            mime_parser.repl_msg_by_error(s.to_string());
-        }
+    if !chat_id.is_unset() && !chat::is_contact_in_chat(context, chat_id, from_id as u32).await {
+        // The From-address is not part of this group.
+        // It could be a new user or a DSN from a mailer-daemon.
+        // in any case we do not want to recreate the member list
+        // but still show the message as part of the chat.
+        // After all, the sender has a reference/in-reply-to that
+        // points to this chat.
+        let s = context.stock_str(StockMessage::UnknownSenderForChat).await;
+        mime_parser.repl_msg_by_error(s.to_string());
     }
 
     // check if the group does not exist but should be created
@@ -1171,7 +1225,7 @@ async fn create_or_lookup_group(
                 || X_MrAddToGrp.is_some() && addr_cmp(&self_addr, X_MrAddToGrp.as_ref().unwrap()))
     {
         // group does not exist but should be created
-        let create_verified = if mime_parser.get(HeaderDef::ChatVerified).is_some() {
+        let create_protected = if mime_parser.get(HeaderDef::ChatVerified).is_some() {
             if let Err(err) =
                 check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
             {
@@ -1179,9 +1233,9 @@ async fn create_or_lookup_group(
                 let s = format!("{}. See 'Info' for more details", err);
                 mime_parser.repl_msg_by_error(&s);
             }
-            VerifiedStatus::Verified
+            ProtectionStatus::Protected
         } else {
-            VerifiedStatus::Unverified
+            ProtectionStatus::Unprotected
         };
 
         if !allow_creation {
@@ -1194,11 +1248,22 @@ async fn create_or_lookup_group(
             &grpid,
             grpname.as_ref().unwrap(),
             create_blocked,
-            create_verified,
+            create_protected,
         )
         .await;
         chat_id_blocked = create_blocked;
         recreate_member_list = true;
+
+        // once, we have protected-chats explained in UI, we can uncomment the following lines.
+        // ("verified groups" did not add a message anyway)
+        //
+        //if create_protected == ProtectionStatus::Protected {
+        // set from_id=0 as it is not clear that the sender of this random group message
+        // actually really has enabled chat-protection at some point.
+        //chat_id
+        //    .add_protection_msg(context, ProtectionStatus::Protected, false, 0)
+        //    .await?;
+        //}
     }
 
     // again, check chat_id
@@ -1250,7 +1315,10 @@ async fn create_or_lookup_group(
                 }
             }
         }
+    } else if mime_parser.is_system_message == SystemMessage::ChatProtectionEnabled {
+        recreate_member_list = true;
     }
+
     if let Some(avatar_action) = &mime_parser.group_avatar {
         info!(context, "group-avatar change for {}", chat_id);
         if let Ok(mut chat) = Chat::load_from_db(context, chat_id).await {
@@ -1435,7 +1503,7 @@ async fn create_or_lookup_adhoc_group(
         &grpid,
         grpname,
         create_blocked,
-        VerifiedStatus::Unverified,
+        ProtectionStatus::Unprotected,
     )
     .await;
     for &member_id in &member_ids {
@@ -1452,20 +1520,17 @@ async fn create_group_record(
     grpid: impl AsRef<str>,
     grpname: impl AsRef<str>,
     create_blocked: Blocked,
-    create_verified: VerifiedStatus,
+    create_protected: ProtectionStatus,
 ) -> ChatId {
     if context.sql.execute(
-        "INSERT INTO chats (type, name, grpid, blocked, created_timestamp) VALUES(?, ?, ?, ?, ?);",
+        "INSERT INTO chats (type, name, grpid, blocked, created_timestamp, protected) VALUES(?, ?, ?, ?, ?, ?);",
         paramsv![
-            if VerifiedStatus::Unverified != create_verified {
-                Chattype::VerifiedGroup
-            } else {
-                Chattype::Group
-            },
+            Chattype::Group,
             grpname.as_ref(),
             grpid.as_ref(),
             create_blocked,
             time(),
+            create_protected,
         ],
     ).await
     .is_err()
@@ -1530,7 +1595,7 @@ async fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> String {
             },
         )
         .await
-        .unwrap_or_else(|_| member_cs);
+        .unwrap_or(member_cs);
 
     hex_hash(&members)
 }
@@ -1558,7 +1623,7 @@ async fn search_chat_ids_by_contact_ids(
             }
         }
         if !contact_ids.is_empty() {
-            contact_ids.sort();
+            contact_ids.sort_unstable();
             let contact_ids_str = join(contact_ids.iter().map(|x| x.to_string()), ",");
             context.sql.query_map(
                 format!(
@@ -1617,6 +1682,17 @@ async fn check_verified_properties(
 
     ensure!(mimeparser.was_encrypted(), "This message is not encrypted.");
 
+    if mimeparser.get(HeaderDef::ChatVerified).is_none() {
+        // we do not fail here currently, this would exclude (a) non-deltas
+        // and (b) deltas with different protection views across multiple devices.
+        // for group creation or protection enabled/disabled, however, Chat-Verified is respected.
+        warn!(
+            context,
+            "{} did not mark message as protected.",
+            contact.get_addr()
+        );
+    }
+
     // ensure, the contact is verified
     // and the message is signed with a verified key of the sender.
     // this check is skipped for SELF as there is no proper SELF-peerstate
@@ -1706,7 +1782,7 @@ async fn check_verified_properties(
         }
         if !is_verified {
             bail!(
-                "{} is not a member of this verified group",
+                "{} is not a member of this protected chat",
                 to_addr.to_string()
             );
         }
@@ -2154,7 +2230,7 @@ mod tests {
         assert!(one2one.get_visibility() == ChatVisibility::Archived);
 
         // create a group with bob, archive group
-        let group_id = chat::create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
+        let group_id = chat::create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "foo")
             .await
             .unwrap();
         chat::add_contact_to_chat(&t.ctx, group_id, bob_id).await;
@@ -2455,7 +2531,7 @@ mod tests {
             "shenauithz@testrun.org",
             "Mr.un2NYERi1RM.lbQ5F9q-QyJ@tiscali.it",
             include_bytes!("../test-data/message/tiscali_ndn.eml"),
-            "",
+            None,
         )
         .await;
     }
@@ -2467,7 +2543,7 @@ mod tests {
             "hcksocnsofoejx@five.chat",
             "Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org",
             include_bytes!("../test-data/message/testrun_ndn.eml"),
-            "Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"
+            Some("Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"),
         )
         .await;
     }
@@ -2479,7 +2555,7 @@ mod tests {
             "haeclirth.sinoenrat@yahoo.com",
             "1680295672.3657931.1591783872936@mail.yahoo.com",
             include_bytes!("../test-data/message/yahoo_ndn.eml"),
-            "Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com [...]"
+            Some("Failure Notice – Sorry, we were unable to deliver your message to the following address.\n\n<haeclirth.sinoenrat@yahoo.com>:\n554: delivery error: dd Not a valid recipient - atlas117.free.mail.ne1.yahoo.com [...]"),
         )
         .await;
     }
@@ -2491,7 +2567,7 @@ mod tests {
             "assidhfaaspocwaeofi@gmail.com",
             "CABXKi8zruXJc_6e4Dr087H5wE7sLp+u250o0N2q5DdjF_r-8wg@mail.gmail.com",
             include_bytes!("../test-data/message/gmail_ndn.eml"),
-            "Delivery Status Notification (Failure) – ** Die Adresse wurde nicht gefunden **\n\nIhre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.\n\nHier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser\n\nAntwort:\n\n550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient\'s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp",
+            Some("Delivery Status Notification (Failure) – ** Die Adresse wurde nicht gefunden **\n\nIhre Nachricht wurde nicht an assidhfaaspocwaeofi@gmail.com zugestellt, weil die Adresse nicht gefunden wurde oder keine E-Mails empfangen kann.\n\nHier erfahren Sie mehr: https://support.google.com/mail/?p=NoSuchUser\n\nAntwort:\n\n550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient\'s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser i18sor6261697wrs.38 - gsmtp"),
         )
         .await;
     }
@@ -2503,7 +2579,7 @@ mod tests {
             "snaerituhaeirns@gmail.com",
             "9c9c2a32-056b-3592-c372-d7e8f0bd4bc2@gmx.de",
             include_bytes!("../test-data/message/gmx_ndn.eml"),
-            "Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp [...]"
+            Some("Mail delivery failed: returning message to sender – This message was created automatically by mail delivery software.\n\nA message that you sent could not be delivered to one or more of\nits recipients. This is a permanent error. The following address(es)\nfailed:\n\nsnaerituhaeirns@gmail.com:\nSMTP error from remote server for RCPT TO command, host: gmail-smtp-in.l.google.com (66.102.1.27) reason: 550-5.1.1 The email account that you tried to reach does not exist. Please\n try\n550-5.1.1 double-checking the recipient\'s email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1  https://support.google.com/mail/?p=NoSuchUser f6si2517766wmc.21\n9 - gsmtp [...]"),
         )
         .await;
     }
@@ -2515,7 +2591,7 @@ mod tests {
             "hanerthaertidiuea@gmx.de",
             "04422840-f884-3e37-5778-8192fe22d8e1@posteo.de",
             include_bytes!("../test-data/message/posteo_ndn.eml"),
-            "Undelivered Mail Returned to Sender – This is the mail system at host mout01.posteo.de.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550\n    Requested action not taken: mailbox unavailable (in reply to RCPT TO\n    command)",
+            Some("Undelivered Mail Returned to Sender – This is the mail system at host mout01.posteo.de.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hanerthaertidiuea@gmx.de>: host mx01.emig.gmx.net[212.227.17.5] said: 550\n    Requested action not taken: mailbox unavailable (in reply to RCPT TO\n    command)"),
         )
         .await;
     }
@@ -2526,7 +2602,7 @@ mod tests {
         foreign_addr: &str,
         rfc724_mid_outgoing: &str,
         raw_ndn: &[u8],
-        error_msg: &str,
+        error_msg: Option<&str>,
     ) {
         let t = TestContext::new().await;
         t.configure_addr(self_addr).await;
@@ -2569,7 +2645,8 @@ mod tests {
         let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
 
         assert_eq!(msg.state, MessageState::OutFailed);
-        assert_eq!(msg.error, error_msg);
+
+        assert_eq!(msg.error(), error_msg.map(|error| error.to_string()));
     }
 
     #[async_std::test]

src/dc_tools.rs

@@ -15,9 +15,14 @@ use async_std::{fs, io};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
+use crate::chat::{add_device_msg, add_device_msg_with_importance};
+use crate::constants::{Viewtype, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::error::{bail, Error};
 use crate::events::EventType;
+use crate::message::Message;
+use crate::provider::get_provider_update_timestamp;
+use crate::stock::StockMessage;
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
@@ -151,6 +156,73 @@ pub(crate) async fn dc_create_smeared_timestamps(context: &Context, count: usize
     start
 }
 
+// if the system time is not plausible, once a day, add a device message.
+// for testing we're using time() as that is also used for message timestamps.
+// moreover, add a warning if the app is outdated.
+pub(crate) async fn maybe_add_time_based_warnings(context: &Context) {
+    if !maybe_warn_on_bad_time(context, time(), get_provider_update_timestamp()).await {
+        maybe_warn_on_outdated(context, time(), get_provider_update_timestamp()).await;
+    }
+}
+
+async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestamp: i64) -> bool {
+    if now < known_past_timestamp {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(
+            context
+                .stock_string_repl_str(
+                    StockMessage::BadTimeMsgBody,
+                    Local
+                        .timestamp(now, 0)
+                        .format("%Y-%m-%d %H:%M:%S")
+                        .to_string(),
+                )
+                .await,
+        );
+        add_device_msg_with_importance(
+            context,
+            Some(
+                format!(
+                    "bad-time-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m-%d") // repeat every day
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+            true,
+        )
+        .await
+        .ok();
+        return true;
+    }
+    false
+}
+
+async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time: i64) {
+    if now > approx_compile_time + DC_OUTDATED_WARNING_DAYS * 24 * 60 * 60 {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(
+            context
+                .stock_str(StockMessage::UpdateReminderMsgBody)
+                .await
+                .into(),
+        );
+        add_device_msg(
+            context,
+            Some(
+                format!(
+                    "outdated-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m") // repeat every month
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+        )
+        .await
+        .ok();
+    }
+}
+
 /* Message-ID tools */
 pub(crate) fn dc_create_id() -> String {
     /* generate an id. the generated ID should be as short and as unique as possible:
@@ -637,6 +709,18 @@ pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
         .to_string()
 }
 
+pub(crate) trait IsNoneOrEmpty<T> {
+    fn is_none_or_empty(&self) -> bool;
+}
+impl<T> IsNoneOrEmpty<T> for Option<T>
+where
+    T: AsRef<str>,
+{
+    fn is_none_or_empty(&self) -> bool {
+        !matches!(self, Some(s) if !s.as_ref().is_empty())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
@@ -800,6 +884,9 @@ mod tests {
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
+    use crate::chat;
+    use crate::chatlist::Chatlist;
+    use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
     use proptest::prelude::*;
 
     proptest! {
@@ -993,4 +1080,123 @@ mod tests {
         assert_eq!(improve_single_line_input("Hi\naiae "), "Hi aiae");
         assert_eq!(improve_single_line_input("\r\nahte\n\r"), "ahte");
     }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_bad_time() {
+        let t = TestContext::new().await;
+        let timestamp_now = time();
+        let timestamp_future = timestamp_now + 60 * 60 * 24 * 7;
+        let timestamp_past = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 9, 1),
+            NaiveTime::from_hms(0, 0, 0),
+        )
+        .timestamp_millis()
+            / 1_000;
+
+        // a correct time must not add a device message
+        maybe_warn_on_bad_time(&t.ctx, timestamp_now, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // we cannot find out if a date in the future is wrong - a device message is not added
+        maybe_warn_on_bad_time(&t.ctx, timestamp_future, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // a date in the past must add a device message
+        maybe_warn_on_bad_time(&t.ctx, timestamp_past, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // the message should be added only once a day - test that an hour later and nearly a day later
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60 * 24 - 1,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // next day, there should be another device message
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60 * 24,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        assert_eq!(device_chat_id, chats.get_chat_id(0));
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 2);
+    }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_outdated() {
+        let t = TestContext::new().await;
+        let timestamp_now: i64 = time();
+
+        // in about 6 months, the app should not be outdated
+        // (if this fails, provider-db is not updated since 6 months)
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + 180 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // in 1 year, the app should be considered as outdated
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + 365 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // do not repeat the warning every day ...
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + (365 + 1) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // ... but every month
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + (365 + 31) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 2);
+    }
 }

src/dehtml.rs

@@ -2,12 +2,10 @@
 //!
 //! A module to remove HTML tags from the email text
 
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
-lazy_static! {
-    static ref LINE_RE: regex::Regex = regex::Regex::new(r"(\r?\n)+").unwrap();
-}
+static LINE_RE: Lazy<regex::Regex> = Lazy::new(|| regex::Regex::new(r"(\r?\n)+").unwrap());
 
 struct Dehtml {
     strbuilder: String,
@@ -24,16 +22,16 @@ enum AddText {
 
 // dehtml() returns way too many newlines; however, an optimisation on this issue is not needed as
 // the newlines are typically removed in further processing by the caller
-pub fn dehtml(buf: &str) -> String {
+pub fn dehtml(buf: &str) -> Option<String> {
     let s = dehtml_quick_xml(buf);
     if !s.trim().is_empty() {
-        return s;
+        return Some(s);
     }
     let s = dehtml_manually(buf);
     if !s.trim().is_empty() {
-        return s;
+        return Some(s);
     }
-    buf.to_string()
+    None
 }
 
 pub fn dehtml_quick_xml(buf: &str) -> String {
@@ -222,21 +220,23 @@ mod tests {
                 "<a href='https://get.delta.chat/'/>",
                 "[](https://get.delta.chat/)",
             ),
-            ("", ""),
             ("<!doctype html>\n<b>fat text</b>", "*fat text*"),
             // Invalid html (at least DC should show the text if the html is invalid):
             ("<!some invalid html code>\n<b>some text</b>", "some text"),
-            ("<This text is in brackets>", "<This text is in brackets>"),
         ];
         for (input, output) in cases {
-            assert_eq!(simplify(dehtml(input), true).0, output);
+            assert_eq!(simplify(dehtml(input).unwrap(), true).0, output);
+        }
+        let none_cases = vec!["<html> </html>", ""];
+        for input in none_cases {
+            assert_eq!(dehtml(input), None);
         }
     }
 
     #[test]
     fn test_dehtml_parse_br() {
         let html = "\r\r\nline1<br>\r\n\r\n\r\rline2<br/>line3\n\r";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "line1\n\r\r\rline2\nline3");
     }
@@ -244,7 +244,7 @@ mod tests {
     #[test]
     fn test_dehtml_parse_href() {
         let html = "<a href=url>text</a";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "[text](url)");
     }
@@ -252,7 +252,7 @@ mod tests {
     #[test]
     fn test_dehtml_bold_text() {
         let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "text *bold*<>");
     }
@@ -262,7 +262,7 @@ mod tests {
         let html =
                 "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
 
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(
             plain,
@@ -285,7 +285,7 @@ mod tests {
         </body>
         </html>
         "##;
-        let txt = dehtml(input);
+        let txt = dehtml(input).unwrap();
         assert_eq!(txt.trim(), "lots of text");
     }
 }

src/e2ee.rs

@@ -51,23 +51,42 @@ impl EncryptHelper {
     }
 
     /// Determines if we can and should encrypt.
+    ///
+    /// For encryption to be enabled, `e2ee_guaranteed` should be true, or strictly more than a half
+    /// of peerstates should prefer encryption. Own preference is counted equally to peer
+    /// preferences, even if message copy is not sent to self.
+    ///
+    /// `e2ee_guaranteed` should be set to true for replies to encrypted messages (as required by
+    /// Autocrypt Level 1, version 1.1) and for messages sent in protected groups.
+    ///
+    /// Returns an error if `e2ee_guaranteed` is true, but one or more keys are missing.
     pub fn should_encrypt(
         &self,
         context: &Context,
         e2ee_guaranteed: bool,
         peerstates: &[(Option<Peerstate>, &str)],
     ) -> Result<bool> {
-        if !(self.prefer_encrypt == EncryptPreference::Mutual || e2ee_guaranteed) {
-            return Ok(false);
-        }
-
+        let mut prefer_encrypt_count = if self.prefer_encrypt == EncryptPreference::Mutual {
+            1
+        } else {
+            0
+        };
         for (peerstate, addr) in peerstates {
             match peerstate {
                 Some(peerstate) => {
-                    if peerstate.prefer_encrypt != EncryptPreference::Mutual && !e2ee_guaranteed {
-                        info!(context, "peerstate for {:?} is no-encrypt", addr);
-                        return Ok(false);
-                    }
+                    info!(
+                        context,
+                        "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
+                    );
+                    match peerstate.prefer_encrypt {
+                        EncryptPreference::NoPreference => {}
+                        EncryptPreference::Mutual => prefer_encrypt_count += 1,
+                        EncryptPreference::Reset => {
+                            if !e2ee_guaranteed {
+                                return Ok(false);
+                            }
+                        }
+                    };
                 }
                 None => {
                     let msg = format!("peerstate for {:?} missing, cannot encrypt", addr);
@@ -81,7 +100,11 @@ impl EncryptHelper {
             }
         }
 
-        Ok(true)
+        // Count number of recipients, including self.
+        // This does not depend on whether we send a copy to self or not.
+        let recipients_count = peerstates.len() + 1;
+
+        Ok(e2ee_guaranteed || 2 * prefer_encrypt_count > recipients_count)
     }
 
     /// Tries to encrypt the passed in `mail`.
@@ -212,9 +235,9 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
     }
 }
 
-async fn decrypt_if_autocrypt_message<'a>(
+async fn decrypt_if_autocrypt_message(
     context: &Context,
-    mail: &ParsedMail<'a>,
+    mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
     ret_valid_signatures: &mut HashSet<Fingerprint>,
@@ -488,4 +511,59 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
         Ok(())
     }
+
+    fn new_peerstates(
+        ctx: &Context,
+        prefer_encrypt: EncryptPreference,
+    ) -> Vec<(Option<Peerstate<'_>>, &str)> {
+        let addr = "bob@foo.bar";
+        let pub_key = bob_keypair().public;
+        let peerstate = Peerstate {
+            context: &ctx,
+            addr: addr.into(),
+            last_seen: 13,
+            last_seen_autocrypt: 14,
+            prefer_encrypt,
+            public_key: Some(pub_key.clone()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
+            gossip_key: Some(pub_key.clone()),
+            gossip_timestamp: 15,
+            gossip_key_fingerprint: Some(pub_key.fingerprint()),
+            verified_key: Some(pub_key.clone()),
+            verified_key_fingerprint: Some(pub_key.fingerprint()),
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        };
+        let mut peerstates = Vec::new();
+        peerstates.push((Some(peerstate), addr));
+        peerstates
+    }
+
+    #[async_std::test]
+    async fn test_should_encrypt() {
+        let t = TestContext::new_alice().await;
+        let encrypt_helper = EncryptHelper::new(&t.ctx).await.unwrap();
+
+        // test with EncryptPreference::NoPreference:
+        // if e2ee_eguaranteed is unset, there is no encryption as not more than half of peers want encryption
+        let ps = new_peerstates(&t.ctx, EncryptPreference::NoPreference);
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+
+        // test with EncryptPreference::Reset
+        let ps = new_peerstates(&t.ctx, EncryptPreference::Reset);
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+
+        // test with EncryptPreference::Mutual (self is also Mutual)
+        let ps = new_peerstates(&t.ctx, EncryptPreference::Mutual);
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+
+        // test with missing peerstate
+        let mut ps = Vec::new();
+        ps.push((None, "bob@foo.bar"));
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).is_err());
+        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+    }
 }

src/events.rs

@@ -58,12 +58,18 @@ impl EventEmitter {
 
     /// Async recv of an event. Return `None` if the `Sender` has been droped.
     pub async fn recv(&self) -> Option<Event> {
-        // TODO: change once we can use async channels internally.
         self.0.recv().await.ok()
     }
+}
 
-    pub fn try_recv(&self) -> Result<Event, async_std::sync::TryRecvError> {
-        self.0.try_recv()
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
     }
 }
 
@@ -186,6 +192,11 @@ pub enum EventType {
     #[strum(props(id = "2005"))]
     IncomingMsg { chat_id: ChatId, msg_id: MsgId },
 
+    /// Messages were seen or noticed.
+    /// chat id is always set.
+    #[strum(props(id = "2008"))]
+    MsgsNoticed(ChatId),
+
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
     #[strum(props(id = "2010"))]

src/format_flowed.rs

@@ -21,21 +21,28 @@
 /// length. However, this should be rare and should not result in
 /// immediate mail rejection: SMTP (RFC 2821) limit is 998 characters,
 /// and Spam Assassin limit is 78 characters.
-fn format_line_flowed(line: &str) -> String {
+fn format_line_flowed(line: &str, prefix: &str) -> String {
     let mut result = String::new();
-    let mut buffer = String::new();
+    let mut buffer = prefix.to_string();
     let mut after_space = false;
 
     for c in line.chars() {
         if c == ' ' {
             buffer.push(c);
             after_space = true;
+        } else if c == '>' {
+            if buffer.is_empty() {
+                // Space stuffing, see RFC 3676
+                buffer.push(' ');
+            }
+            buffer.push(c);
+            after_space = false;
         } else {
-            if after_space && buffer.len() >= 72 && !c.is_whitespace() && c != '>' {
+            if after_space && buffer.len() >= 72 && !c.is_whitespace() {
                 // Flush the buffer and insert soft break (SP CRLF).
                 result += &buffer;
                 result += "\r\n";
-                buffer = String::new();
+                buffer = prefix.to_string();
             }
             buffer.push(c);
             after_space = false;
@@ -44,6 +51,28 @@ fn format_line_flowed(line: &str) -> String {
     result + &buffer
 }
 
+fn format_flowed_prefix(text: &str, prefix: &str) -> String {
+    let mut result = String::new();
+
+    for line in text.split('\n') {
+        if !result.is_empty() {
+            result += "\r\n";
+        }
+        let line = line.trim_end();
+        if prefix.len() + line.len() > 78 {
+            result += &format_line_flowed(line, prefix);
+        } else {
+            result += prefix;
+            if prefix.is_empty() && line.starts_with('>') {
+                // Space stuffing, see RFC 3676
+                result.push(' ');
+            }
+            result += line;
+        }
+    }
+    result
+}
+
 /// Returns text formatted according to RFC 3767 (format=flowed).
 ///
 /// This function accepts text separated by LF, but returns text
@@ -52,20 +81,12 @@ fn format_line_flowed(line: &str) -> String {
 /// RFC 2646 technique is used to insert soft line breaks, so DelSp
 /// SHOULD be set to "no" when sending.
 pub fn format_flowed(text: &str) -> String {
-    let mut result = String::new();
+    format_flowed_prefix(text, "")
+}
 
-    for line in text.split('\n') {
-        if !result.is_empty() {
-            result += "\r\n";
-        }
-        let line = line.trim_end();
-        if line.len() > 78 {
-            result += &format_line_flowed(line);
-        } else {
-            result += line;
-        }
-    }
-    result
+/// Same as format_flowed(), but adds "> " prefix to each line.
+pub fn format_flowed_quote(text: &str) -> String {
+    format_flowed_prefix(text, "> ")
 }
 
 /// Joins lines in format=flowed text.
@@ -122,6 +143,9 @@ mod tests {
                         To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
                         client and enter the setup code presented on the generating device.";
         assert_eq!(format_flowed(text), expected);
+
+        let text = "> Not a quote";
+        assert_eq!(format_flowed(text), " > Not a quote");
     }
 
     #[test]
@@ -133,4 +157,21 @@ mod tests {
                         unwrapped on the receiver";
         assert_eq!(unformat_flowed(text, false), expected);
     }
+
+    #[test]
+    fn test_format_flowed_quote() {
+        let quote = "this is a quoted line";
+        let expected = "> this is a quoted line";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "> foo bar baz";
+        let expected = "> > foo bar baz";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "this is a very long quote that should be wrapped using format=flowed and unwrapped on the receiver";
+        let expected =
+            "> this is a very long quote that should be wrapped using format=flowed and \r\n\
+            > unwrapped on the receiver";
+        assert_eq!(format_flowed_quote(quote), expected);
+    }
 }

src/imap/idle.rs

@@ -113,12 +113,13 @@ impl Imap {
         // in this case, we're waiting for a configure job (and an interrupt).
 
         let fake_idle_start_time = SystemTime::now();
-        info!(context, "IMAP-fake-IDLEing...");
 
         // Do not poll, just wait for an interrupt when no folder is passed in.
         if watch_folder.is_none() {
+            info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
             return self.idle_interrupt.recv().await.unwrap_or_default();
         }
+        info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
         // check every minute if there are new messages
         // TODO: grow sleep durations / make them more flexible
@@ -160,7 +161,7 @@ impl Imap {
                     // will not find any new.
 
                     if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_new_messages(context, watch_folder).await {
+                        match self.fetch_new_messages(context, watch_folder, false).await {
                             Ok(res) => {
                                 info!(context, "fetch_new_messages returned {:?}", res);
                                 if res {

src/imap/mod.rs

@@ -3,8 +3,9 @@
 //! uses [async-email/async-imap](https://github.com/async-email/async-imap)
 //! to implement connect, fetch, delete functionality with standard IMAP servers.
 
-use std::collections::BTreeMap;
+use std::{cmp, collections::BTreeMap};
 
+use anyhow::Context as _;
 use async_imap::{
     error::Result as ImapResult,
     types::{Capability, Fetch, Flag, Mailbox, Name, NameAttribute},
@@ -13,12 +14,9 @@ use async_std::prelude::*;
 use async_std::sync::Receiver;
 use num_traits::FromPrimitive;
 
-use crate::config::*;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_receive_imf::{
-    dc_receive_imf, from_field_to_contact_id, is_msgrmsg_rfc724_mid_in_list,
-};
+use crate::dc_receive_imf::{from_field_to_contact_id, is_msgrmsg_rfc724_mid_in_list};
 use crate::error::{bail, format_err, Result};
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
@@ -32,6 +30,7 @@ use crate::provider::{get_provider_info, Socket};
 use crate::{
     chat, dc_tools::dc_extract_grpid_from_rfc724_mid, scheduler::InterruptInfo, stock::StockMessage,
 };
+use crate::{config::*, dc_receive_imf::dc_receive_imf_inner};
 
 mod client;
 mod idle;
@@ -40,6 +39,7 @@ mod session;
 
 use chat::get_chat_id_by_grpid;
 use client::Client;
+use mailparse::SingleInfo;
 use message::Message;
 use session::Session;
 
@@ -229,19 +229,7 @@ impl Imap {
                 }
             }
             Err(err) => {
-                let message = {
-                    let config = &self.config;
-                    let imap_server: &str = config.lp.server.as_ref();
-                    let imap_port = config.lp.port;
-                    context
-                        .stock_string_repl_str2(
-                            StockMessage::ServerResponse,
-                            format!("IMAP {}:{}", imap_server, imap_port),
-                            err.to_string(),
-                        )
-                        .await
-                };
-                bail!("{}: {}", message, err);
+                bail!(err);
             }
         };
 
@@ -286,7 +274,7 @@ impl Imap {
                 }
 
                 self.trigger_reconnect();
-                Err(format_err!("{}: {}", message, err))
+                Err(format_err!("{}\n\n{}", message, err))
             }
         }
     }
@@ -460,7 +448,10 @@ impl Imap {
         }
         self.setup_handle(context).await?;
 
-        while self.fetch_new_messages(context, &watch_folder).await? {
+        while self
+            .fetch_new_messages(context, &watch_folder, false)
+            .await?
+        {
             // We fetch until no more new messages are there.
         }
         Ok(())
@@ -476,16 +467,8 @@ impl Imap {
             // the entry has the format `imap.mailbox.<folder>=<uidvalidity>:<lastseenuid>`
             let mut parts = entry.split(':');
             (
-                parts
-                    .next()
-                    .unwrap_or_default()
-                    .parse()
-                    .unwrap_or_else(|_| 0),
-                parts
-                    .next()
-                    .unwrap_or_default()
-                    .parse()
-                    .unwrap_or_else(|_| 0),
+                parts.next().unwrap_or_default().parse().unwrap_or(0),
+                parts.next().unwrap_or_default().parse().unwrap_or(0),
             )
         } else {
             (0, 0)
@@ -663,10 +646,11 @@ impl Imap {
         Ok((new_uid_validity, new_last_seen_uid))
     }
 
-    async fn fetch_new_messages<S: AsRef<str>>(
+    pub(crate) async fn fetch_new_messages<S: AsRef<str>>(
         &mut self,
         context: &Context,
         folder: S,
+        fetch_existing_msgs: bool,
     ) -> Result<bool> {
         let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await)
             .unwrap_or_default();
@@ -675,7 +659,11 @@ impl Imap {
             .select_with_uidvalidity(context, folder.as_ref())
             .await?;
 
-        let msgs = self.fetch_after(context, last_seen_uid).await?;
+        let msgs = if fetch_existing_msgs {
+            self.fetch_existing_msgs_prefetch().await?
+        } else {
+            self.fetch_after(context, last_seen_uid).await?
+        };
         let read_cnt = msgs.len();
         let folder: &str = folder.as_ref();
 
@@ -715,8 +703,9 @@ impl Imap {
         }
 
         // check passed, go fetch the emails
-        let (new_last_seen_uid_processed, error_cnt) =
-            self.fetch_many_msgs(context, &folder, &uids).await;
+        let (new_last_seen_uid_processed, error_cnt) = self
+            .fetch_many_msgs(context, &folder, &uids, fetch_existing_msgs)
+            .await;
         read_errors += error_cnt;
 
         // determine which last_seen_uid to use to update  to
@@ -741,17 +730,66 @@ impl Imap {
         Ok(read_cnt > 0)
     }
 
+    /// Gets the from, to and bcc addresses from all existing outgoing emails.
+    pub async fn get_all_recipients(&mut self, context: &Context) -> Result<Vec<SingleInfo>> {
+        if self.session.is_none() {
+            bail!("IMAP No Connection established");
+        }
+
+        let session = self.session.as_mut().unwrap();
+        let self_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await
+            .ok_or_else(|| format_err!("Not configured"))?;
+
+        let search_command = format!("FROM \"{}\"", self_addr);
+        let uids = session.uid_search(search_command).await?;
+        let uid_strings: Vec<String> = uids.into_iter().map(|s| s.to_string()).collect();
+
+        let mut result = Vec::new();
+        // We fetch the emails in chunks of 100 because according to https://tools.ietf.org/html/rfc2683#section-3.2.1.5
+        // command lines should not be much more than 1000 chars and UIDs can get up to 9- or 10-digit
+        // (servers should allow at least 8000 chars)
+        for uid_chunk in uid_strings.chunks(100) {
+            let uid_set = uid_chunk.join(",");
+
+            let mut list = session
+                .uid_fetch(uid_set, "(UID BODY.PEEK[HEADER.FIELDS (FROM TO CC BCC)])")
+                .await
+                .map_err(|err| {
+                    format_err!("IMAP Could not fetch (get_all_recipients()): {}", err)
+                })?;
+
+            while let Some(fetch) = list.next().await {
+                let msg = fetch?;
+                match get_fetch_headers(&msg) {
+                    Ok(headers) => {
+                        let (from_id, _, _) =
+                            from_field_to_contact_id(context, &mimeparser::get_from(&headers))
+                                .await?;
+                        if from_id == DC_CONTACT_ID_SELF {
+                            result.extend(mimeparser::get_recipients(&headers));
+                        }
+                    }
+
+                    Err(err) => {
+                        warn!(context, "{}", err);
+                        continue;
+                    }
+                };
+            }
+        }
+        Ok(result)
+    }
+
     /// Fetch all uids larger than the passed in. Returns a sorted list of fetch results.
     async fn fetch_after(
         &mut self,
         context: &Context,
         uid: u32,
     ) -> Result<BTreeMap<u32, async_imap::types::Fetch>> {
-        if self.session.is_none() {
-            bail!("IMAP No Connection established");
-        }
-
-        let session = self.session.as_mut().unwrap();
+        let session = self.session.as_mut();
+        let session = session.context("fetch_after(): IMAP No Connection established")?;
 
         // fetch messages with larger UID than the last one seen
         // `(UID FETCH lastseenuid+1:*)`, see RFC 4549
@@ -789,6 +827,40 @@ impl Imap {
         Ok(new_msgs)
     }
 
+    /// Like fetch_after(), but not for new messages but existing ones (the DC_FETCH_EXISTING_MSGS_COUNT newest messages)
+    async fn fetch_existing_msgs_prefetch(
+        &mut self,
+    ) -> Result<BTreeMap<u32, async_imap::types::Fetch>> {
+        let exists: i64 = {
+            let mailbox = self.config.selected_mailbox.as_ref();
+            let mailbox = mailbox.context("fetch_existing_msgs_prefetch(): no mailbox selected")?;
+            mailbox.exists.into()
+        };
+        let session = self.session.as_mut();
+        let session =
+            session.context("fetch_existing_msgs_prefetch(): IMAP No Connection established")?;
+
+        // Fetch last DC_FETCH_EXISTING_MSGS_COUNT (100) messages.
+        // Sequence numbers are sequential. If there are 1000 messages in the inbox,
+        // we can fetch the sequence numbers 900-1000 and get the last 100 messages.
+        let first = cmp::max(1, exists - DC_FETCH_EXISTING_MSGS_COUNT);
+        let set = format!("{}:*", first);
+        let mut list = session
+            .fetch(&set, PREFETCH_FLAGS)
+            .await
+            .map_err(|err| format_err!("IMAP Could not fetch: {}", err))?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(fetch) = list.next().await {
+            let msg = fetch?;
+            if let Some(msg_uid) = msg.uid {
+                msgs.insert(msg_uid, msg);
+            }
+        }
+
+        Ok(msgs)
+    }
+
     async fn set_config_last_seen_uid<S: AsRef<str>>(
         &self,
         context: &Context,
@@ -815,6 +887,7 @@ impl Imap {
         context: &Context,
         folder: S,
         server_uids: &[u32],
+        fetching_existing_messages: bool,
     ) -> (Option<u32>, usize) {
         let set = match server_uids {
             [] => return (None, 0),
@@ -888,7 +961,16 @@ impl Imap {
             let body = msg.body().unwrap();
             let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
-            match dc_receive_imf(&context, &body, &folder, server_uid, is_seen).await {
+            match dc_receive_imf_inner(
+                &context,
+                &body,
+                &folder,
+                server_uid,
+                is_seen,
+                fetching_existing_messages,
+            )
+            .await
+            {
                 Ok(_) => last_uid = Some(server_uid),
                 Err(err) => {
                     warn!(context, "dc_receive_imf error: {}", err);
@@ -1266,7 +1348,9 @@ impl Imap {
                 } else if let FolderMeaning::SentObjects = get_folder_meaning(&folder) {
                     // Always takes precedent
                     sentbox_folder = Some(folder.name().to_string());
-                } else if let FolderMeaning::SentObjects = get_folder_meaning_by_name(&folder) {
+                } else if let FolderMeaning::SentObjects =
+                    get_folder_meaning_by_name(&folder.name())
+                {
                     // only set iff none has been already set
                     if sentbox_folder.is_none() {
                         sentbox_folder = Some(folder.name().to_string());
@@ -1345,11 +1429,43 @@ impl Imap {
 // only watching this folder is not working. at least, this is no show stopper.
 // CAVE: if possible, take care not to add a name here that is "sent" in one language
 // but sth. different in others - a hard job.
-fn get_folder_meaning_by_name(folder_name: &Name) -> FolderMeaning {
-    let sent_names = vec!["sent", "sentmail", "sent objects", "gesendet"];
-    let lower = folder_name.name().to_lowercase();
+fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
+    // source: https://stackoverflow.com/questions/2185391/localized-gmail-imap-folders
+    let sent_names = vec![
+        "sent",
+        "sentmail",
+        "sent objects",
+        "gesendet",
+        "Sent Mail",
+        "Sendte e-mails",
+        "Enviados",
+        "Messages envoyés",
+        "Messages envoyes",
+        "Posta inviata",
+        "Verzonden berichten",
+        "Wyslane",
+        "E-mails enviados",
+        "Correio enviado",
+        "Enviada",
+        "Enviado",
+        "Gönderildi",
+        "Inviati",
+        "Odeslaná pošta",
+        "Sendt",
+        "Skickat",
+        "Verzonden",
+        "Wysłane",
+        "Éléments envoyés",
+        "Απεσταλμένα",
+        "Отправленные",
+        "寄件備份",
+        "已发送邮件",
+        "送信済み",
+        "보낸편지함",
+    ];
+    let lower = folder_name.to_lowercase();
 
-    if sent_names.into_iter().any(|s| s == lower) {
+    if sent_names.into_iter().any(|s| s.to_lowercase() == lower) {
         FolderMeaning::SentObjects
     } else {
         FolderMeaning::Unknown
@@ -1443,17 +1559,16 @@ async fn precheck_imf(
 
         if old_server_folder != server_folder || old_server_uid != server_uid {
             update_server_uid(context, rfc724_mid, server_folder, server_uid).await;
-            if let Ok(MessageState::InSeen) = msg_id.get_state(context).await {
-                job::add(
-                    context,
-                    job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
-                )
-                .await;
-            };
-            context
-                .interrupt_inbox(InterruptInfo::new(false, Some(msg_id)))
-                .await;
-            info!(context, "Updating server_uid and interrupting")
+            if let Ok(message_state) = msg_id.get_state(context).await {
+                if message_state == MessageState::InSeen || message_state.is_outgoing() {
+                    job::add(
+                        context,
+                        job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+                    )
+                    .await;
+                }
+            }
+            info!(context, "Updating server_uid and adding markseen job");
         }
         Ok(true)
     } else {
@@ -1595,3 +1710,32 @@ async fn message_needs_processing(
 fn get_fallback_folder(delimiter: &str) -> String {
     format!("INBOX{}DeltaChat", delimiter)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_get_folder_meaning_by_name() {
+        assert_eq!(
+            get_folder_meaning_by_name("Gesendet"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(
+            get_folder_meaning_by_name("GESENDET"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(
+            get_folder_meaning_by_name("gesendet"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(
+            get_folder_meaning_by_name("Messages envoyés"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(
+            get_folder_meaning_by_name("mEsSaGes envoyÉs"),
+            FolderMeaning::SentObjects
+        );
+        assert_eq!(get_folder_meaning_by_name("xxx"), FolderMeaning::Unknown);
+    }
+}

src/imex.rs

@@ -6,6 +6,7 @@ use std::{
     ffi::OsStr,
 };
 
+use anyhow::Context as _;
 use async_std::path::{Path, PathBuf};
 use async_std::{
     fs::{self, File},
@@ -94,7 +95,8 @@ pub async fn imex(
             }
             Err(err) => {
                 cleanup_aborted_imex(context, what).await;
-                error!(context, "{}", err);
+                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                error!(context, "{:#}", err);
                 context.emit_event(EventType::ImexProgress(0));
                 bail!("IMEX FAILED to complete: {}", err);
             }
@@ -118,7 +120,9 @@ async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
         dc_delete_files_in_dir(context, context.get_blobdir()).await;
     }
     if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
-        context.sql.open(context, context.get_dbfile(), false).await;
+        if let Err(e) = context.sql.open(context, context.get_dbfile(), false).await {
+            warn!(context, "Re-opening db after imex failed: {}", e);
+        }
     }
 }
 
@@ -166,17 +170,23 @@ pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Re
             let name = name.to_string_lossy();
             if name.starts_with("delta-chat") && name.ends_with(".bak") {
                 let sql = Sql::new();
-                if sql.open(context, &path, true).await {
-                    let curr_backup_time = sql
-                        .get_raw_config_int(context, "backup_time")
-                        .await
-                        .unwrap_or_default();
-                    if curr_backup_time > newest_backup_time {
-                        newest_backup_path = Some(path);
-                        newest_backup_time = curr_backup_time;
+                match sql.open(context, &path, true).await {
+                    Ok(_) => {
+                        let curr_backup_time = sql
+                            .get_raw_config_int(context, "backup_time")
+                            .await
+                            .unwrap_or_default();
+                        if curr_backup_time > newest_backup_time {
+                            newest_backup_path = Some(path);
+                            newest_backup_time = curr_backup_time;
+                        }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close().await;
                     }
-                    info!(context, "backup_time of {} is {}", name, curr_backup_time);
-                    sql.close().await;
+                    Err(e) => warn!(
+                        context,
+                        "Found backup file {} which could not be opened: {}", name, e
+                    ),
                 }
             }
         }
@@ -520,13 +530,11 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         }
     }
 
-    ensure!(
-        context
-            .sql
-            .open(&context, &context.get_dbfile(), false)
-            .await,
-        "could not re-open db"
-    );
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
 
     delete_and_reset_all_device_msgs(&context).await?;
 
@@ -558,13 +566,11 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
     );
     /* error already logged */
     /* re-open copied database file */
-    ensure!(
-        context
-            .sql
-            .open(&context, &context.get_dbfile(), false)
-            .await,
-        "could not re-open db"
-    );
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
 
     delete_and_reset_all_device_msgs(&context).await?;
 
@@ -743,7 +749,7 @@ async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<(
     context
         .sql
         .open(&context, &context.get_dbfile(), false)
-        .await;
+        .await?;
 
     if !copied {
         bail!(
@@ -753,11 +759,11 @@ async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<(
         );
     }
     let dest_sql = Sql::new();
-    ensure!(
-        dest_sql.open(context, &dest_path_filename, false).await,
-        "could not open exported database {}",
-        dest_path_string
-    );
+    dest_sql
+        .open(context, &dest_path_filename, false)
+        .await
+        .with_context(|| format!("could not open exported database {}", dest_path_string))?;
+
     let res = match add_files_to_export(context, &dest_sql).await {
         Err(err) => {
             dc_delete_file(context, &dest_path_filename).await;
@@ -1048,6 +1054,21 @@ mod tests {
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
+    #[async_std::test]
+    async fn test_export_and_import_key() {
+        let context = TestContext::new().await;
+        context.configure_alice().await;
+        let blobdir = "$BLOBDIR";
+        assert!(imex(&context.ctx, ImexMode::ExportSelfKeys, Some(blobdir))
+            .await
+            .is_ok());
+
+        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        assert!(imex(&context.ctx, ImexMode::ImportSelfKeys, Some(blobdir))
+            .await
+            .is_ok());
+    }
+
     #[test]
     fn test_normalize_setup_code() {
         let norm = normalize_setup_code("123422343234423452346234723482349234");

src/job.rs

@@ -14,7 +14,6 @@ use async_smtp::smtp::response::Category;
 use async_smtp::smtp::response::Code;
 use async_smtp::smtp::response::Detail;
 
-use crate::blob::BlobObject;
 use crate::chat::{self, ChatId};
 use crate::config::Config;
 use crate::contact::Contact;
@@ -30,6 +29,7 @@ use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
+use crate::{blob::BlobObject, contact::normalize_name, contact::Modifier, contact::Origin};
 use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
@@ -92,6 +92,7 @@ pub enum Action {
 
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
+    FetchExistingMsgs = 110,
     MarkseenMsgOnImap = 130,
 
     // Moving message is prioritized lower than deletion so we don't
@@ -124,6 +125,7 @@ impl From<Action> for Thread {
             Unknown => Thread::Unknown,
 
             Housekeeping => Thread::Imap,
+            FetchExistingMsgs => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
             ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
@@ -619,6 +621,43 @@ impl Job {
         }
     }
 
+    /// Read the recipients from old emails sent by the user and add them as contacts.
+    /// This way, we can already offer them some email addresses they can write to.
+    ///
+    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
+    /// and show them in the chat list.
+    async fn fetch_existing_msgs(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if context.get_config_bool(Config::Bot).await {
+            return Status::Finished(Ok(())); // Bots don't want those messages
+        }
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredSentboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredMvboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredInboxFolder).await;
+
+        if context.get_config_bool(Config::FetchExisting).await {
+            for config in &[
+                Config::ConfiguredMvboxFolder,
+                Config::ConfiguredInboxFolder,
+                Config::ConfiguredSentboxFolder,
+            ] {
+                if let Some(folder) = context.get_config(*config).await {
+                    if let Err(e) = imap.fetch_new_messages(context, folder, true).await {
+                        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                        warn!(context, "Could not fetch messages, retrying: {:#}", e);
+                        return Status::RetryLater;
+                    };
+                }
+            }
+        }
+        info!(context, "Done fetching existing messages.");
+        Status::Finished(Ok(()))
+    }
+
     /// Synchronizes UIDs for sentbox, inbox and mvbox, in this order.
     ///
     /// If a copy of the message is present in multiple folders, mvbox
@@ -759,6 +798,50 @@ async fn set_delivered(context: &Context, msg_id: MsgId) {
     context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
 }
 
+async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
+    let mailbox = if let Some(m) = context.get_config(folder).await {
+        m
+    } else {
+        return;
+    };
+    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
+        warn!(context, "Could not select {}: {}", mailbox, e);
+        return;
+    }
+    match imap.get_all_recipients(context).await {
+        Ok(contacts) => {
+            let mut any_modified = false;
+            for contact in contacts {
+                let display_name_normalized = contact
+                    .display_name
+                    .as_ref()
+                    .map(normalize_name)
+                    .unwrap_or_default();
+
+                match Contact::add_or_lookup(
+                    context,
+                    display_name_normalized,
+                    contact.addr,
+                    Origin::OutgoingTo,
+                )
+                .await
+                {
+                    Ok((_, modified)) => {
+                        if modified != Modifier::None {
+                            any_modified = true;
+                        }
+                    }
+                    Err(e) => warn!(context, "Could not add recipient: {}", e),
+                }
+            }
+            if any_modified {
+                context.emit_event(EventType::ContactsChanged(None));
+            }
+        }
+        Err(e) => warn!(context, "Could not add recipients: {}", e),
+    };
+}
+
 /// Constructs a job for sending a message.
 ///
 /// Returns `None` if no messages need to be sent out.
@@ -1007,6 +1090,7 @@ async fn perform_job_action(
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
+        Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::Housekeeping => {
             sql::housekeeping(context).await;
             Status::Finished(Ok(()))
@@ -1072,6 +1156,7 @@ pub async fn add(context: &Context, job: Job) {
             | Action::DeleteMsgOnImap
             | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
+            | Action::FetchExistingMsgs
             | Action::MoveMsg => {
                 info!(context, "interrupt: imap");
                 context

src/key.rs

@@ -222,7 +222,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     let addr = context
         .get_config(Config::ConfiguredAddr)
         .await
-        .ok_or_else(|| Error::NoConfiguredAddr)?;
+        .ok_or(Error::NoConfiguredAddr)?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -431,11 +431,9 @@ mod tests {
     use std::error::Error;
 
     use async_std::sync::Arc;
-    use lazy_static::lazy_static;
+    use once_cell::sync::Lazy;
 
-    lazy_static! {
-        static ref KEYPAIR: KeyPair = alice_keypair();
-    }
+    static KEYPAIR: Lazy<KeyPair> = Lazy::new(alice_keypair);
 
     #[test]
     fn test_from_armored_string() {

src/message.rs

@@ -2,7 +2,7 @@
 
 use async_std::path::{Path, PathBuf};
 use deltachat_derive::{FromSql, ToSql};
-use lazy_static::lazy_static;
+use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
@@ -19,10 +19,7 @@ use crate::mimeparser::{FailureReport, SystemMessage};
 use crate::param::*;
 use crate::pgp::*;
 use crate::stock::StockMessage;
-
-lazy_static! {
-    static ref UNWRAP_RE: regex::Regex = regex::Regex::new(r"\s+").unwrap();
-}
+use std::collections::BTreeMap;
 
 // In practice, the user additionally cuts the string themselves
 // pixel-accurate.
@@ -266,10 +263,9 @@ pub struct Message {
     pub(crate) server_folder: Option<String>,
     pub(crate) server_uid: u32,
     pub(crate) is_dc_message: MessengerMessage,
-    pub(crate) starred: bool,
     pub(crate) chat_blocked: Blocked,
     pub(crate) location_id: u32,
-    pub(crate) error: String,
+    error: Option<String>,
     pub(crate) param: Params,
 }
 
@@ -310,7 +306,6 @@ impl Message {
                     "    m.msgrmsg AS msgrmsg,",
                     "    m.txt AS txt,",
                     "    m.param AS param,",
-                    "    m.starred AS starred,",
                     "    m.hidden AS hidden,",
                     "    m.location_id AS location,",
                     "    c.blocked AS blocked",
@@ -336,7 +331,8 @@ impl Message {
                     msg.ephemeral_timestamp = row.get("ephemeral_timestamp")?;
                     msg.viewtype = row.get("type")?;
                     msg.state = row.get("state")?;
-                    msg.error = row.get("error")?;
+                    let error: String = row.get("error")?;
+                    msg.error = Some(error).filter(|error| !error.is_empty());
                     msg.is_dc_message = row.get("msgrmsg")?;
 
                     let text;
@@ -360,7 +356,6 @@ impl Message {
                     msg.text = Some(text);
 
                     msg.param = row.get::<_, String>("param")?.parse().unwrap_or_default();
-                    msg.starred = row.get("starred")?;
                     msg.hidden = row.get("hidden")?;
                     msg.location_id = row.get("location")?;
                     msg.chat_blocked = row
@@ -549,9 +544,7 @@ impl Message {
             return ret;
         };
 
-        let contact = if self.from_id != DC_CONTACT_ID_SELF as u32
-            && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
-        {
+        let contact = if self.from_id != DC_CONTACT_ID_SELF as u32 && chat.typ == Chattype::Group {
             Contact::get_by_id(context, self.from_id).await.ok()
         } else {
             None
@@ -585,10 +578,6 @@ impl Message {
         self.state as i32 >= MessageState::OutDelivered as i32
     }
 
-    pub fn is_starred(&self) -> bool {
-        self.starred
-    }
-
     pub fn is_forwarded(&self) -> bool {
         0 != self.param.get_int(Param::Forwarded).unwrap_or_default()
     }
@@ -600,6 +589,10 @@ impl Message {
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
+    pub fn get_info_type(&self) -> SystemMessage {
+        self.param.get_cmd()
+    }
+
     pub fn is_system_message(&self) -> bool {
         let cmd = self.param.get_cmd();
         cmd != SystemMessage::Unknown
@@ -750,6 +743,61 @@ impl Message {
         self.update_param(context).await;
     }
 
+    /// Sets message quote.
+    ///
+    /// Message-Id is used to set Reply-To field, message text is used for quote.
+    ///
+    /// Encryption is required if quoted message was encrypted.
+    ///
+    /// The message itself is not required to exist in the database,
+    /// it may even be deleted from the database by the time the message is prepared.
+    pub async fn set_quote(&mut self, context: &Context, quote: &Message) -> Result<(), Error> {
+        ensure!(
+            !quote.rfc724_mid.is_empty(),
+            "Message without Message-Id cannot be quoted"
+        );
+        self.in_reply_to = Some(quote.rfc724_mid.clone());
+
+        if quote
+            .param
+            .get_bool(Param::GuaranteeE2ee)
+            .unwrap_or_default()
+        {
+            self.param.set(Param::GuaranteeE2ee, "1");
+        }
+
+        let text = quote.get_text().unwrap_or_default();
+        self.param.set(
+            Param::Quote,
+            if text.is_empty() {
+                // Use summary, similar to "Image" to avoid sending empty quote.
+                quote.get_summarytext(context, 500).await
+            } else {
+                text
+            },
+        );
+
+        Ok(())
+    }
+
+    pub fn quoted_text(&self) -> Option<String> {
+        self.param.get(Param::Quote).map(|s| s.to_string())
+    }
+
+    pub async fn quoted_message(&self, context: &Context) -> Result<Option<Message>, Error> {
+        if self.param.get(Param::Quote).is_some() {
+            if let Some(in_reply_to) = &self.in_reply_to {
+                let rfc724_mid = in_reply_to.trim_start_matches('<').trim_end_matches('>');
+                if !rfc724_mid.is_empty() {
+                    if let Some((_, _, msg_id)) = rfc724_mid_exists(context, rfc724_mid).await? {
+                        return Ok(Some(Message::load_from_db(context, msg_id).await?));
+                    }
+                }
+            }
+        }
+        Ok(None)
+    }
+
     pub async fn update_param(&mut self, context: &Context) -> bool {
         context
             .sql
@@ -760,6 +808,22 @@ impl Message {
             .await
             .is_ok()
     }
+
+    /// Gets the error status of the message.
+    ///
+    /// A message can have an associated error status if something went wrong when sending or
+    /// receiving message itself.  The error status is free-form text and should not be further parsed,
+    /// rather it's presence is meant to indicate *something* went wrong with the message and the
+    /// text of the error is detailed information on what.
+    ///
+    /// Some common reasons error can be associated with messages are:
+    /// * Lack of valid signature on an e2ee message, usually for received messages.
+    /// * Failure to decrypt an e2ee message, usually for received messages.
+    /// * When a message could not be delivered to one or more recipients the non-delivery
+    ///    notification text can be stored in the error status.
+    pub fn error(&self) -> Option<String> {
+        self.error.clone()
+    }
 }
 
 #[derive(
@@ -866,13 +930,18 @@ impl From<MessageState> for LotState {
 
 impl MessageState {
     pub fn can_fail(self) -> bool {
-        match self {
-            MessageState::OutPreparing
-            | MessageState::OutPending
-            | MessageState::OutDelivered
-            | MessageState::OutMdnRcvd => true, // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
-            _ => false,
-        }
+        use MessageState::*;
+        matches!(
+            self,
+            OutPreparing | OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
+        )
+    }
+    pub fn is_outgoing(self) -> bool {
+        use MessageState::*;
+        matches!(
+            self,
+            OutPreparing | OutDraft | OutPending | OutFailed | OutDelivered | OutMdnRcvd
+        )
     }
 }
 
@@ -909,7 +978,7 @@ impl Lot {
                 );
                 self.text1_meaning = Meaning::Text1Self;
             }
-        } else if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
+        } else if chat.typ == Chattype::Group {
             if msg.is_info() || contact.is_none() {
                 self.text1 = None;
                 self.text1_meaning = Meaning::None;
@@ -929,16 +998,23 @@ impl Lot {
             }
         }
 
-        self.text2 = Some(
-            get_summarytext_by_raw(
-                msg.viewtype,
-                msg.text.as_ref(),
-                &msg.param,
-                SUMMARY_CHARACTERS,
-                context,
-            )
-            .await,
-        );
+        let mut text2 = get_summarytext_by_raw(
+            msg.viewtype,
+            msg.text.as_ref(),
+            &msg.param,
+            SUMMARY_CHARACTERS,
+            context,
+        )
+        .await;
+
+        if text2.is_empty() && msg.quoted_text().is_some() {
+            text2 = context
+                .stock_str(StockMessage::ReplyNoun)
+                .await
+                .into_owned()
+        }
+
+        self.text2 = Some(text2);
 
         self.timestamp = msg.get_timestamp();
         self.state = msg.state.into();
@@ -1054,8 +1130,8 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
 
     ret += "\n";
 
-    if !msg.error.is_empty() {
-        ret += &format!("Error: {}", msg.error);
+    if let Some(error) = msg.error.as_ref() {
+        ret += &format!("Error: {}", error);
     }
 
     if let Some(path) = msg.get_file(context) {
@@ -1120,6 +1196,7 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
         "jpg" => (Viewtype::Image, "image/jpeg"),
         "json" => (Viewtype::File, "application/json"),
         "mov" => (Viewtype::Video, "video/quicktime"),
+        "m4a" => (Viewtype::Audio, "audio/m4a"),
         "mp3" => (Viewtype::Audio, "audio/mpeg"),
         "mp4" => (Viewtype::Video, "video/mp4"),
         "odp" => (
@@ -1230,6 +1307,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
         .with_conn(move |conn| {
             let mut stmt = conn.prepare_cached(concat!(
                 "SELECT",
+                "    m.chat_id AS chat_id,",
                 "    m.state AS state,",
                 "    c.blocked AS blocked",
                 " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
@@ -1240,6 +1318,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
             for id in msg_ids.into_iter() {
                 let query_res = stmt.query_row(paramsv![id], |row| {
                     Ok((
+                        row.get::<_, ChatId>("chat_id")?,
                         row.get::<_, MessageState>("state")?,
                         row.get::<_, Option<Blocked>>("blocked")?
                             .unwrap_or_default(),
@@ -1248,8 +1327,8 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
                 if let Err(rusqlite::Error::QueryReturnedNoRows) = query_res {
                     continue;
                 }
-                let (state, blocked) = query_res.map_err(Into::<anyhow::Error>::into)?;
-                msgs.push((id, state, blocked));
+                let (chat_id, state, blocked) = query_res.map_err(Into::<anyhow::Error>::into)?;
+                msgs.push((id, chat_id, state, blocked));
             }
 
             Ok(msgs)
@@ -1257,9 +1336,9 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
         .await
         .unwrap_or_default();
 
-    let mut send_event = false;
+    let mut updated_chat_ids = BTreeMap::new();
 
-    for (id, curr_state, curr_blocked) in msgs.into_iter() {
+    for (id, curr_chat_id, curr_state, curr_blocked) in msgs.into_iter() {
         if let Err(err) = id.start_ephemeral_timer(context).await {
             error!(
                 context,
@@ -1278,19 +1357,16 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
                     job::Job::new(Action::MarkseenMsgOnImap, id.to_u32(), Params::new(), 0),
                 )
                 .await;
-                send_event = true;
+                updated_chat_ids.insert(curr_chat_id, true);
             }
         } else if curr_state == MessageState::InFresh {
             update_msg_state(context, id, MessageState::InNoticed).await;
-            send_event = true;
+            updated_chat_ids.insert(ChatId::new(DC_CHAT_ID_DEADDROP), true);
         }
     }
 
-    if send_event {
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
+    for updated_chat_id in updated_chat_ids.keys() {
+        context.emit_event(EventType::MsgsNoticed(*updated_chat_id));
     }
 
     true
@@ -1307,23 +1383,6 @@ pub async fn update_msg_state(context: &Context, msg_id: MsgId, state: MessageSt
         .is_ok()
 }
 
-pub async fn star_msgs(context: &Context, msg_ids: Vec<MsgId>, star: bool) -> bool {
-    if msg_ids.is_empty() {
-        return false;
-    }
-    context
-        .sql
-        .with_conn(move |conn| {
-            let mut stmt = conn.prepare("UPDATE msgs SET starred=? WHERE id=?;")?;
-            for msg_id in msg_ids.into_iter() {
-                stmt.execute(paramsv![star as i32, msg_id])?;
-            }
-            Ok(())
-        })
-        .await
-        .is_ok()
-}
-
 /// Returns a summary text.
 pub async fn get_summarytext_by_raw(
     viewtype: Viewtype,
@@ -1402,7 +1461,7 @@ pub async fn get_summarytext_by_raw(
         prefix
     };
 
-    UNWRAP_RE.replace_all(&summary, " ").to_string()
+    summary.split_whitespace().join(" ")
 }
 
 // as we do not cut inside words, this results in about 32-42 characters.
@@ -1579,14 +1638,16 @@ pub(crate) async fn handle_ndn(
     context: &Context,
     failed: &FailureReport,
     error: Option<impl AsRef<str>>,
-) {
+) -> anyhow::Result<()> {
     if failed.rfc724_mid.is_empty() {
-        return;
+        return Ok(());
     }
 
-    let res = context
+    // The NDN might be for a message-id that had attachments and was sent from a non-Delta Chat client.
+    // In this case we need to mark multiple "msgids" as failed that all refer to the same message-id.
+    let msgs: Vec<_> = context
         .sql
-        .query_row(
+        .query_map(
             concat!(
                 "SELECT",
                 "    m.id AS msg_id,",
@@ -1603,37 +1664,42 @@ pub(crate) async fn handle_ndn(
                     row.get::<_, Chattype>("type")?,
                 ))
             },
+            |rows| Ok(rows.collect::<Vec<_>>()),
         )
-        .await;
-    if let Err(ref err) = res {
-        info!(context, "Failed to select NDN {:?}", err);
-    }
+        .await?;
 
-    if let Ok((msg_id, chat_id, chat_type)) = res {
-        set_msg_failed(context, msg_id, error).await;
-
-        if chat_type == Chattype::Group || chat_type == Chattype::VerifiedGroup {
-            if let Some(failed_recipient) = &failed.failed_recipient {
-                let contact_id =
-                    Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
-                if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-                    // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
-                    chat::add_info_msg(
-                        context,
-                        chat_id,
-                        context
-                            .stock_string_repl_str(
-                                StockMessage::FailedSendingTo,
-                                contact.get_display_name(),
-                            )
-                            .await,
-                    )
-                    .await;
-                    context.emit_event(EventType::ChatModified(chat_id));
-                }
-            }
+    for (i, msg) in msgs.into_iter().enumerate() {
+        let (msg_id, chat_id, chat_type) = msg?;
+        set_msg_failed(context, msg_id, error.as_ref()).await;
+        if i == 0 {
+            // Add only one info msg for all failed messages
+            ndn_maybe_add_info_msg(context, failed, chat_id, chat_type).await?;
         }
     }
+
+    Ok(())
+}
+
+async fn ndn_maybe_add_info_msg(
+    context: &Context,
+    failed: &FailureReport,
+    chat_id: ChatId,
+    chat_type: Chattype,
+) -> anyhow::Result<()> {
+    if chat_type == Chattype::Group {
+        if let Some(failed_recipient) = &failed.failed_recipient {
+            let contact_id =
+                Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown).await;
+            let contact = Contact::load_from_db(context, contact_id).await?;
+            // Tell the user which of the recipients failed if we know that (because in a group, this might otherwise be unclear)
+            let text = context
+                .stock_string_repl_str(StockMessage::FailedSendingTo, contact.get_display_name())
+                .await;
+            chat::add_info_msg(context, chat_id, text).await;
+            context.emit_event(EventType::ChatModified(chat_id));
+        }
+    }
+    Ok(())
 }
 
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)
@@ -1837,12 +1903,29 @@ mod tests {
         assert_eq!(_msg2.get_filemime(), None);
     }
 
+    /// Tests that message cannot be prepared if account has no configured address.
+    #[async_std::test]
+    async fn test_prepare_not_configured() {
+        let d = test::TestContext::new().await;
+        let ctx = &d.ctx;
+
+        let contact = Contact::create(ctx, "", "dest@example.com")
+            .await
+            .expect("failed to create contact");
+
+        let chat = chat::create_by_contact_id(ctx, contact).await.unwrap();
+
+        let mut msg = Message::new(Viewtype::Text);
+
+        assert!(chat::prepare_msg(ctx, chat, &mut msg).await.is_err());
+    }
+
     #[async_std::test]
     async fn test_get_summarytext_by_raw() {
         let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
-        let some_text = Some("bla bla".to_string());
+        let some_text = Some(" bla \t\n\tbla\n\t".to_string());
         let empty_text = Some("".to_string());
         let no_text: Option<String> = None;
 
@@ -2008,4 +2091,43 @@ mod tests {
         }
         assert!(has_image);
     }
+
+    #[async_std::test]
+    async fn test_quote() {
+        use crate::config::Config;
+
+        let d = test::TestContext::new().await;
+        let ctx = &d.ctx;
+
+        let contact = Contact::create(ctx, "", "dest@example.com")
+            .await
+            .expect("failed to create contact");
+
+        let res = ctx
+            .set_config(Config::ConfiguredAddr, Some("self@example.com"))
+            .await;
+        assert!(res.is_ok());
+
+        let chat = chat::create_by_contact_id(ctx, contact).await.unwrap();
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("Quoted message".to_string()));
+
+        // Prepare message for sending, so it gets a Message-Id.
+        assert!(msg.rfc724_mid.is_empty());
+        let msg_id = chat::prepare_msg(ctx, chat, &mut msg).await.unwrap();
+        let msg = Message::load_from_db(ctx, msg_id).await.unwrap();
+        assert!(!msg.rfc724_mid.is_empty());
+
+        let mut msg2 = Message::new(Viewtype::Text);
+        msg2.set_quote(ctx, &msg).await.expect("can't set quote");
+        assert!(msg2.quoted_text() == msg.get_text());
+
+        let quoted_msg = msg2
+            .quoted_message(ctx)
+            .await
+            .expect("error while retrieving quoted message")
+            .expect("quoted message not found");
+        assert!(quoted_msg.get_text() == msg2.quoted_text());
+    }
 }

src/mimefactory.rs

@@ -11,7 +11,7 @@ use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
-use crate::format_flowed::format_flowed;
+use crate::format_flowed::{format_flowed, format_flowed_quote};
 use crate::location;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
@@ -145,7 +145,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             selfstatus: context
                 .get_config(Config::Selfstatus)
                 .await
-                .unwrap_or_else(|| default_str),
+                .unwrap_or(default_str),
             recipients,
             timestamp: msg.timestamp_sort,
             loaded: Loaded::Message { chat },
@@ -183,7 +183,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let selfstatus = context
             .get_config(Config::Selfstatus)
             .await
-            .unwrap_or_else(|| default_str);
+            .unwrap_or(default_str);
         let timestamp = dc_create_smeared_timestamp(context).await;
 
         let res = MimeFactory::<'a, 'b> {
@@ -233,7 +233,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     fn is_e2ee_guaranteed(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.typ == Chattype::VerifiedGroup {
+                if chat.is_protected() {
                     return true;
                 }
 
@@ -255,7 +255,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     fn min_verified(&self) -> PeerstateVerifiedStatus {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.typ == Chattype::VerifiedGroup {
+                if chat.is_protected() {
                     PeerstateVerifiedStatus::BidirectVerified
                 } else {
                     PeerstateVerifiedStatus::Unverified
@@ -268,7 +268,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     fn should_force_plaintext(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.typ == Chattype::VerifiedGroup {
+                if chat.is_protected() {
                     false
                 } else {
                     self.msg
@@ -345,7 +345,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                         .stock_str(StockMessage::AcSetupMsgSubject)
                         .await
                         .into_owned()
-                } else if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
+                } else if chat.typ == Chattype::Group {
                     let re = if self.in_reply_to.is_empty() {
                         ""
                     } else {
@@ -708,11 +708,11 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let mut placeholdertext = None;
         let mut meta_part = None;
 
-        if chat.typ == Chattype::VerifiedGroup {
+        if chat.is_protected() {
             protected_headers.push(Header::new("Chat-Verified".to_string(), "1".to_string()));
         }
 
-        if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
+        if chat.typ == Chattype::Group {
             protected_headers.push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
             let encoded = encode_words(&chat.name);
@@ -846,6 +846,18 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                 }
             }
+            SystemMessage::ChatProtectionEnabled => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "protection-enabled".to_string(),
+                ));
+            }
+            SystemMessage::ChatProtectionDisabled => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "protection-disabled".to_string(),
+                ));
+            }
             _ => {}
         }
 
@@ -917,12 +929,17 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         };
 
+        let quoted_text = self
+            .msg
+            .quoted_text()
+            .map(|quote| format_flowed_quote(&quote) + "\r\n\r\n");
         let flowed_text = format_flowed(final_text);
 
         let footer = &self.selfstatus;
         let message_text = format!(
-            "{}{}{}{}{}",
+            "{}{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
+            quoted_text.unwrap_or_default(),
             escape_message_footer_marks(&flowed_text),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"

src/mimeparser.rs

@@ -3,9 +3,9 @@ use std::future::Future;
 use std::pin::Pin;
 
 use deltachat_derive::{FromSql, ToSql};
-use lazy_static::lazy_static;
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
+use once_cell::sync::Lazy;
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
@@ -86,6 +86,10 @@ pub enum SystemMessage {
 
     /// Chat ephemeral message timer is changed.
     EphemeralTimerChanged = 10,
+
+    // Chat protection state changed
+    ChatProtectionEnabled = 11,
+    ChatProtectionDisabled = 12,
 }
 
 impl Default for SystemMessage {
@@ -123,6 +127,7 @@ impl MimeMessage {
 
         // remove headers that are allowed _only_ in the encrypted part
         headers.remove("secure-join-fingerprint");
+        headers.remove("chat-verified");
 
         // Memory location for a possible decrypted message.
         let mail_raw;
@@ -218,12 +223,13 @@ impl MimeMessage {
             failure_report: None,
         };
         parser.parse_mime_recursive(context, &mail).await?;
+        parser.maybe_remove_bad_parts().await;
         parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context)?;
 
         if warn_empty_signature && parser.signatures.is_empty() {
             for part in parser.parts.iter_mut() {
-                part.error = "No valid signature".to_string();
+                part.error = Some("No valid signature".to_string());
             }
         }
 
@@ -253,6 +259,10 @@ impl MimeMessage {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
             } else if value == "ephemeral-timer-changed" {
                 self.is_system_message = SystemMessage::EphemeralTimerChanged;
+            } else if value == "protection-enabled" {
+                self.is_system_message = SystemMessage::ChatProtectionEnabled;
+            } else if value == "protection-disabled" {
+                self.is_system_message = SystemMessage::ChatProtectionDisabled;
             }
         }
         Ok(())
@@ -589,7 +599,7 @@ impl MimeMessage {
                 part.typ = Viewtype::Text;
                 part.msg_raw = Some(txt.clone());
                 part.msg = txt;
-                part.error = "Decryption failed".to_string();
+                part.error = Some("Decryption failed".to_string());
 
                 self.parts.push(part);
 
@@ -704,12 +714,17 @@ impl MimeMessage {
                             }
                         };
 
-                        let (simplified_txt, is_forwarded) = if decoded_data.is_empty() {
-                            ("".into(), false)
+                        let mut dehtml_failed = false;
+
+                        let (simplified_txt, is_forwarded, top_quote) = if decoded_data.is_empty() {
+                            ("".to_string(), false, None)
                         } else {
                             let is_html = mime_type == mime::TEXT_HTML;
                             let out = if is_html {
-                                dehtml(&decoded_data)
+                                dehtml(&decoded_data).unwrap_or_else(|| {
+                                    dehtml_failed = true;
+                                    decoded_data.clone()
+                                })
                             } else {
                                 decoded_data.clone()
                             };
@@ -723,7 +738,7 @@ impl MimeMessage {
                             false
                         };
 
-                        let simplified_txt = if mime_type.type_() == mime::TEXT
+                        let (simplified_txt, simplified_quote) = if mime_type.type_() == mime::TEXT
                             && mime_type.subtype() == mime::PLAIN
                             && is_format_flowed
                         {
@@ -732,16 +747,22 @@ impl MimeMessage {
                             } else {
                                 false
                             };
-                            unformat_flowed(&simplified_txt, delsp)
+                            let unflowed_text = unformat_flowed(&simplified_txt, delsp);
+                            let unflowed_quote = top_quote.map(|q| unformat_flowed(&q, delsp));
+                            (unflowed_text, unflowed_quote)
                         } else {
-                            simplified_txt
+                            (simplified_txt, top_quote)
                         };
 
-                        if !simplified_txt.is_empty() {
+                        if !simplified_txt.is_empty() || simplified_quote.is_some() {
                             let mut part = Part::default();
+                            part.dehtlm_failed = dehtml_failed;
                             part.typ = Viewtype::Text;
                             part.mimetype = Some(mime_type);
                             part.msg = simplified_txt;
+                            if let Some(quote) = simplified_quote {
+                                part.param.set(Param::Quote, quote);
+                            }
                             part.msg_raw = Some(decoded_data);
                             self.do_add_single_part(part);
                         }
@@ -843,6 +864,7 @@ impl MimeMessage {
     }
 
     pub fn repl_msg_by_error(&mut self, error_msg: impl AsRef<str>) {
+        self.is_system_message = SystemMessage::Unknown;
         if let Some(part) = self.parts.first_mut() {
             part.typ = Viewtype::Text;
             part.msg = format!("[{}]", error_msg.as_ref());
@@ -977,11 +999,21 @@ impl MimeMessage {
         Ok(None)
     }
 
+    async fn maybe_remove_bad_parts(&mut self) {
+        let good_parts = self.parts.iter().filter(|p| !p.dehtlm_failed).count();
+        if good_parts == 0 {
+            // We have no good part but show at least one bad part in order to show anything at all
+            self.parts.truncate(1);
+        } else if good_parts < self.parts.len() {
+            self.parts.retain(|p| !p.dehtlm_failed);
+        }
+    }
+
     /// Some providers like GMX and Yahoo do not send standard NDNs (Non Delivery notifications).
     /// If you improve heuristics here you might also have to change prefetch_should_download() in imap/mod.rs.
     /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
     #[allow(clippy::indexing_slicing)]
-    async fn heuristically_parse_ndn(&mut self, context: &Context) -> Option<()> {
+    async fn heuristically_parse_ndn(&mut self, context: &Context) {
         let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
             let from = from.to_ascii_lowercase();
             from.contains("mailer-daemon") || from.contains("mail-daemon")
@@ -989,9 +1021,8 @@ impl MimeMessage {
             false
         };
         if maybe_ndn && self.failure_report.is_none() {
-            lazy_static! {
-                static ref RE: regex::Regex = regex::Regex::new(r"Message-ID:(.*)").unwrap();
-            }
+            static RE: Lazy<regex::Regex> =
+                Lazy::new(|| regex::Regex::new(r"Message-ID:(.*)").unwrap());
             for captures in self
                 .parts
                 .iter()
@@ -1011,7 +1042,6 @@ impl MimeMessage {
                 }
             }
         }
-        None // Always return None, we just return anything so that we can use the '?' operator.
     }
 
     /// Handle reports
@@ -1041,7 +1071,9 @@ impl MimeMessage {
                 .iter()
                 .find(|p| p.typ == Viewtype::Text)
                 .map(|p| p.msg.clone());
-            message::handle_ndn(context, failure_report, error).await
+            if let Err(e) = message::handle_ndn(context, failure_report, error).await {
+                warn!(context, "Could not handle ndn: {}", e);
+            }
         }
     }
 
@@ -1153,11 +1185,21 @@ pub(crate) fn parse_message_id(ids: &str) -> Result<String> {
 }
 
 fn is_known(key: &str) -> bool {
-    match key {
-        "return-path" | "date" | "from" | "sender" | "reply-to" | "to" | "cc" | "bcc"
-        | "message-id" | "in-reply-to" | "references" | "subject" => true,
-        _ => false,
-    }
+    matches!(
+        key,
+        "return-path"
+            | "date"
+            | "from"
+            | "sender"
+            | "reply-to"
+            | "to"
+            | "cc"
+            | "bcc"
+            | "message-id"
+            | "in-reply-to"
+            | "references"
+            | "subject"
+    )
 }
 
 #[derive(Debug, Default, Clone)]
@@ -1169,7 +1211,8 @@ pub struct Part {
     pub bytes: usize,
     pub param: Params,
     org_filename: Option<String>,
-    pub error: String,
+    pub error: Option<String>,
+    dehtlm_failed: bool,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1271,9 +1314,9 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String
 }
 
 /// Returned addresses are normalized and lowercased.
-fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
+pub(crate) fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
     get_all_addresses_from_header(headers, |header_key| {
-        header_key == "to" || header_key == "cc"
+        header_key == "to" || header_key == "cc" || header_key == "bcc"
     })
 }
 
@@ -1851,6 +1894,55 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
         assert_eq!(message.parts[0].msg, "Hello!");
     }
 
+    #[async_std::test]
+    async fn test_hide_html_without_content() {
+        let t = TestContext::new().await;
+        let raw = br#"Date: Thu, 13 Feb 2020 22:41:20 +0000 (UTC)
+From: sender@example.com
+To: receiver@example.com
+Subject: Mail with inline attachment
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+	boundary="----=_Part_25_46172632.1581201680436"
+
+------=_Part_25_46172632.1581201680436
+Content-Type: text/html; charset=utf-8
+
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=Windows-1252">
+<meta name="GENERATOR" content="MSHTML 11.00.10570.1001"></head>
+<body><img align="baseline" alt="" src="cid:1712254131-1" border="0" hspace="0">
+</body>
+
+------=_Part_25_46172632.1581201680436
+Content-Type: application/pdf; name="some_pdf.pdf"
+Content-Transfer-Encoding: base64
+Content-Disposition: inline; filename="some_pdf.pdf"
+
+JVBERi0xLjUKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURl
+Y29kZT4+CnN0cmVhbQp4nGVOuwoCMRDs8xVbC8aZvC4Hx4Hno7ATAhZi56MTtPH33YtXiLKQ3ZnM
+MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
+------=_Part_25_46172632.1581201680436--
+"#;
+
+        let message = MimeMessage::from_bytes(&t.ctx, &raw[..]).await.unwrap();
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::File);
+        assert_eq!(message.parts[0].msg, "");
+
+        // Make sure the file is there even though the html is wrong:
+        let param = &message.parts[0].param;
+        let blob: BlobObject = param
+            .get_blob(Param::File, &t.ctx, false)
+            .await
+            .unwrap()
+            .unwrap();
+        let f = async_std::fs::File::open(blob.to_abs_path()).await.unwrap();
+        let size = f.metadata().await.unwrap().len();
+        assert_eq!(size, 154);
+    }
+
     #[async_std::test]
     async fn parse_inline_image() {
         let context = TestContext::new().await;
@@ -2109,12 +2201,71 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
         assert!(test.is_empty());
     }
 
-    #[test]
-    fn test_mime_parse_format_flowed() {
-        let mime_type = "text/plain; charset=utf-8; Format=Flowed; DelSp=No"
-            .parse::<mime::Mime>()
+    #[async_std::test]
+    async fn parse_format_flowed_quote() {
+        let context = TestContext::new().await;
+        let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+Subject: Re: swipe-to-reply
+MIME-Version: 1.0
+In-Reply-To: <bar@example.org>
+Date: Tue, 06 Oct 2020 00:00:00 +0000
+Chat-Version: 1.0
+Message-ID: <foo@example.org>
+To: bob <bob@example.org>
+From: alice <alice@example.org>
+
+> Long 
+> quote.
+
+Reply
+"##;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
             .unwrap();
-        let format_param = mime_type.get_param("format").unwrap();
-        assert_eq!(format_param.as_str().to_ascii_lowercase(), "flowed");
+        assert_eq!(
+            message.get_subject(),
+            Some("Re: swipe-to-reply".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Text);
+        assert_eq!(
+            message.parts[0].param.get(Param::Quote).unwrap(),
+            "Long quote."
+        );
+        assert_eq!(message.parts[0].msg, "Reply");
+    }
+
+    #[async_std::test]
+    async fn parse_quote_without_reply() {
+        let context = TestContext::new().await;
+        let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+Subject: Re: swipe-to-reply
+MIME-Version: 1.0
+In-Reply-To: <bar@example.org>
+Date: Tue, 06 Oct 2020 00:00:00 +0000
+Message-ID: <foo@example.org>
+To: bob <bob@example.org>
+From: alice <alice@example.org>
+
+> Just a quote.
+"##;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(
+            message.get_subject(),
+            Some("Re: swipe-to-reply".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Text);
+        assert_eq!(
+            message.parts[0].param.get(Param::Quote).unwrap(),
+            "Just a quote."
+        );
+        assert_eq!(message.parts[0].msg, "");
     }
 }

src/oauth2.rs

@@ -170,16 +170,14 @@ pub async fn dc_get_oauth2_access_token(
         }
 
         // ... and POST
-        let response = surf::post(post_url).body_form(&post_param);
-        if response.is_err() {
-            warn!(
-                context,
-                "Error calling OAuth2 at {}: {:?}", token_url, response
-            );
+        let mut req = surf::post(post_url).build();
+        if let Err(err) = req.body_form(&post_param) {
+            warn!(context, "Error calling OAuth2 at {}: {:?}", token_url, err);
             return None;
         }
 
-        let parsed: Result<Response, _> = response.unwrap().recv_json().await;
+        let client = surf::Client::new();
+        let parsed: Result<Response, _> = client.recv_json(req).await;
         if parsed.is_err() {
             warn!(
                 context,
@@ -305,7 +303,7 @@ impl Oauth2 {
 
                     let mut fqdn: String = String::from(domain.as_ref());
                     if !fqdn.ends_with('.') {
-                        fqdn.push_str(".");
+                        fqdn.push('.');
                     }
 
                     if let Ok(res) = resolver.mx_lookup(fqdn).await {
@@ -323,7 +321,7 @@ impl Oauth2 {
     }
 
     async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
-        let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
+        let userinfo_url = self.get_userinfo.unwrap_or("");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
 
         // should returns sth. as

src/param.rs

@@ -3,12 +3,13 @@ use std::fmt;
 use std::str;
 
 use async_std::path::PathBuf;
+use itertools::Itertools;
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
-use crate::error::{self, bail, ensure};
+use crate::error::{self, bail};
 use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 
@@ -52,6 +53,9 @@ pub enum Param {
     /// For Messages
     Forwarded = b'a',
 
+    /// For Messages: quoted text.
+    Quote = b'q',
+
     /// For Messages
     Cmd = b'S',
 
@@ -146,7 +150,12 @@ impl fmt::Display for Params {
             if i > 0 {
                 writeln!(f)?;
             }
-            write!(f, "{}={}", *key as u8 as char, value)?;
+            write!(
+                f,
+                "{}={}",
+                *key as u8 as char,
+                value.split('\n').join("\n\n")
+            )?;
         }
         Ok(())
     }
@@ -157,27 +166,28 @@ impl str::FromStr for Params {
 
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let mut inner = BTreeMap::new();
-        for pair in s.trim().lines() {
-            let pair = pair.trim();
-            if pair.is_empty() {
-                continue;
-            }
-            // TODO: probably nicer using a regex
-            ensure!(pair.len() > 1, "Invalid key pair: '{}'", pair);
-            let mut split = pair.splitn(2, '=');
-            let key = split.next();
-            let value = split.next();
+        let mut lines = s.lines().peekable();
 
-            ensure!(key.is_some(), "Missing key");
-            ensure!(value.is_some(), "Missing value");
+        while let Some(line) = lines.next() {
+            if let [key, value] = line.splitn(2, '=').collect::<Vec<_>>()[..] {
+                let key = key.to_string();
+                let mut value = value.to_string();
+                while let Some(s) = lines.peek() {
+                    if !s.is_empty() {
+                        break;
+                    }
+                    lines.next();
+                    value.push('\n');
+                    value += lines.next().unwrap_or_default();
+                }
 
-            let key = key.unwrap_or_default().trim();
-            let value = value.unwrap_or_default().trim();
-
-            if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
-                inner.insert(key, value.to_string());
+                if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
+                    inner.insert(key, value);
+                } else {
+                    bail!("Unknown key: {}", key);
+                }
             } else {
-                bail!("Unknown key: {}", key);
+                bail!("Not a key-value pair: {:?}", line);
             }
         }
 
@@ -373,7 +383,7 @@ mod tests {
 
     #[test]
     fn test_dc_param() {
-        let mut p1: Params = "\r\n\r\na=1\nf=2\n\nc = 3 ".parse().unwrap();
+        let mut p1: Params = "a=1\nf=2\nc=3".parse().unwrap();
 
         assert_eq!(p1.get_int(Param::Forwarded), Some(1));
         assert_eq!(p1.get_int(Param::File), Some(2));
@@ -407,6 +417,14 @@ mod tests {
         assert_eq!(p1.len(), 0)
     }
 
+    #[test]
+    fn test_roundtrip() {
+        let mut params = Params::new();
+        params.set(Param::Height, "foo\nbar=baz\nquux");
+        params.set(Param::Width, "\n\n\na=\n=");
+        assert_eq!(params.to_string().parse::<Params>().unwrap(), params);
+    }
+
     #[test]
     fn test_regression() {
         let p1: Params = "a=cli%40deltachat.de\nn=\ni=TbnwJ6lSvD5\ns=0ejvbdFSQxB"

src/pgp.rs

@@ -381,7 +381,7 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
 mod tests {
     use super::*;
     use crate::test_utils::*;
-    use lazy_static::lazy_static;
+    use once_cell::sync::Lazy;
 
     #[test]
     fn test_split_armored_data_1() {
@@ -449,26 +449,29 @@ mod tests {
     /// The original text of [CTEXT_SIGNED]
     static CLEARTEXT: &[u8] = b"This is a test";
 
-    lazy_static! {
-        /// Initialised [TestKeys] for tests.
-        static ref KEYS: TestKeys = TestKeys::new();
+    /// Initialised [TestKeys] for tests.
+    static KEYS: Lazy<TestKeys> = Lazy::new(TestKeys::new);
 
-        /// A cyphertext encrypted to Alice & Bob, signed by Alice.
-        static ref CTEXT_SIGNED: String = {
-            let mut keyring = Keyring::new();
-            keyring.add(KEYS.alice_public.clone());
-            keyring.add(KEYS.bob_public.clone());
-            smol::block_on(pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))).unwrap()
-        };
+    /// A cyphertext encrypted to Alice & Bob, signed by Alice.
+    static CTEXT_SIGNED: Lazy<String> = Lazy::new(|| {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_public.clone());
+        keyring.add(KEYS.bob_public.clone());
+        futures_lite::future::block_on(pk_encrypt(
+            CLEARTEXT,
+            keyring,
+            Some(KEYS.alice_secret.clone()),
+        ))
+        .unwrap()
+    });
 
-        /// A cyphertext encrypted to Alice & Bob, not signed.
-        static ref CTEXT_UNSIGNED: String = {
-            let mut keyring = Keyring::new();
-            keyring.add(KEYS.alice_public.clone());
-            keyring.add(KEYS.bob_public.clone());
-            smol::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
-        };
-    }
+    /// A cyphertext encrypted to Alice & Bob, not signed.
+    static CTEXT_UNSIGNED: Lazy<String> = Lazy::new(|| {
+        let mut keyring = Keyring::new();
+        keyring.add(KEYS.alice_public.clone());
+        keyring.add(KEYS.bob_public.clone());
+        futures_lite::future::block_on(pk_encrypt(CLEARTEXT, keyring, None)).unwrap()
+    });
 
     #[test]
     fn test_encrypt_signed() {

src/provider/mod.rs

@@ -4,7 +4,8 @@ mod data;
 
 use crate::config::Config;
 use crate::dc_tools::EmailAddress;
-use crate::provider::data::PROVIDER_DATA;
+use crate::provider::data::{PROVIDER_DATA, PROVIDER_UPDATED};
+use chrono::{NaiveDateTime, NaiveTime};
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
@@ -91,11 +92,18 @@ pub fn get_provider_info(addr: &str) -> Option<&Provider> {
     None
 }
 
+// returns update timestamp in seconds, UTC, compatible for comparison with time() and database times
+pub fn get_provider_update_timestamp() -> i64 {
+    NaiveDateTime::new(*PROVIDER_UPDATED, NaiveTime::from_hms(0, 0, 0)).timestamp_millis() / 1_000
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
+    use crate::dc_tools::time;
+    use chrono::NaiveDate;
 
     #[test]
     fn test_get_provider_info_unexistant() {
@@ -138,4 +146,16 @@ mod tests {
         let provider = get_provider_info("user@googlemail.com").unwrap();
         assert!(provider.status == Status::PREPARATION);
     }
+
+    #[test]
+    fn test_get_provider_update_timestamp() {
+        let timestamp_past = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 9, 9),
+            NaiveTime::from_hms(0, 0, 0),
+        )
+        .timestamp_millis()
+            / 1_000;
+        assert!(get_provider_update_timestamp() <= time());
+        assert!(get_provider_update_timestamp() > timestamp_past);
+    }
 }

src/provider/update.py

@@ -4,6 +4,7 @@
 import sys
 import os
 import yaml
+import datetime
 
 out_all = ""
 out_domains = ""
@@ -41,8 +42,8 @@ def process_config_defaults(data):
     config_defaults = data.get("config_defaults", "")
     for key in config_defaults:
         value = str(config_defaults[key])
-        defaults += "            ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
-    defaults += "        ])"
+        defaults += "        ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
+    defaults += "    ])"
     return defaults
 
 
@@ -65,7 +66,7 @@ def process_data(data, file):
             raise TypeError("domain used twice: " + domain)
         domains_dict[domain] = True
 
-        domains += "        (\"" + domain + "\", &*" + file2varname(file) + "),\n"
+        domains += "    (\"" + domain + "\", &*" + file2varname(file) + "),\n"
         comment += domain + ", "
 
 
@@ -95,7 +96,7 @@ def process_data(data, file):
             if username_pattern != "EMAIL" and username_pattern != "EMAILLOCALPART":
                 raise TypeError("bad username pattern")
 
-            server += ("            Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
+            server += ("        Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
             + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern + " },\n")
 
     config_defaults = process_config_defaults(data)
@@ -110,16 +111,16 @@ def process_data(data, file):
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
     if (not has_imap and not has_smtp) or (has_imap and has_smtp):
-        provider += "    static ref " + file2varname(file) + ": Provider = Provider {\n"
-        provider += "        status: Status::" + status + ",\n"
-        provider += "        before_login_hint: \"" + before_login_hint + "\",\n"
-        provider += "        after_login_hint: \"" + after_login_hint + "\",\n"
-        provider += "        overview_page: \"" + file2url(file) + "\",\n"
-        provider += "        server: vec![\n" + server + "        ],\n"
-        provider += "        config_defaults: " + config_defaults + ",\n"
-        provider += "        strict_tls: " + strict_tls + ",\n"
-        provider += "        oauth2_authorizer: " + oauth2 + ",\n"
-        provider += "    };\n\n"
+        provider += "static " + file2varname(file) + ": Lazy<Provider> = Lazy::new(|| Provider {\n"
+        provider += "    status: Status::" + status + ",\n"
+        provider += "    before_login_hint: \"" + before_login_hint + "\",\n"
+        provider += "    after_login_hint: \"" + after_login_hint + "\",\n"
+        provider += "    overview_page: \"" + file2url(file) + "\",\n"
+        provider += "    server: vec![\n" + server + "    ],\n"
+        provider += "    config_defaults: " + config_defaults + ",\n"
+        provider += "    strict_tls: " + strict_tls + ",\n"
+        provider += "    oauth2_authorizer: " + oauth2 + ",\n"
+        provider += "});\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
 
@@ -128,7 +129,7 @@ def process_data(data, file):
 
     # finally, add the provider
     global out_all, out_domains
-    out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
+    out_all += "// " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
 
     # also add provider with no special things to do -
     # eg. _not_ supporting oauth2 is also an information and we can skip the mx-lookup in this case
@@ -163,12 +164,16 @@ if __name__ == "__main__":
     "use crate::provider::UsernamePattern::*;\n"
     "use crate::provider::*;\n"
     "use std::collections::HashMap;\n\n"
-    "lazy_static::lazy_static! {\n\n")
+    "use once_cell::sync::Lazy;\n\n")
 
     process_dir(sys.argv[1])
 
-    out_all += "    pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [\n"
+    out_all += "pub static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| [\n"
     out_all += out_domains;
-    out_all += "    ].iter().copied().collect();\n}"
+    out_all += "].iter().copied().collect());\n\n"
+
+    now = datetime.datetime.utcnow()
+    out_all += "pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> = "\
+               "Lazy::new(|| chrono::NaiveDate::from_ymd("+str(now.year)+", "+str(now.month)+", "+str(now.day)+"));\n"
 
     print(out_all)

src/qr.rs

@@ -1,6 +1,6 @@
 //! # QR code module
 
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use percent_encoding::percent_decode_str;
 use serde::Deserialize;
 
@@ -358,12 +358,10 @@ async fn decode_matmsg(context: &Context, qr: &str) -> Lot {
     Lot::from_address(context, name, addr).await
 }
 
-lazy_static! {
-    static ref VCARD_NAME_RE: regex::Regex =
-        regex::Regex::new(r"(?m)^N:([^;]*);([^;\n]*)").unwrap();
-    static ref VCARD_EMAIL_RE: regex::Regex =
-        regex::Regex::new(r"(?m)^EMAIL([^:\n]*):([^;\n]*)").unwrap();
-}
+static VCARD_NAME_RE: Lazy<regex::Regex> =
+    Lazy::new(|| regex::Regex::new(r"(?m)^N:([^;]*);([^;\n]*)").unwrap());
+static VCARD_EMAIL_RE: Lazy<regex::Regex> =
+    Lazy::new(|| regex::Regex::new(r"(?m)^EMAIL([^:\n]*):([^;\n]*)").unwrap());
 
 /// Extract address for the matmsg scheme.
 ///

src/scheduler.rs

@@ -3,6 +3,7 @@ use async_std::sync::{channel, Receiver, Sender};
 use async_std::task;
 
 use crate::context::Context;
+use crate::dc_tools::maybe_add_time_based_warnings;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
 use crate::{config::Config, message::MsgId, smtp::Smtp};
@@ -81,6 +82,8 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                         warn!(ctx, "failed to close folder: {:?}", err);
                     }
 
+                    maybe_add_time_based_warnings(&ctx).await;
+
                     info = if ctx.get_config_bool(Config::InboxWatch).await {
                         fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
                     } else {
@@ -128,7 +131,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
             // connect and fake idle if unable to connect
             if let Err(err) = connection.connect_configured(&ctx).await {
                 warn!(ctx, "imap connection failed: {}", err);
-                return connection.fake_idle(&ctx, None).await;
+                return connection.fake_idle(&ctx, Some(watch_folder)).await;
             }
 
             // fetch
@@ -412,10 +415,7 @@ impl Scheduler {
 
     /// Check if the scheduler is running.
     pub fn is_running(&self) -> bool {
-        match self {
-            Scheduler::Running { .. } => true,
-            _ => false,
-        }
+        matches!(self, Scheduler::Running { .. })
     }
 }
 

src/securejoin.rs

@@ -348,7 +348,7 @@ async fn securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
                 let bob = context.bob.read().await;
                 let grpid = bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap();
                 match chat::get_chat_id_by_grpid(context, grpid).await {
-                    Ok((chatid, _is_verified, _blocked)) => break chatid,
+                    Ok((chatid, _is_protected, _blocked)) => break chatid,
                     Err(err) => {
                         if start.elapsed() > Duration::from_secs(7) {
                             return Err(JoinError::MissingChat(err));
@@ -791,19 +791,19 @@ pub(crate) async fn handle_securejoin_handshake(
 
             let vg_expect_encrypted = if join_vg {
                 let group_id = get_qr_attr!(context, text2).to_string();
-                // This is buggy, is_verified_group will always be
+                // This is buggy, is_protected_group will always be
                 // false since the group is created by receive_imf by
                 // the very handshake message we're handling now.  But
                 // only after we have returned.  It does not impact
                 // the security invariants of secure-join however.
-                let (_, is_verified_group, _) = chat::get_chat_id_by_grpid(context, &group_id)
+                let (_, is_protected_group, _) = chat::get_chat_id_by_grpid(context, &group_id)
                     .await
                     .unwrap_or((ChatId::new(0), false, Blocked::Not));
                 // when joining a non-verified group
                 // the vg-member-added message may be unencrypted
                 // when not all group members have keys or prefer encryption.
                 // So only expect encryption if this is a verified group
-                is_verified_group
+                is_protected_group
             } else {
                 // setup contact is always encrypted
                 true
@@ -1102,6 +1102,7 @@ mod tests {
     use super::*;
 
     use crate::chat;
+    use crate::chat::ProtectionStatus;
     use crate::peerstate::Peerstate;
     use crate::test_utils::TestContext;
 
@@ -1328,7 +1329,7 @@ mod tests {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
-        let chatid = chat::create_group_chat(&alice.ctx, VerifiedStatus::Verified, "the chat")
+        let chatid = chat::create_group_chat(&alice.ctx, ProtectionStatus::Protected, "the chat")
             .await
             .unwrap();
 
@@ -1427,6 +1428,6 @@ mod tests {
 
         let bob_chatid = joiner.await;
         let bob_chat = Chat::load_from_db(&bob.ctx, bob_chatid).await.unwrap();
-        assert!(bob_chat.is_verified());
+        assert!(bob_chat.is_protected());
     }
 }

src/simplify.rs

@@ -1,3 +1,5 @@
+use itertools::Itertools;
+
 // protect lines starting with `--` against being treated as a footer.
 // for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B);
 // this should be invisible on most systems and there is no need to unescape it again
@@ -64,33 +66,28 @@ fn split_lines(buf: &str) -> Vec<&str> {
 
 /// Simplify message text for chat display.
 /// Remove quotes, signatures, trailing empty lines etc.
-pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
+pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool, Option<String>) {
     input.retain(|c| c != '\r');
     let lines = split_lines(&input);
     let (lines, is_forwarded) = skip_forward_header(&lines);
 
+    let (lines, top_quote) = remove_top_quote(lines);
     let original_lines = &lines;
-
     let lines = remove_message_footer(lines);
 
     let text = if is_chat_message {
-        render_message(lines, false, false)
+        render_message(lines, false)
     } else {
         let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
         let (lines, has_bottom_quote) = remove_bottom_quote(lines);
-        let (lines, has_top_quote) = remove_top_quote(lines);
 
         if lines.iter().all(|it| it.trim().is_empty()) {
-            render_message(original_lines, false, false)
+            render_message(original_lines, false)
         } else {
-            render_message(
-                lines,
-                has_top_quote,
-                has_nonstandard_footer || has_bottom_quote,
-            )
+            render_message(lines, has_nonstandard_footer || has_bottom_quote)
         }
     };
-    (text, is_forwarded)
+    (text, is_forwarded, top_quote)
 }
 
 /// Skips "forwarded message" header.
@@ -134,11 +131,15 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
 }
 
 #[allow(clippy::indexing_slicing)]
-fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<String>) {
+    let mut first_quoted_line = 0;
     let mut last_quoted_line = None;
     let mut has_quoted_headline = false;
     for (l, line) in lines.iter().enumerate() {
         if is_plain_quote(line) {
+            if last_quoted_line.is_none() {
+                first_quoted_line = l;
+            }
             last_quoted_line = Some(l)
         } else if !is_empty_line(line) {
             if is_quoted_headline(line) && !has_quoted_headline && last_quoted_line.is_none() {
@@ -150,17 +151,25 @@ fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
         }
     }
     if let Some(last_quoted_line) = last_quoted_line {
-        (&lines[last_quoted_line + 1..], true)
+        (
+            &lines[last_quoted_line + 1..],
+            Some(
+                lines[first_quoted_line..last_quoted_line + 1]
+                    .iter()
+                    .map(|s| {
+                        s.strip_prefix(">")
+                            .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u))
+                    })
+                    .join("\n"),
+            ),
+        )
     } else {
-        (lines, false)
+        (lines, None)
     }
 }
 
-fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) -> String {
+fn render_message(lines: &[&str], is_cut_at_end: bool) -> String {
     let mut ret = String::new();
-    if is_cut_at_begin {
-        ret += "[...]";
-    }
     /* we write empty lines only in case and non-empty line follows */
     let mut pending_linebreaks = 0;
     let mut empty_body = true;
@@ -183,7 +192,7 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) ->
             pending_linebreaks = 1
         }
     }
-    if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
+    if is_cut_at_end && !empty_body {
         ret += " [...]";
     }
     // redo escaping done by escape_message_footer_marks()
@@ -231,7 +240,7 @@ mod tests {
         #[test]
         // proptest does not support [[:graphical:][:space:]] regex.
         fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let (output, _is_forwarded) = simplify(input, true);
+            let (output, _is_forwarded, _) = simplify(input, true);
             assert!(output.split('\n').all(|s| s != "-- "));
         }
     }
@@ -239,7 +248,7 @@ mod tests {
     #[test]
     fn test_dont_remove_whole_message() {
         let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, _) = simplify(input, false);
         assert_eq!(
             plain,
             "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
@@ -250,7 +259,7 @@ mod tests {
     #[test]
     fn test_chat_message() {
         let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
-        let (plain, is_forwarded) = simplify(input, true);
+        let (plain, is_forwarded, _) = simplify(input, true);
         assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
         assert!(!is_forwarded);
     }
@@ -258,7 +267,7 @@ mod tests {
     #[test]
     fn test_simplify_trim() {
         let input = "line1\n\r\r\rline2".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, _) = simplify(input, false);
 
         assert_eq!(plain, "line1\nline2");
         assert!(!is_forwarded);
@@ -267,7 +276,7 @@ mod tests {
     #[test]
     fn test_simplify_forwarded_message() {
         let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let (plain, is_forwarded, _) = simplify(input, false);
 
         assert_eq!(plain, "Forwarded message");
         assert!(is_forwarded);
@@ -287,17 +296,17 @@ mod tests {
 
     #[test]
     fn test_remove_top_quote() {
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second"]);
+        let (lines, top_quote) = remove_top_quote(&["> first", "> second"]);
         assert!(lines.is_empty());
-        assert!(has_top_quote);
+        assert_eq!(top_quote.unwrap(), "first\nsecond");
 
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
+        let (lines, top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
         assert_eq!(lines, &["not a quote"]);
-        assert!(has_top_quote);
+        assert_eq!(top_quote.unwrap(), "first\nsecond");
 
-        let (lines, has_top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
+        let (lines, top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
         assert_eq!(lines, &["not a quote", "> first", "> second"]);
-        assert!(!has_top_quote);
+        assert!(top_quote.is_none());
     }
 
     #[test]
@@ -312,41 +321,41 @@ mod tests {
     #[test]
     fn test_remove_message_footer() {
         let input = "text\n--\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n--\nno footer");
 
         let input = "text\n\n--\n\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\n\nno footer");
 
         let input = "text\n\n-- no footer\n\n".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n\n-- no footer");
 
         let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
-        let (plain, _) = simplify(input, true);
+        let (plain, _, _) = simplify(input, true);
         assert_eq!(plain, "text\n\n--\nno footer");
 
         let input = "text\n\n--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, _) = simplify(input.clone(), true);
         assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
+        let (plain, _, _) = simplify(escaped, true);
         assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
 
         // Nonstandard footer sent by https://siju.es/
         let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
-        let (plain, _) = simplify(input.clone(), false);
+        let (plain, _, _) = simplify(input.clone(), false);
         assert_eq!(plain, "Message text here [...]");
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, _) = simplify(input.clone(), true);
         assert_eq!(plain, input);
 
         let input = "--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
+        let (plain, _, _) = simplify(input.clone(), true);
         assert_eq!(plain, ""); // see remove_message_footer() for some explanations
 
         let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
+        let (plain, _, _) = simplify(escaped, true);
         assert_eq!(plain, "--\ntreated as footer when unescaped");
     }
 }

src/smtp/mod.rs

@@ -30,7 +30,7 @@ pub enum Error {
         error: error::Error,
     },
 
-    #[error("SMTP: failed to connect: {0:?}")]
+    #[error("SMTP: failed to connect: {0}")]
     ConnectionFailure(#[source] smtp::error::Error),
 
     #[error("SMTP: failed to setup connection {0:?}")]

src/sql.rs

@@ -14,6 +14,7 @@ use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH};
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::ephemeral::start_ephemeral_timers;
+use crate::error::format_err;
 use crate::param::*;
 use crate::peerstate::*;
 
@@ -77,18 +78,29 @@ impl Sql {
         // drop closes the connection
     }
 
-    // return true on success, false on failure
-    pub async fn open<T: AsRef<Path>>(&self, context: &Context, dbfile: T, readonly: bool) -> bool {
-        match open(context, self, dbfile, readonly).await {
-            Ok(_) => true,
-            Err(err) => match err.downcast_ref::<Error>() {
-                Some(Error::SqlAlreadyOpen) => false,
+    pub async fn open<T: AsRef<Path>>(
+        &self,
+        context: &Context,
+        dbfile: T,
+        readonly: bool,
+    ) -> crate::error::Result<()> {
+        let res = open(context, self, &dbfile, readonly).await;
+        if let Err(err) = &res {
+            match err.downcast_ref::<Error>() {
+                Some(Error::SqlAlreadyOpen) => {}
                 _ => {
                     self.close().await;
-                    false
                 }
-            },
+            }
         }
+        res.map_err(|e| {
+            format_err!(
+                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                "Could not open db file {}: {:#}",
+                dbfile.as_ref().to_string_lossy(),
+                e
+            )
+        })
     }
 
     pub async fn execute<S: AsRef<str>>(
@@ -130,7 +142,7 @@ impl Sql {
         &self,
     ) -> Result<r2d2::PooledConnection<r2d2_sqlite::SqliteConnectionManager>> {
         let lock = self.pool.read().await;
-        let pool = lock.as_ref().ok_or_else(|| Error::SqlNoConnection)?;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
         let conn = pool.get()?;
 
         Ok(conn)
@@ -144,7 +156,7 @@ impl Sql {
             + FnOnce(r2d2::PooledConnection<r2d2_sqlite::SqliteConnectionManager>) -> Result<H>,
     {
         let lock = self.pool.read().await;
-        let pool = lock.as_ref().ok_or_else(|| Error::SqlNoConnection)?;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
         let conn = pool.get()?;
 
         g(conn)
@@ -156,7 +168,7 @@ impl Sql {
         Fut: Future<Output = Result<H>> + Send,
     {
         let lock = self.pool.read().await;
-        let pool = lock.as_ref().ok_or_else(|| Error::SqlNoConnection)?;
+        let pool = lock.as_ref().ok_or(Error::SqlNoConnection)?;
 
         let conn = pool.get()?;
         g(conn).await
@@ -666,7 +678,10 @@ async fn open(
         .with_flags(open_flags)
         .with_init(|c| {
             c.execute_batch(&format!(
-                "PRAGMA secure_delete=on; PRAGMA busy_timeout = {};",
+                "PRAGMA secure_delete=on;
+                 PRAGMA busy_timeout = {};
+                 PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android
+                 ",
                 Duration::from_secs(10).as_millis()
             ))?;
             Ok(())
@@ -693,156 +708,208 @@ async fn open(
             .ok();
 
         let mut exists_before_update = false;
-        let mut dbversion_before_update: i32 = 0;
-        /* Init tables to dbversion=0 */
+        // Init tables to dbversion=68
+        let mut dbversion_before_update: i32 = 68;
         if !sql.table_exists("config").await? {
             info!(
                 context,
                 "First time init: creating tables in {:?}.",
                 dbfile.as_ref(),
             );
-            sql.execute(
-                "CREATE TABLE config (id INTEGER PRIMARY KEY, keyname TEXT, value TEXT);",
-                paramsv![],
-            )
+            sql.with_conn(move |mut conn| {
+                let tx = conn.transaction()?;
+                tx.execute_batch(
+                    r#"
+CREATE TABLE config (id INTEGER PRIMARY KEY, keyname TEXT, value TEXT);
+CREATE INDEX config_index1 ON config (keyname);
+CREATE TABLE contacts (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT DEFAULT '',
+    addr TEXT DEFAULT '' COLLATE NOCASE,
+    origin INTEGER DEFAULT 0,
+    blocked INTEGER DEFAULT 0,
+    last_seen INTEGER DEFAULT 0,
+    param TEXT DEFAULT '',
+    authname TEXT DEFAULT '',
+    selfavatar_sent INTEGER DEFAULT 0
+);
+CREATE INDEX contacts_index1 ON contacts (name COLLATE NOCASE);
+CREATE INDEX contacts_index2 ON contacts (addr COLLATE NOCASE);
+INSERT INTO contacts (id,name,origin) VALUES
+(1,'self',262144), (2,'info',262144), (3,'rsvd',262144),
+(4,'rsvd',262144), (5,'device',262144), (6,'rsvd',262144),
+(7,'rsvd',262144), (8,'rsvd',262144), (9,'rsvd',262144);
+
+CREATE TABLE chats (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    type INTEGER DEFAULT 0,
+    name TEXT DEFAULT '',
+    draft_timestamp INTEGER DEFAULT 0,
+    draft_txt TEXT DEFAULT '',
+    blocked INTEGER DEFAULT 0,
+    grpid TEXT DEFAULT '',
+    param TEXT DEFAULT '',
+    archived INTEGER DEFAULT 0,
+    gossiped_timestamp INTEGER DEFAULT 0,
+    locations_send_begin INTEGER DEFAULT 0,
+    locations_send_until INTEGER DEFAULT 0,
+    locations_last_sent INTEGER DEFAULT 0,
+    created_timestamp INTEGER DEFAULT 0,
+    muted_until INTEGER DEFAULT 0,
+    ephemeral_timer INTEGER
+);
+CREATE INDEX chats_index1 ON chats (grpid);
+CREATE INDEX chats_index2 ON chats (archived);
+CREATE INDEX chats_index3 ON chats (locations_send_until);
+INSERT INTO chats (id,type,name) VALUES
+(1,120,'deaddrop'), (2,120,'rsvd'), (3,120,'trash'),
+(4,120,'msgs_in_creation'), (5,120,'starred'), (6,120,'archivedlink'),
+(7,100,'rsvd'), (8,100,'rsvd'), (9,100,'rsvd');
+
+CREATE TABLE chats_contacts (chat_id INTEGER, contact_id INTEGER);
+CREATE INDEX chats_contacts_index1 ON chats_contacts (chat_id);
+CREATE INDEX chats_contacts_index2 ON chats_contacts (contact_id);
+
+CREATE TABLE msgs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    rfc724_mid TEXT DEFAULT '',
+    server_folder TEXT DEFAULT '',
+    server_uid INTEGER DEFAULT 0,
+    chat_id INTEGER DEFAULT 0,
+    from_id INTEGER DEFAULT 0,
+    to_id INTEGER DEFAULT 0,
+    timestamp INTEGER DEFAULT 0,
+    type INTEGER DEFAULT 0,
+    state INTEGER DEFAULT 0,
+    msgrmsg INTEGER DEFAULT 1,
+    bytes INTEGER DEFAULT 0,
+    txt TEXT DEFAULT '',
+    txt_raw TEXT DEFAULT '',
+    param TEXT DEFAULT '',
+    starred INTEGER DEFAULT 0,
+    timestamp_sent INTEGER DEFAULT 0,
+    timestamp_rcvd INTEGER DEFAULT 0,
+    hidden INTEGER DEFAULT 0,
+    mime_headers TEXT,
+    mime_in_reply_to TEXT,
+    mime_references TEXT,
+    move_state INTEGER DEFAULT 1,
+    location_id INTEGER DEFAULT 0,
+    error TEXT DEFAULT '',
+
+-- Timer value in seconds. For incoming messages this
+-- timer starts when message is read, so we want to have
+-- the value stored here until the timer starts.
+    ephemeral_timer INTEGER DEFAULT 0,
+
+-- Timestamp indicating when the message should be
+-- deleted. It is convenient to store it here because UI
+-- needs this value to display how much time is left until
+-- the message is deleted.
+    ephemeral_timestamp INTEGER DEFAULT 0
+);
+
+CREATE INDEX msgs_index1 ON msgs (rfc724_mid);
+CREATE INDEX msgs_index2 ON msgs (chat_id);
+CREATE INDEX msgs_index3 ON msgs (timestamp);
+CREATE INDEX msgs_index4 ON msgs (state);
+CREATE INDEX msgs_index5 ON msgs (starred);
+CREATE INDEX msgs_index6 ON msgs (location_id);
+CREATE INDEX msgs_index7 ON msgs (state, hidden, chat_id);
+INSERT INTO msgs (id,msgrmsg,txt) VALUES
+(1,0,'marker1'), (2,0,'rsvd'), (3,0,'rsvd'),
+(4,0,'rsvd'), (5,0,'rsvd'), (6,0,'rsvd'), (7,0,'rsvd'),
+(8,0,'rsvd'), (9,0,'daymarker');
+
+CREATE TABLE jobs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    added_timestamp INTEGER,
+    desired_timestamp INTEGER DEFAULT 0,
+    action INTEGER,
+    foreign_id INTEGER,
+    param TEXT DEFAULT '',
+    thread INTEGER DEFAULT 0,
+    tries INTEGER DEFAULT 0
+);
+CREATE INDEX jobs_index1 ON jobs (desired_timestamp);
+
+CREATE TABLE leftgrps (
+    id INTEGER PRIMARY KEY,
+    grpid TEXT DEFAULT ''
+);
+CREATE INDEX leftgrps_index1 ON leftgrps (grpid);
+
+CREATE TABLE keypairs (
+    id INTEGER PRIMARY KEY,
+    addr TEXT DEFAULT '' COLLATE NOCASE,
+    is_default INTEGER DEFAULT 0,
+    private_key,
+    public_key,
+    created INTEGER DEFAULT 0
+);
+
+CREATE TABLE acpeerstates (
+    id INTEGER PRIMARY KEY,
+    addr TEXT DEFAULT '' COLLATE NOCASE,
+    last_seen INTEGER DEFAULT 0,
+    last_seen_autocrypt INTEGER DEFAULT 0,
+    public_key,
+    prefer_encrypted INTEGER DEFAULT 0,
+    gossip_timestamp INTEGER DEFAULT 0,
+    gossip_key,
+    public_key_fingerprint TEXT DEFAULT '',
+    gossip_key_fingerprint TEXT DEFAULT '',
+    verified_key,
+    verified_key_fingerprint TEXT DEFAULT ''
+);
+CREATE INDEX acpeerstates_index1 ON acpeerstates (addr);
+CREATE INDEX acpeerstates_index3 ON acpeerstates (public_key_fingerprint);
+CREATE INDEX acpeerstates_index4 ON acpeerstates (gossip_key_fingerprint);
+CREATE INDEX acpeerstates_index5 ON acpeerstates (verified_key_fingerprint);
+
+CREATE TABLE msgs_mdns (
+    msg_id INTEGER,
+    contact_id INTEGER,
+    timestamp_sent INTEGER DEFAULT 0
+);
+CREATE INDEX msgs_mdns_index1 ON msgs_mdns (msg_id);
+
+CREATE TABLE tokens (
+    id INTEGER PRIMARY KEY,
+    namespc INTEGER DEFAULT 0,
+    foreign_id INTEGER DEFAULT 0,
+    token TEXT DEFAULT '',
+    timestamp INTEGER DEFAULT 0
+);
+
+CREATE TABLE locations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    latitude REAL DEFAULT 0.0,
+    longitude REAL DEFAULT 0.0,
+    accuracy REAL DEFAULT 0.0,
+    timestamp INTEGER DEFAULT 0,
+    chat_id INTEGER DEFAULT 0,
+    from_id INTEGER DEFAULT 0,
+    independent INTEGER DEFAULT 0
+);
+CREATE INDEX locations_index1 ON locations (from_id);
+CREATE INDEX locations_index2 ON locations (timestamp);
+
+CREATE TABLE devmsglabels (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    label TEXT,
+    msg_id INTEGER DEFAULT 0
+);
+CREATE INDEX devmsglabels_index1 ON devmsglabels (label);
+"#,
+                )?;
+                tx.commit()?;
+                Ok(())
+            })
             .await?;
-            sql.execute(
-                "CREATE INDEX config_index1 ON config (keyname);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE contacts (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT, \
-                 name TEXT DEFAULT '', \
-                 addr TEXT DEFAULT '' COLLATE NOCASE, \
-                 origin INTEGER DEFAULT 0, \
-                 blocked INTEGER DEFAULT 0, \
-                 last_seen INTEGER DEFAULT 0, \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX contacts_index1 ON contacts (name COLLATE NOCASE);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX contacts_index2 ON contacts (addr COLLATE NOCASE);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "INSERT INTO contacts (id,name,origin) VALUES \
-                 (1,'self',262144), (2,'info',262144), (3,'rsvd',262144), \
-                 (4,'rsvd',262144), (5,'device',262144), (6,'rsvd',262144), \
-                 (7,'rsvd',262144), (8,'rsvd',262144), (9,'rsvd',262144);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE chats (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT,  \
-                 type INTEGER DEFAULT 0, \
-                 name TEXT DEFAULT '', \
-                 draft_timestamp INTEGER DEFAULT 0, \
-                 draft_txt TEXT DEFAULT '', \
-                 blocked INTEGER DEFAULT 0, \
-                 grpid TEXT DEFAULT '', \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute("CREATE INDEX chats_index1 ON chats (grpid);", paramsv![])
+
+            sql.set_raw_config_int(context, "dbversion", dbversion_before_update)
                 .await?;
-            sql.execute(
-                "CREATE TABLE chats_contacts (chat_id INTEGER, contact_id INTEGER);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX chats_contacts_index1 ON chats_contacts (chat_id);",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "INSERT INTO chats (id,type,name) VALUES \
-                 (1,120,'deaddrop'), (2,120,'rsvd'), (3,120,'trash'), \
-                 (4,120,'msgs_in_creation'), (5,120,'starred'), (6,120,'archivedlink'), \
-                 (7,100,'rsvd'), (8,100,'rsvd'), (9,100,'rsvd');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE msgs (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT, \
-                 rfc724_mid TEXT DEFAULT '', \
-                 server_folder TEXT DEFAULT '', \
-                 server_uid INTEGER DEFAULT 0, \
-                 chat_id INTEGER DEFAULT 0, \
-                 from_id INTEGER DEFAULT 0, \
-                 to_id INTEGER DEFAULT 0, \
-                 timestamp INTEGER DEFAULT 0, \
-                 type INTEGER DEFAULT 0, \
-                 state INTEGER DEFAULT 0, \
-                 msgrmsg INTEGER DEFAULT 1, \
-                 bytes INTEGER DEFAULT 0, \
-                 txt TEXT DEFAULT '', \
-                 txt_raw TEXT DEFAULT '', \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute("CREATE INDEX msgs_index1 ON msgs (rfc724_mid);", paramsv![])
-                .await?;
-            sql.execute("CREATE INDEX msgs_index2 ON msgs (chat_id);", paramsv![])
-                .await?;
-            sql.execute("CREATE INDEX msgs_index3 ON msgs (timestamp);", paramsv![])
-                .await?;
-            sql.execute("CREATE INDEX msgs_index4 ON msgs (state);", paramsv![])
-                .await?;
-            sql.execute(
-                "INSERT INTO msgs (id,msgrmsg,txt) VALUES \
-                 (1,0,'marker1'), (2,0,'rsvd'), (3,0,'rsvd'), \
-                 (4,0,'rsvd'), (5,0,'rsvd'), (6,0,'rsvd'), (7,0,'rsvd'), \
-                 (8,0,'rsvd'), (9,0,'daymarker');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE TABLE jobs (\
-                 id INTEGER PRIMARY KEY AUTOINCREMENT, \
-                 added_timestamp INTEGER, \
-                 desired_timestamp INTEGER DEFAULT 0, \
-                 action INTEGER, \
-                 foreign_id INTEGER, \
-                 param TEXT DEFAULT '');",
-                paramsv![],
-            )
-            .await?;
-            sql.execute(
-                "CREATE INDEX jobs_index1 ON jobs (desired_timestamp);",
-                paramsv![],
-            )
-            .await?;
-            if !sql.table_exists("config").await?
-                || !sql.table_exists("contacts").await?
-                || !sql.table_exists("chats").await?
-                || !sql.table_exists("chats_contacts").await?
-                || !sql.table_exists("msgs").await?
-                || !sql.table_exists("jobs").await?
-            {
-                error!(
-                    context,
-                    "Cannot create tables in new database \"{:?}\".",
-                    dbfile.as_ref(),
-                );
-                // cannot create the tables - maybe we cannot write?
-                return Err(Error::SqlFailedToOpen.into());
-            } else {
-                sql.set_raw_config_int(context, "dbversion", 0).await?;
-            }
         } else {
             exists_before_update = true;
             dbversion_before_update = sql
@@ -858,7 +925,7 @@ async fn open(
 
         let mut dbversion = dbversion_before_update;
         let mut recalc_fingerprints = false;
-        let mut update_icons = false;
+        let mut update_icons = !exists_before_update;
 
         if dbversion < 1 {
             info!(context, "[migration] v1");
@@ -872,7 +939,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 1;
             sql.set_raw_config_int(context, "dbversion", 1).await?;
         }
         if dbversion < 2 {
@@ -882,7 +948,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 2;
             sql.set_raw_config_int(context, "dbversion", 2).await?;
         }
         if dbversion < 7 {
@@ -898,7 +963,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 7;
             sql.set_raw_config_int(context, "dbversion", 7).await?;
         }
         if dbversion < 10 {
@@ -919,7 +983,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 10;
             sql.set_raw_config_int(context, "dbversion", 10).await?;
         }
         if dbversion < 12 {
@@ -934,7 +997,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 12;
             sql.set_raw_config_int(context, "dbversion", 12).await?;
         }
         if dbversion < 17 {
@@ -946,6 +1008,8 @@ async fn open(
             .await?;
             sql.execute("CREATE INDEX chats_index2 ON chats (archived);", paramsv![])
                 .await?;
+            // 'starred' column is not used currently
+            // (dropping is not easily doable and stop adding it will make reusing it complicated)
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN starred INTEGER DEFAULT 0;",
                 paramsv![],
@@ -953,7 +1017,6 @@ async fn open(
             .await?;
             sql.execute("CREATE INDEX msgs_index5 ON msgs (starred);", paramsv![])
                 .await?;
-            dbversion = 17;
             sql.set_raw_config_int(context, "dbversion", 17).await?;
         }
         if dbversion < 18 {
@@ -968,7 +1031,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 18;
             sql.set_raw_config_int(context, "dbversion", 18).await?;
         }
         if dbversion < 27 {
@@ -992,7 +1054,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 27;
             sql.set_raw_config_int(context, "dbversion", 27).await?;
         }
         if dbversion < 34 {
@@ -1028,7 +1089,6 @@ async fn open(
             )
             .await?;
             recalc_fingerprints = true;
-            dbversion = 34;
             sql.set_raw_config_int(context, "dbversion", 34).await?;
         }
         if dbversion < 39 {
@@ -1052,7 +1112,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 39;
             sql.set_raw_config_int(context, "dbversion", 39).await?;
         }
         if dbversion < 40 {
@@ -1062,14 +1121,12 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 40;
             sql.set_raw_config_int(context, "dbversion", 40).await?;
         }
         if dbversion < 44 {
             info!(context, "[migration] v44");
             sql.execute("ALTER TABLE msgs ADD COLUMN mime_headers TEXT;", paramsv![])
                 .await?;
-            dbversion = 44;
             sql.set_raw_config_int(context, "dbversion", 44).await?;
         }
         if dbversion < 46 {
@@ -1094,7 +1151,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 47;
             sql.set_raw_config_int(context, "dbversion", 47).await?;
         }
         if dbversion < 48 {
@@ -1105,8 +1161,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-
-            dbversion = 48;
             sql.set_raw_config_int(context, "dbversion", 48).await?;
         }
         if dbversion < 49 {
@@ -1116,7 +1170,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 49;
             sql.set_raw_config_int(context, "dbversion", 49).await?;
         }
         if dbversion < 50 {
@@ -1128,7 +1181,6 @@ async fn open(
                 sql.set_raw_config_int(context, "show_emails", ShowEmails::All as i32)
                     .await?;
             }
-            dbversion = 50;
             sql.set_raw_config_int(context, "dbversion", 50).await?;
         }
         if dbversion < 53 {
@@ -1169,7 +1221,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 53;
             sql.set_raw_config_int(context, "dbversion", 53).await?;
         }
         if dbversion < 54 {
@@ -1184,7 +1235,6 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            dbversion = 54;
             sql.set_raw_config_int(context, "dbversion", 54).await?;
         }
         if dbversion < 55 {
@@ -1264,18 +1314,11 @@ async fn open(
                 paramsv![],
             )
             .await?;
-            // Timer value in seconds. For incoming messages this
-            // timer starts when message is read, so we want to have
-            // the value stored here until the timer starts.
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN ephemeral_timer INTEGER DEFAULT 0",
                 paramsv![],
             )
             .await?;
-            // Timestamp indicating when the message should be
-            // deleted. It is convenient to store it here because UI
-            // needs this value to display how much time is left until
-            // the message is deleted.
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0",
                 paramsv![],
@@ -1317,7 +1360,7 @@ async fn open(
         }
         if dbversion < 68 {
             info!(context, "[migration] v68");
-            // the index is used to speed up get_fresh_msg_cnt(), see comment there for more details
+            // the index is used to speed up get_fresh_msg_cnt() (see comment there for more details) and marknoticed_chat()
             sql.execute(
                 "CREATE INDEX IF NOT EXISTS msgs_index7 ON msgs (state, hidden, chat_id);",
                 paramsv![],
@@ -1325,6 +1368,20 @@ async fn open(
             .await?;
             sql.set_raw_config_int(context, "dbversion", 68).await?;
         }
+        if dbversion < 69 {
+            info!(context, "[migration] v69");
+            sql.execute(
+                "ALTER TABLE chats ADD COLUMN protected INTEGER DEFAULT 0;",
+                paramsv![],
+            )
+            .await?;
+            sql.execute(
+                "UPDATE chats SET protected=1, type=120 WHERE type=130;", // 120=group, 130=old verified group
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 69).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)

src/stock.rs

@@ -5,8 +5,8 @@ use std::borrow::Cow;
 use strum::EnumProperty;
 use strum_macros::EnumProperty;
 
-use crate::blob::BlobObject;
 use crate::chat;
+use crate::chat::ProtectionStatus;
 use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::contact::*;
 use crate::context::Context;
@@ -14,6 +14,7 @@ use crate::error::{bail, Error};
 use crate::message::Message;
 use crate::param::Param;
 use crate::stock::StockMessage::{DeviceMessagesHint, WelcomeMessage};
+use crate::{blob::BlobObject, config::Config};
 
 /// Stock strings
 ///
@@ -119,9 +120,6 @@ pub enum StockMessage {
     #[strum(props(fallback = "Archived chats"))]
     ArchivedChats = 40,
 
-    #[strum(props(fallback = "Starred messages"))]
-    StarredMsgs = 41,
-
     #[strum(props(fallback = "Autocrypt Setup Message"))]
     AcSetupMsgSubject = 42,
 
@@ -217,8 +215,35 @@ pub enum StockMessage {
     #[strum(props(fallback = "You are invited to a video chat, click %1$s to join."))]
     VideochatInviteMsgBody = 83,
 
-    #[strum(props(fallback = "Configuration failed. Error: “%1$s”"))]
+    #[strum(props(fallback = "Error:\n\n“%1$s”"))]
     ConfigurationFailed = 84,
+
+    #[strum(props(
+        fallback = "⚠️ Date or time of your device seem to be inaccurate (%1$s).\n\n\
+                    Adjust your clock ⏰🔧 to ensure your messages are received correctly."
+    ))]
+    BadTimeMsgBody = 85,
+
+    #[strum(props(fallback = "⚠️ Your Delta Chat version might be outdated.\n\n\
+                    This may cause problems because your chat partners use newer versions - \
+                    and you are missing the latest features 😳\n\
+                    Please check https://get.delta.chat or your app store for updates."))]
+    UpdateReminderMsgBody = 86,
+
+    #[strum(props(
+        fallback = "Could not find your mail server.\n\nPlease check your internet connection."
+    ))]
+    ErrorNoNetwork = 87,
+
+    #[strum(props(fallback = "Chat protection enabled."))]
+    ProtectionEnabled = 88,
+
+    #[strum(props(fallback = "Chat protection disabled."))]
+    ProtectionDisabled = 89,
+
+    // used in summaries, a noun, not a verb (not: "to reply")
+    #[strum(props(fallback = "Reply"))]
+    ReplyNoun = 90,
 }
 
 /*
@@ -384,16 +409,27 @@ impl Context {
         }
     }
 
-    pub async fn update_device_chats(&self) -> Result<(), Error> {
-        // check for the LAST added device message - if it is present, we can skip message creation.
-        // this is worthwhile as this function is typically called
-        // by the ui on every probram start or even on every opening of the chatlist.
-        if chat::was_device_msg_ever_added(&self, "core-welcome").await? {
+    /// Returns a stock message saying that protection status has changed.
+    pub async fn stock_protection_msg(&self, protect: ProtectionStatus, from_id: u32) -> String {
+        self.stock_system_msg(
+            match protect {
+                ProtectionStatus::Protected => StockMessage::ProtectionEnabled,
+                ProtectionStatus::Unprotected => StockMessage::ProtectionDisabled,
+            },
+            "",
+            "",
+            from_id,
+        )
+        .await
+    }
+
+    pub(crate) async fn update_device_chats(&self) -> Result<(), Error> {
+        if self.get_config_bool(Config::Bot).await {
             return Ok(());
         }
 
-        // create saved-messages chat;
-        // we do this only once, if the user has deleted the chat, he can recreate it manually.
+        // create saved-messages chat; we do this only once, if the user has deleted the chat,
+        // he can recreate it manually (make sure we do not re-add it when configure() was called a second time)
         if !self.sql.get_raw_config_bool(&self, "self-chat-added").await {
             self.sql
                 .set_raw_config_bool(&self, "self-chat-added", true)

src/test_utils.rs

@@ -9,13 +9,15 @@ use async_std::path::PathBuf;
 use async_std::sync::RwLock;
 use tempfile::{tempdir, TempDir};
 
-use crate::chat::ChatId;
+use crate::chat;
+use crate::chat::{ChatId, ChatItem};
 use crate::config::Config;
 use crate::context::Context;
 use crate::dc_receive_imf::dc_receive_imf;
 use crate::dc_tools::EmailAddress;
 use crate::job::Action;
 use crate::key::{self, DcKey};
+use crate::message::Message;
 use crate::mimeparser::MimeMessage;
 use crate::param::{Param, Params};
 
@@ -187,6 +189,19 @@ impl TestContext {
             .await
             .unwrap();
     }
+
+    /// Get the most recent message of a chat.
+    ///
+    /// Panics on errors or if the most recent message is a marker.
+    pub async fn get_last_msg(&self, chat_id: ChatId) -> Message {
+        let msgs = chat::get_chat_msgs(&self.ctx, chat_id, 0, None).await;
+        let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() {
+            msg_id
+        } else {
+            panic!("Wrong item type");
+        };
+        Message::load_from_db(&self.ctx, *msg_id).await.unwrap()
+    }
 }
 
 /// A raw message as it was scheduled to be sent.

tests/stress.rs

@@ -1,110 +0,0 @@
-//! Stress some functions for testing; if used as a lib, this file is obsolete.
-
-use deltachat::config;
-use deltachat::context::*;
-use tempfile::tempdir;
-
-/* some data used for testing
- ******************************************************************************/
-
-async fn stress_functions(context: &Context) {
-    let res = context
-        .get_config(config::Config::SysConfigKeys)
-        .await
-        .unwrap();
-
-    assert!(!res.contains(" probably_never_a_key "));
-    assert!(res.contains(" addr "));
-    assert!(res.contains(" mail_server "));
-    assert!(res.contains(" mail_user "));
-    assert!(res.contains(" mail_pw "));
-    assert!(res.contains(" mail_port "));
-    assert!(res.contains(" send_server "));
-    assert!(res.contains(" send_user "));
-    assert!(res.contains(" send_pw "));
-    assert!(res.contains(" send_port "));
-    assert!(res.contains(" server_flags "));
-    assert!(res.contains(" imap_folder "));
-    assert!(res.contains(" displayname "));
-    assert!(res.contains(" selfstatus "));
-    assert!(res.contains(" selfavatar "));
-    assert!(res.contains(" e2ee_enabled "));
-    assert!(res.contains(" mdns_enabled "));
-    assert!(res.contains(" save_mime_headers "));
-    assert!(res.contains(" configured_addr "));
-    assert!(res.contains(" configured_mail_server "));
-    assert!(res.contains(" configured_mail_user "));
-    assert!(res.contains(" configured_mail_pw "));
-    assert!(res.contains(" configured_mail_port "));
-    assert!(res.contains(" configured_send_server "));
-    assert!(res.contains(" configured_send_user "));
-    assert!(res.contains(" configured_send_pw "));
-    assert!(res.contains(" configured_send_port "));
-    assert!(res.contains(" configured_server_flags "));
-
-    // Cant check, no configured context
-    // assert!(dc_is_configured(context) != 0, "Missing configured context");
-
-    // let setupcode = dc_create_setup_code(context);
-    // let setupcode_c = CString::new(setupcode.clone()).unwrap();
-    // let setupfile = dc_render_setup_file(context, &setupcode).unwrap();
-    // let setupfile_c = CString::new(setupfile).unwrap();
-    // let mut headerline_2: *const libc::c_char = ptr::null();
-    // let payload = dc_decrypt_setup_file(context, setupcode_c.as_ptr(), setupfile_c.as_ptr());
-
-    // assert!(payload.is_null());
-    // assert!(!dc_split_armored_data(
-    //     payload,
-    //     &mut headerline_2,
-    //     ptr::null_mut(),
-    //     ptr::null_mut(),
-    //     ptr::null_mut(),
-    // ));
-    // assert!(!headerline_2.is_null());
-    // assert_eq!(
-    //     strcmp(
-    //         headerline_2,
-    //         b"-----BEGIN PGP PRIVATE KEY BLOCK-----\x00" as *const u8 as *const libc::c_char,
-    //     ),
-    //     0
-    // );
-    // free(payload as *mut libc::c_void);
-
-    // Cant check, no configured context
-    // assert!(dc_is_configured(context) != 0, "missing configured context");
-
-    // let qr = dc_get_securejoin_qr(context, 0);
-    // assert!(!qr.is_null(), "Invalid qr code generated");
-    // let qr_r = as_str(qr);
-
-    // assert!(qr_r.len() > 55);
-    // assert!(qr_r.starts_with("OPENPGP4FPR:"));
-
-    // let res = dc_check_qr(context, qr);
-    // let s = res.get_state();
-
-    // assert!(
-    //     s == QrState::AskVerifyContact
-    //         || s == QrState::FprMissmatch
-    //         || s == QrState::FprWithoutAddr
-    // );
-
-    // free(qr.cast());
-}
-
-async fn create_test_context() -> Context {
-    use rand::Rng;
-
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = rand::thread_rng().gen();
-    Context::new("FakeOs".into(), dbfile.into(), id)
-        .await
-        .unwrap()
-}
-
-#[async_std::test]
-async fn test_stress_tests() {
-    let context = create_test_context().await;
-    stress_functions(&context).await;
-}