Merge remote-tracking branch 'origin/master' into feat/async-jobs

.circleci/config.yml

@@ -7,6 +7,10 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
+  python:
+    docker:
+      - image: 3.7.7-stretch
+            
 
 restore-workspace: &restore-workspace
   attach_workspace:

CHANGELOG.md

@@ -1,5 +1,73 @@
 # Changelog 
 
+## 1.32.0
+
+- fix endless loop when trying to download messages with bad RFC Message-ID,
+  also be more reliable on similar errors #1463 #1466 #1462
+
+- fix bug with comma in contact request #1438
+
+- do not refer to hidden messages on replies #1459
+
+- improve error handling #1468 #1465 #1464
+
+
+## 1.31.0
+
+- always describe the context of the displayed error #1451
+
+- do not emit `DC_EVENT_ERROR` when message sending fails;
+  `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451
+
+- new config-option `media_quality` #1449
+
+- try over if writing message to database fails #1447
+
+
+## 1.30.0
+
+- expunge deleted messages #1440
+
+- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439
+
+
+## 1.29.0
+
+- new config options `delete_device_after` and `delete_server_after`,
+  each taking an amount of seconds after which messages
+  are deleted from the device and/or the server #1310 #1335 #1411 #1417 #1423
+
+- new api `dc_estimate_deletion_cnt()` to estimate the effect
+  of `delete_device_after` and `delete_server_after`
+
+- use Ed25519 keys by default, these keys are much shorter
+  than RSA keys, which results in saving traffic and speed improvements #1362
+
+- improve message ellipsizing #1397 #1430
+
+- emit `DC_EVENT_ERROR_NETWORK` also on smtp-errors #1378
+
+- do not show badly formatted non-delta-messages as empty #1384
+
+- try over SMTP on potentially recoverable error 5.5.0 #1379
+
+- remove device-chat from forward-to-chat-list #1367
+
+- improve group-handling #1368
+
+- `dc_get_info()` returns uptime (how long the context is in use)
+
+- python improvements and adaptions #1408 #1415
+
+- log to the stdout and stderr in tests #1416
+
+- refactoring, code improvements #1363 #1365 #1366 #1370 #1375 #1389 #1390 #1418 #1419
+
+- removed api: `dc_chat_get_subtitle()`, `dc_get_version_str()`, `dc_array_add_id()`
+
+- removed events: `DC_EVENT_MEMBER_ADDED`, `DC_EVENT_MEMBER_REMOVED`
+
+
 ## 1.28.0
 
 - new flag DC_GCL_FOR_FORWARDING for dc_get_chatlist()

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.28.0"
+version = "1.32.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -25,15 +25,13 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = { git = "https://github.com/async-email/async-imap", branch = "feat/send" }
 async-native-tls = "0.3.1"
-async-std = { git = "https://github.com/async-rs/async-std", version = "1.5", features = ["unstable"] }
+async-std = { git = "https://github.com/async-rs/async-std", rev = "3ff9e98f20a193eb63e43fb9d71f9d60c33f6d58", features = ["unstable"] }
 base64 = "0.11"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
-failure = "0.1.5"
-failure_derive = "0.1.5"
 indexmap = "1.3.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
@@ -59,19 +57,21 @@ native-tls = "0.2.3"
 image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
 futures = "0.3.4"
 crossbeam-queue = "0.2.1"
-
+thiserror = "1.0.14"
+anyhow = "1.0.28"
 
 pretty_env_logger = { version = "0.3.1", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
+async-trait = "0.1.31"
 
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
 pretty_env_logger = "0.3.0"
 proptest = "0.9.4"
-async-std = { git = "https://github.com/async-rs/async-std", version = "1.5", features = ["unstable", "attributes"] }
+async-std = { git = "https://github.com/async-rs/async-std", rev = "3ff9e98f20a193eb63e43fb9d71f9d60c33f6d58", features = ["unstable", "attributes"] }
 
 [workspace]
 members = [

README.md

@@ -17,8 +17,9 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=info cargo run --example repl --features repl -- /path/to/db
+$ RUST_LOG=info cargo run --example repl --features repl -- ~/deltachat-db
 ```
+where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Configure your account (if not already configured):
 

ci_scripts/ci_upload.sh

@@ -37,7 +37,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
-pip3 install -U pip
+pip3 install -U pip setuptools
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN

ci_scripts/cleanup_devpi_indices.py

@@ -48,7 +48,7 @@ def run():
         projectnames = get_projectnames(baseurl, username, indexname)
         if indexname == "master" or not indexname:
             continue
-        assert projectnames == ["deltachat"]
+        clear_index = not projectnames
         for projectname in projectnames:
             dates = get_release_dates(baseurl, username, indexname, projectname)
             if not dates:
@@ -60,8 +60,11 @@ def run():
                 date = datetime.datetime(*max(dates))
             if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
                 assert username and indexname
-                url = baseurl + username + "/" + indexname
+                clear_index = True
-                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+                break
+        if clear_index:
+            url = baseurl + username + "/" + indexname
+            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
 
 
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.28.0"
+version = "1.32.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -19,9 +19,10 @@ deltachat = { path = "../", default-features = false }
 libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
-failure = "0.1.6"
 serde_json = "1.0"
 async-std = "1.5.0" 
+anyhow = "1.0.28"
+thiserror = "1.0.14"
 
 [features]
 default = ["vendored", "nightly"]

deltachat-ffi/deltachat.h

@@ -42,7 +42,7 @@ typedef struct _dc_event    dc_event_t;
  * uintptr_t event_handler_func(dc_context_t* context, int event,
  *                              uintptr_t data1, uintptr_t data2)
  * {
- *     return 0; // for unhandled events, it is always safe to return 0
+ *     return 0;
  * }
  *
  * dc_context_t* context = dc_context_new(event_handler_func, NULL, NULL);
@@ -236,7 +236,7 @@ void          dc_event_unref   (dc_event_t* event);
  *       otherwise!
  *     - The callback SHOULD return _fast_, for GUI updates etc. you should
  *       post yourself an asynchronous message to your GUI thread, if needed.
- *     - If not mentioned otherweise, the callback should return 0.
+ *     - events do not expect a return value, just always return 0.
  * @param userdata can be used by the client for any purpuse.  He finds it
  *     later in dc_get_userdata().
  * @param os_name is only for decorative use
@@ -349,7 +349,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `displayname`  = Own name to use when sending messages.  MUAs are allowed to spread this way eg. using CC, defaults to empty
  * - `selfstatus`   = Own status to display eg. in email footers, defaults to a standard text
- * - `selfavatar`   = File containing avatar. Will be copied to blob directory.
+ * - `selfavatar`   = File containing avatar. Will immediately be copied to the 
+ *                    `blobdir`; the original image will not be needed anymore.
  *                    NULL to remove the avatar.
  *                    It is planned for future versions
  *                    to send this image together with the next messages.
@@ -384,10 +385,20 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    >=1=seconds, after which messages are deleted automatically from the device.
  *                    Messages in the "saved messages" chat (see dc_chat_is_self_talk()) are skipped.
  *                    Messages are deleted whether they were seen or not, the UI should clearly point that out.
+ *                    See also dc_estimate_deletion_cnt().
  * - `delete_server_after` = 0=do not delete messages from server automatically (default),
  *                    >=1=seconds, after which messages are deleted automatically from the server.
  *                    "Saved messages" are deleted from the server as well as
  *                    emails matching the `show_emails` settings above, the UI should clearly point that out.
+ *                    See also dc_estimate_deletion_cnt().
+ * - `media_quality` = DC_MEDIA_QUALITY_BALANCED (0) =
+ *                    good outgoing images/videos/voice quality at reasonable sizes (default)
+ *                    DC_MEDIA_QUALITY_WORSE (1)
+ *                    allow worse images/videos/voice quality to gain smaller sizes,
+ *                    suitable for providers or areas known to have a bad connection.
+ *                    In contrast to other options, the implementation of this option is currently up to the UIs;
+ *                    this may change in future, however,
+ *                    having the option in the core allows provider-specific-defaults already today.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -665,8 +676,10 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
  *       if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and
  *       the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are _any_ archived
  *       chats
- *     - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist,
+ *     - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
+ *       and hides the "Device chat" and the deaddrop.
  *       typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
+ *       to also hide the archive link.
  *     - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
  *       to the list (may be used eg. for selecting chats on forwarding, the flag is
  *       not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -1026,6 +1039,7 @@ int             dc_get_fresh_msg_cnt         (dc_context_t* context, uint32_t ch
  * by the dc_set_config()-options `delete_device_after` or `delete_server_after`.
  * This is typically used to show the estimated impact to the user before actually enabling ephemeral messages.
  *
+ * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param from_server 1=Estimate deletion count for server, 0=Estimate deletion count for device
  * @param seconds Count messages older than the given number of seconds.
@@ -1329,8 +1343,10 @@ int             dc_set_chat_name             (dc_context_t* context, uint32_t ch
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
  * @param chat_id The chat ID to set the image for.
- * @param image Full path of the image to use as the group image.  If you pass NULL here,
+ * @param image Full path of the image to use as the group image. The image will immediately be copied to the 
- *     the group image is deleted (for promoted groups, all members are informed about this change anyway).
+ *     `blobdir`; the original image will not be needed anymore.
+ *      If you pass NULL here, the group image is deleted (for promoted groups, all members are informed about 
+ *      this change anyway).
  * @return 1=success, 0=error
  */
 int             dc_set_chat_profile_image    (dc_context_t* context, uint32_t chat_id, const char* image);
@@ -1503,7 +1519,7 @@ int             dc_may_be_valid_addr         (const char* addr);
 
 /**
  * Check if an e-mail address belongs to a known and unblocked contact.
- * Known and unblocked contacts will be returned by dc_get_contacts().
+ * To get a list of all known and unblocked contacts, use dc_get_contacts().
  *
  * To validate an e-mail address independently of the contact database
  * use dc_may_be_valid_addr().
@@ -1511,7 +1527,8 @@ int             dc_may_be_valid_addr         (const char* addr);
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @param addr The e-mail-address to check.
- * @return 1=address is a contact in use, 0=address is not a contact in use.
+ * @return Contact ID of the contact belonging to the e-mail-address
+ *     or 0 if there is no contact that is or was introduced by an accepted contact.  
  */
 uint32_t        dc_lookup_contact_id_by_addr (dc_context_t* context, const char* addr);
 
@@ -2543,19 +2560,6 @@ int             dc_chat_get_type             (const dc_chat_t* chat);
 char*           dc_chat_get_name             (const dc_chat_t* chat);
 
 
-/*
- * Get a subtitle for a chat.  The subtitle is eg. the email-address or the
- * number of group members.
- *
- * Deprecated function. Subtitles should be created in the ui
- * where plural forms and other specials can be handled more gracefully.
- *
- * @param chat The chat object to calulate the subtitle for.
- * @return Subtitle as a string. Must be released using dc_str_unref() after usage. Never NULL.
- */
-char*           dc_chat_get_subtitle         (const dc_chat_t* chat);
-
-
 /**
  * Get the chat's profile image.
  * For groups, this is the image set by any group member
@@ -3876,8 +3880,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * These constants are used as events
  * reported to the callback given to dc_context_new().
- * If you do not want to handle an event, it is always safe to return 0,
- * so there is no need to add a "case" for every event.
  *
  * @addtogroup DC_EVENT
  * @{
@@ -3892,7 +3894,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_INFO                     100
 
@@ -3903,7 +3904,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_SMTP_CONNECTED           101
 
@@ -3914,7 +3914,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_IMAP_CONNECTED           102
 
@@ -3924,7 +3923,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_SMTP_MESSAGE_SENT        103
 
@@ -3934,7 +3932,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_IMAP_MESSAGE_DELETED   104
 
@@ -3944,7 +3941,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
@@ -3954,7 +3950,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) folder name.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_IMAP_FOLDER_EMPTIED  106
 
@@ -3964,7 +3959,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) path name
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_NEW_BLOB_FILE 150
 
@@ -3974,7 +3968,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) path name
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_DELETED_BLOB_FILE 151
 
@@ -3987,7 +3980,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Warning string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_WARNING                  300
 
@@ -4010,7 +4002,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     Some error strings are taken from dc_set_stock_translation(),
  *     however, most error strings will be in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_ERROR                    400
 
@@ -4034,7 +4025,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     0=subsequent network error, should be logged only
  * @param data2 (const char*) Error string, always set, never NULL.
  *     Must not be unref'd or modified and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_ERROR_NETWORK            401
 
@@ -4050,7 +4040,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified
  *     and is valid only until the callback returns.
- * @return 0
  */
 #define DC_EVENT_ERROR_SELF_NOT_IN_GROUP  410
 
@@ -4064,7 +4053,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id for single added messages
  * @param data2 (int) msg_id for single added messages
- * @return 0
  */
 #define DC_EVENT_MSGS_CHANGED             2000
 
@@ -4077,7 +4065,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
- * @return 0
  */
 #define DC_EVENT_INCOMING_MSG             2005
 
@@ -4088,7 +4075,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
- * @return 0
  */
 #define DC_EVENT_MSG_DELIVERED            2010
 
@@ -4099,7 +4085,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
- * @return 0
  */
 #define DC_EVENT_MSG_FAILED               2012
 
@@ -4110,7 +4095,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
- * @return 0
  */
 #define DC_EVENT_MSG_READ                 2015
 
@@ -4123,7 +4107,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 0
- * @return 0
  */
 #define DC_EVENT_CHAT_MODIFIED            2020
 
@@ -4133,7 +4116,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) If not 0, this is the contact_id of an added contact that should be selected.
  * @param data2 0
- * @return 0
  */
 #define DC_EVENT_CONTACTS_CHANGED         2030
 
@@ -4146,7 +4128,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     If the locations of several contacts have been changed,
  *     eg. after calling dc_delete_all_locations(), this parameter is set to 0.
  * @param data2 0
- * @return 0
  */
 #define DC_EVENT_LOCATION_CHANGED         2035
 
@@ -4156,7 +4137,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done
  * @param data2 0
- * @return 0
  */
 #define DC_EVENT_CONFIGURE_PROGRESS       2041
 
@@ -4166,7 +4146,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done
  * @param data2 0
- * @return 0
  */
 #define DC_EVENT_IMEX_PROGRESS            2051
 
@@ -4181,7 +4160,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 (const char*) Path and file name.
  *     Must not be unref'd or modified and is valid only until the callback returns.
  * @param data2 0
- * @return 0
  */
 #define DC_EVENT_IMEX_FILE_WRITTEN        2052
 
@@ -4199,7 +4177,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified".
  *     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
  *     1000=Protocol finished for this contact.
- * @return 0
  */
 #define DC_EVENT_SECUREJOIN_INVITER_PROGRESS      2060
 
@@ -4215,21 +4192,9 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data2 (int) Progress as:
  *     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
  *     (Bob has verified alice and waits until Alice does the same for him)
- * @return 0
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
-
-/**
- * This event is sent out to the inviter when a joiner successfully joined a group.
- *
- * @param data1 (int) chat_id
- * @param data2 (int) contact_id
- * @return 0
- */
-#define DC_EVENT_SECUREJOIN_MEMBER_ADDED 2062
-
-
 /**
  * @}
  */
@@ -4245,8 +4210,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EVENT_DATA2_IS_STRING(e)  ((e)>=100 && (e)<=499)
 #define DC_EVENT_RETURNS_INT(e)      ((e)==DC_EVENT_IS_OFFLINE) // not used anymore
 #define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING) // not used anymore
-char*           dc_get_version_str           (void); // deprecated
-void            dc_array_add_id              (dc_array_t*, uint32_t); // deprecated
 #define dc_archive_chat(a,b,c)  dc_set_chat_visibility((a), (b), (c)? 1 : 0) // not used anymore
 #define dc_chat_get_archived(a) (dc_chat_get_visibility((a))==1? 1 : 0)      // not used anymore
 
@@ -4258,6 +4221,14 @@ void            dc_array_add_id              (dc_array_t*, uint32_t); // depreca
 #define DC_SHOW_EMAILS_ACCEPTED_CONTACTS 1
 #define DC_SHOW_EMAILS_ALL               2
 
+
+/*
+ * Values for dc_get|set_config("media_quality")
+ */
+#define DC_MEDIA_QUALITY_BALANCED 0
+#define DC_MEDIA_QUALITY_WORSE    1
+
+
 /*
  * Values for dc_get|set_config("key_gen_type")
  */
@@ -4376,8 +4347,6 @@ void            dc_array_add_id              (dc_array_t*, uint32_t); // depreca
 #define DC_STR_NOMESSAGES                 1
 #define DC_STR_SELF                       2
 #define DC_STR_DRAFT                      3
-#define DC_STR_MEMBER                     4
-#define DC_STR_CONTACT                    6
 #define DC_STR_VOICEMESSAGE               7
 #define DC_STR_DEADDROP                   8
 #define DC_STR_IMAGE                      9
@@ -4409,7 +4378,6 @@ void            dc_array_add_id              (dc_array_t*, uint32_t); // depreca
 #define DC_STR_STARREDMSGS                41
 #define DC_STR_AC_SETUP_MSG_SUBJECT       42
 #define DC_STR_AC_SETUP_MSG_BODY          43
-#define DC_STR_SELFTALK_SUBTITLE          50
 #define DC_STR_CANNOT_LOGIN               60
 #define DC_STR_SERVER_RESPONSE            61
 #define DC_STR_MSGACTIONBYUSER            62

deltachat-ffi/src/lib.rs

@@ -13,7 +13,7 @@ extern crate human_panic;
 extern crate num_traits;
 extern crate serde_json;
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::convert::TryInto;
 use std::ffi::CString;
 use std::fmt::Write;
@@ -22,13 +22,14 @@ use std::str::FromStr;
 use std::sync::{Arc, RwLock};
 use std::time::{Duration, SystemTime};
 
+use anyhow::anyhow;
 use async_std::task::block_on;
 use libc::uintptr_t;
 use num_traits::{FromPrimitive, ToPrimitive};
 
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
-use deltachat::contact::Contact;
+use deltachat::contact::{Contact, Origin};
 use deltachat::context::Context;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
@@ -177,7 +178,7 @@ macro_rules! try_inner_async {
                     let $name = ctx;
                     $block.await
                 }
-                None => Err(failure::err_msg("context not open")),
+                None => Err(anyhow!("context not open")),
             }
         })
     }};
@@ -416,7 +417,7 @@ pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c
 }
 
 fn render_info(
-    info: HashMap<&'static str, String>,
+    info: BTreeMap<&'static str, String>,
 ) -> std::result::Result<String, std::fmt::Error> {
     let mut res = String::new();
     for (key, value) in &info {
@@ -449,11 +450,6 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
     .unwrap_or_else(|_| ptr::null_mut())
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_get_version_str() -> *mut libc::c_char {
-    context::get_version_str().strdup()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_configure(context: *mut dc_context_t) {
     if context.is_null() {
@@ -656,14 +652,6 @@ unsafe fn translate_event(event: Event) -> *mut dc_event_t {
             data1: contact_id as uintptr_t,
             data2: progress as uintptr_t,
         },
-        Event::SecurejoinMemberAdded {
-            chat_id,
-            contact_id,
-        } => EventWrapper {
-            event_id,
-            data1: chat_id.to_u32() as uintptr_t,
-            data2: contact_id as uintptr_t,
-        },
     };
 
     Box::into_raw(Box::new(wrapper))
@@ -1072,12 +1060,12 @@ pub unsafe extern "C" fn dc_estimate_deletion_cnt(
         return 0;
     }
     let ffi_context = &*context;
-    ffi_context
+    with_inner_async!(ffi_context, ctx, async move {
-        .with_inner(|ctx| {
+        message::estimate_deletion_cnt(ctx, from_server != 0, seconds)
-            message::estimate_deletion_cnt(ctx, from_server != 0, seconds).unwrap_or(0)
+            .await
-                as libc::c_int
+            .unwrap_or(0) as libc::c_int
-        })
+    })
-        .unwrap_or(0)
+    .unwrap_or(0)
 }
 
 #[no_mangle]
@@ -1700,7 +1688,7 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     with_inner_async!(
         ffi_context,
         ctx,
-        Contact::lookup_id_by_addr(&ctx, to_string_lossy(addr))
+        Contact::lookup_id_by_addr(&ctx, to_string_lossy(addr), Origin::IncomingReplyTo)
     )
     .unwrap_or(0)
 }
@@ -2171,16 +2159,6 @@ pub unsafe extern "C" fn dc_array_unref(a: *mut dc_array::dc_array_t) {
     Box::from_raw(a);
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_array_add_id(array: *mut dc_array_t, item: libc::c_uint) {
-    if array.is_null() {
-        eprintln!("ignoring careless call to dc_array_add_id()");
-        return;
-    }
-
-    (*array).add_id(item);
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_cnt(array: *const dc_array_t) -> libc::size_t {
     if array.is_null() {
@@ -2511,20 +2489,6 @@ pub unsafe extern "C" fn dc_chat_get_name(chat: *mut dc_chat_t) -> *mut libc::c_
     ffi_chat.chat.get_name().strdup()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_chat_get_subtitle(chat: *mut dc_chat_t) -> *mut libc::c_char {
-    if chat.is_null() {
-        eprintln!("ignoring careless call to dc_chat_get_subtitle()");
-        return "".strdup();
-    }
-    let ffi_chat = &*chat;
-    let ffi_context: &ContextWrapper = &*ffi_chat.context;
-
-    with_inner_async!(ffi_context, ctx, ffi_chat.chat.get_subtitle(&ctx))
-        .map(|s| s.strdup())
-        .unwrap_or_else(|_| "".strdup())
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_chat_get_profile_image(chat: *mut dc_chat_t) -> *mut libc::c_char {
     if chat.is_null() {
@@ -2848,7 +2812,7 @@ pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_cha
             ffi_msg
                 .message
                 .get_file(ctx)
-                .map(|p| p.strdup())
+                .map(|p| p.to_string_lossy().strdup())
                 .unwrap_or_else(|| "".strdup())
         })
         .unwrap_or_else(|_| "".strdup())

deltachat-ffi/src/string.rs

@@ -1,4 +1,3 @@
-use failure::Fail;
 use std::ffi::{CStr, CString};
 use std::ptr;
 
@@ -31,13 +30,13 @@ unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
 }
 
 /// Error type for the [OsStrExt] trait
-#[derive(Debug, Fail, PartialEq)]
+#[derive(Debug, PartialEq, thiserror::Error)]
 pub(crate) enum CStringError {
     /// The string contains an interior null byte
-    #[fail(display = "String contains an interior null byte")]
+    #[error("String contains an interior null byte")]
     InteriorNullByte,
     /// The string is not valid Unicode
-    #[fail(display = "String is not valid unicode")]
+    #[error("String is not valid unicode")]
     NotUnicode,
 }
 

deltachat_derive/src/lib.rs

@@ -3,7 +3,6 @@ extern crate proc_macro;
 
 use crate::proc_macro::TokenStream;
 use quote::quote;
-use syn;
 
 // For now, assume (not check) that these macroses are applied to enum without
 // data.  If this assumption is violated, compiler error will point to

examples/repl/cmdline.rs

@@ -1,7 +1,7 @@
 use std::str::FromStr;
 
+use anyhow::{bail, ensure};
 use async_std::path::Path;
-
 use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
@@ -92,7 +92,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     });
 }
 
-async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), Error> {
+async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
     let data = dc_read_file(context, filename).await?;
 
     if let Err(err) = dc_receive_imf(context, &data, "import", 0, false).await {
@@ -292,11 +292,7 @@ fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub async fn cmdline(
+pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<(), Error> {
-    context: Context,
-    line: &str,
-    chat_id: &mut ChatId,
-) -> Result<(), failure::Error> {
     let mut sel_chat = if !chat_id.is_unset() {
         Chat::load_from_db(&context, *chat_id).await.ok()
     } else {

examples/repl/main.rs

@@ -7,13 +7,16 @@
 #[macro_use]
 extern crate deltachat;
 #[macro_use]
-extern crate failure;
+extern crate lazy_static;
+#[macro_use]
+extern crate rusqlite;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
 use std::process::Command;
 
 use ansi_term::Color;
+use anyhow::{bail, Error};
 use async_std::path::Path;
 use deltachat::chat::ChatId;
 use deltachat::config;
@@ -269,10 +272,10 @@ impl Highlighter for DcHelper {
 
 impl Helper for DcHelper {}
 
-async fn start(args: Vec<String>) -> Result<(), failure::Error> {
+async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
-        return Err(format_err!("No db-name specified"));
+        bail!("No db-name specified");
     }
     let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
 
@@ -353,7 +356,7 @@ async fn handle_cmd(
     line: &str,
     ctx: Context,
     selected_chat: &mut ChatId,
-) -> Result<ExitResult, failure::Error> {
+) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
@@ -417,7 +420,7 @@ async fn handle_cmd(
     Ok(ExitResult::Continue)
 }
 
-fn main() -> Result<(), failure::Error> {
+fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
     let args = std::env::args().collect();

python/CHANGELOG

@@ -1,3 +1,15 @@
+0.900.0 (DRAFT)
+---------------
+
+- refactored internals to use plugin-approach 
+
+- introduced PerAccount and Global hooks that plugins can implement 
+
+- introduced `ac_member_added()` and `ac_member_removed()` plugin events. 
+
+- introduced two documented examples for an echo and a group-membership
+  tracking plugin. 
+
 0.800.0
 -------
 

python/doc/api.rst

@@ -2,10 +2,6 @@
 high level API reference
 ========================
 
-.. note::
-
-    This API is work in progress and may change in versions prior to 1.0.
-
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
 - :class:`deltachat.contact.Contact`

python/doc/capi.rst

@@ -1,7 +0,0 @@
-
-C deltachat interface
-=====================
-
-See :doc:`lapi` for accessing many of the below functions
-through the ``deltachat.capi.lib`` namespace.
-

python/doc/examples.rst

@@ -1,37 +1,60 @@
 
-
 examples
 ========
 
-
-Playing around on the commandline
-----------------------------------
-
 Once you have :doc:`installed deltachat bindings <install>`
-you can start playing from the python interpreter commandline.
+you need email/password credentials for an IMAP/SMTP account.
-For example you can type ``python`` and then::
+Delta Chat developers and the CI system use a special URL to create
+temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
 
-    # instantiate and configure deltachat account
+Receiving a Chat message from the command line
-    import deltachat
+----------------------------------------------
-    ac = deltachat.Account("/tmp/db")
 
-    # start configuration activity and smtp/imap threads
+Here is a simple bot that:
-    ac.start_threads()
-    ac.configure(addr="test2@hq5.merlinux.eu", mail_pw="********")
 
-    # create a contact and send a message
+- receives a message and sends back ("echoes") a message
-    contact = ac.create_contact("someother@email.address")
-    chat = ac.create_chat_by_contact(contact)
-    chat.send_text("hi from the python interpreter command line")
 
-Checkout our :doc:`api` for the various high-level things you can do
+- terminates the bot if the message `/quit` is sent
-to send/receive messages, create contacts and chats.
 
+.. include:: ../examples/echo_and_quit.py
+    :literal:
 
-Looking at a real example
+With this file in your working directory you can run the bot
+by specifying a database path, an e-mail address and password of
+a SMTP-IMAP account::
+
+    $ cd examples
+    $ python echo_and_quit.py /tmp/db --email ADDRESS --password PASSWORD
+
+While this process is running you can start sending chat messages
+to `ADDRESS`.
+
+Track member additions and removals in a group
+----------------------------------------------
+
+Here is a simple bot that:
+
+- echoes messages sent to it
+
+- tracks if configuration completed
+
+- tracks member additions and removals for all chat groups
+
+.. include:: ../examples/group_tracking.py
+    :literal:
+
+With this file in your working directory you can run the bot
+by specifying a database path, an e-mail address and password of
+a SMTP-IMAP account::
+
+    python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db
+
+When this process is running you can start sending chat messages
+to `ADDRESS`.
+
+Writing bots for real
 -------------------------
 
 The `deltabot repository <https://github.com/deltachat/deltabot#deltachat-example-bot>`_
-contains a real-life example of Python bindings usage.
+contains a little framework for writing deltachat bots in Python.
-
 

python/doc/index.rst

@@ -4,8 +4,9 @@ deltachat python bindings
 The ``deltachat`` Python package provides two layers of bindings for the
 core Rust-library of the https://delta.chat messaging ecosystem:
 
-- :doc:`api` is a high level interface to deltachat-core which aims
+- :doc:`api` is a high level interface to deltachat-core.
-  to be memory safe and thoroughly tested through continous tox/pytest runs.
+
+- :doc:`plugins` is a brief introduction into implementing plugin hooks.
 
 - :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
   <https://github.com/deltachat/deltachat-core-rust>`_.
@@ -28,6 +29,7 @@ getting started
    changelog
    api
    lapi
+   plugins
 
 ..
     Indices and tables

python/doc/plugins.rst

@@ -0,0 +1,38 @@
+
+Implementing Plugin Hooks
+==========================
+
+The Delta Chat Python bindings use `pluggy <https://pluggy.readthedocs.io>`_
+for managing global and per-account plugin registration, and performing
+hook calls. There are two kinds of plugins:
+
+- Global plugins that are active for all accounts; they can implement
+  hooks at account-creation and account-shutdown time.
+
+- Account plugins that are only active during the lifetime of a
+  single Account instance.
+
+
+Registering a plugin
+--------------------
+
+.. autofunction:: deltachat.register_global_plugin
+    :noindex:
+
+.. automethod:: deltachat.account.Account.add_account_plugin
+    :noindex:
+
+
+Per-Account Hook specifications
+-------------------------------
+
+.. autoclass:: deltachat.hookspec.PerAccount
+    :members:
+
+
+Global Hook specifications
+--------------------------
+
+.. autoclass:: deltachat.hookspec.Global
+    :members:
+

python/examples/echo_and_quit.py

@@ -0,0 +1,30 @@
+
+# content of echo_and_quit.py
+
+from deltachat import account_hookimpl, run_cmdline
+
+
+class EchoPlugin:
+    @account_hookimpl
+    def ac_incoming_message(self, message):
+        print("process_incoming message", message)
+        if message.text.strip() == "/quit":
+            message.account.shutdown()
+        else:
+            # unconditionally accept the chat
+            message.accept_sender_contact()
+            addr = message.get_sender_contact().addr
+            text = message.text
+            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+
+    @account_hookimpl
+    def ac_message_delivered(self, message):
+        print("ac_message_delivered", message)
+
+
+def main(argv=None):
+    run_cmdline(argv=argv, account_plugins=[EchoPlugin()])
+
+
+if __name__ == "__main__":
+    main()

python/examples/group_tracking.py

@@ -0,0 +1,52 @@
+
+# content of group_tracking.py
+
+from deltachat import account_hookimpl, run_cmdline
+
+
+class GroupTrackingPlugin:
+    @account_hookimpl
+    def ac_incoming_message(self, message):
+        print("process_incoming message", message)
+        if message.text.strip() == "/quit":
+            message.account.shutdown()
+        else:
+            # unconditionally accept the chat
+            message.accept_sender_contact()
+            addr = message.get_sender_contact().addr
+            text = message.text
+            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+
+    @account_hookimpl
+    def ac_outgoing_message(self, message):
+        print("ac_outgoing_message:", message)
+
+    @account_hookimpl
+    def ac_configure_completed(self, success):
+        print("ac_configure_completed:", success)
+
+    @account_hookimpl
+    def ac_chat_modified(self, chat):
+        print("ac_chat_modified:", chat.id, chat.get_name())
+        for member in chat.get_contacts():
+            print("chat member: {}".format(member.addr))
+
+    @account_hookimpl
+    def ac_member_added(self, chat, contact, message):
+        print("ac_member_added {} to chat {} from {}".format(
+            contact.addr, chat.id, message.get_sender_contact().addr))
+        for member in chat.get_contacts():
+            print("chat member: {}".format(member.addr))
+
+    @account_hookimpl
+    def ac_member_removed(self, chat, contact, message):
+        print("ac_member_removed {} from chat {} by {}".format(
+            contact.addr, chat.id, message.get_sender_contact().addr))
+
+
+def main(argv=None):
+    run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()])
+
+
+if __name__ == "__main__":
+    main()

python/examples/test_examples.py

@@ -0,0 +1,72 @@
+
+import pytest
+import py
+import echo_and_quit
+import group_tracking
+from deltachat.eventlogger import FFIEventLogger
+
+
+@pytest.fixture(scope='session')
+def datadir():
+    """The py.path.local object of the test-data/ directory."""
+    for path in reversed(py.path.local(__file__).parts()):
+        datadir = path.join('test-data')
+        if datadir.isdir():
+            return datadir
+    else:
+        pytest.skip('test-data directory not found')
+
+
+def test_echo_quit_plugin(acfactory):
+    botproc = acfactory.run_bot_process(echo_and_quit)
+
+    ac1 = acfactory.get_one_online_account()
+    bot_contact = ac1.create_contact(botproc.addr)
+    ch1 = ac1.create_chat_by_contact(bot_contact)
+    ch1.send_text("hello")
+    reply = ac1._evtracker.wait_next_incoming_message()
+    assert "hello" in reply.text
+    assert reply.chat == ch1
+    ch1.send_text("/quit")
+    botproc.wait()
+
+
+def test_group_tracking_plugin(acfactory, lp):
+    lp.sec("creating one group-tracking bot and two temp accounts")
+    botproc = acfactory.run_bot_process(group_tracking, ffi=False)
+
+    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
+
+    botproc.fnmatch_lines("""
+        *ac_configure_completed*
+    """)
+    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
+    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+
+    lp.sec("creating bot test group with bot")
+    bot_contact = ac1.create_contact(botproc.addr)
+    ch = ac1.create_group_chat("bot test group")
+    ch.add_contact(bot_contact)
+    ch.send_text("hello")
+
+    botproc.fnmatch_lines("""
+        *ac_chat_modified*bot test group*
+    """)
+
+    lp.sec("adding third member {}".format(ac2.get_config("addr")))
+    contact3 = ac1.create_contact(ac2.get_config("addr"))
+    ch.add_contact(contact3)
+
+    reply = ac1._evtracker.wait_next_incoming_message()
+    assert "hello" in reply.text
+
+    lp.sec("now looking at what the bot received")
+    botproc.fnmatch_lines("""
+        *ac_member_added {}*
+    """.format(contact3.addr))
+
+    lp.sec("contact successfully added, now removing")
+    ch.remove_contact(contact3)
+    botproc.fnmatch_lines("""
+        *ac_member_removed {}*
+    """.format(contact3.addr))

python/setup.py

@@ -22,6 +22,11 @@ def main():
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],
+        entry_points = {
+            'pytest11': [
+                'deltachat.testplugin = deltachat.testplugin',
+            ],
+        },
         classifiers=[
             'Development Status :: 4 - Beta',
             'Intended Audience :: Developers',

python/src/deltachat/__init__.py

@@ -1,6 +1,13 @@
-from deltachat import capi, const
+import sys
-from deltachat.capi import ffi
+
-from deltachat.account import Account  # noqa
+from . import capi, const, hookspec
+from .capi import ffi
+from .account import Account  # noqa
+from .message import Message  # noqa
+from .contact import Contact  # noqa
+from .chat import Chat        # noqa
+from .hookspec import account_hookimpl, global_hookimpl  # noqa
+from . import eventlogger
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -64,3 +71,62 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
             if name.startswith("DC_EVENT_"):
                 _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
+
+
+def register_global_plugin(plugin):
+    """ Register a global plugin which implements one or more
+    of the :class:`deltachat.hookspec.Global` hooks.
+    """
+    gm = hookspec.Global._get_plugin_manager()
+    gm.register(plugin)
+    gm.check_pending()
+
+
+def unregister_global_plugin(plugin):
+    gm = hookspec.Global._get_plugin_manager()
+    gm.unregister(plugin)
+
+
+register_global_plugin(eventlogger)
+
+
+def run_cmdline(argv=None, account_plugins=None):
+    """ Run a simple default command line app, registering the specified
+    account plugins. """
+    import argparse
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(prog=argv[0] if argv else None)
+    parser.add_argument("db", action="store", help="database file")
+    parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events")
+    parser.add_argument("--email", action="store", help="email address")
+    parser.add_argument("--password", action="store", help="password")
+
+    args = parser.parse_args(argv[1:])
+
+    ac = Account(args.db)
+
+    if args.show_ffi:
+        log = eventlogger.FFIEventLogger(ac, "bot")
+        ac.add_account_plugin(log)
+
+    if not ac.is_configured():
+        assert args.email and args.password, (
+            "you must specify --email and --password once to configure this database/account"
+        )
+        ac.set_config("addr", args.email)
+        ac.set_config("mail_pw", args.password)
+        ac.set_config("mvbox_move", "0")
+        ac.set_config("mvbox_watch", "0")
+        ac.set_config("sentbox_watch", "0")
+
+    for plugin in account_plugins or []:
+        ac.add_account_plugin(plugin)
+
+    # start IO threads and configure if neccessary
+    ac.start()
+
+    print("{}: waiting for message".format(ac.get_config("addr")))
+
+    ac.wait_shutdown()

python/src/deltachat/account.py

@@ -2,21 +2,25 @@
 
 from __future__ import print_function
 import atexit
-import threading
+from contextlib import contextmanager
+from email.utils import parseaddr
+import queue
+from threading import Event
 import os
-import time
 from array import array
-from queue import Queue
-
 import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
-from .message import Message
+from .message import Message, map_system_message
 from .contact import Contact
-from .eventlogger import EventLogger
+from .tracker import ImexTracker
-from .hookspec import get_plugin_manager, hookimpl
+from . import hookspec, iothreads
+
+
+class MissingCredentials(ValueError):
+    """ Account is missing `addr` and `mail_pw` config values. """
 
 
 class Account(object):
@@ -24,45 +28,53 @@ class Account(object):
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    def __init__(self, db_path, logid=None, os_name=None, debug=True):
+    MissingCredentials = MissingCredentials
+
+    def __init__(self, db_path, os_name=None):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
         :param os_name: this will be put to the X-Mailer header in outgoing messages
-        :param debug: turn on debug logging for events.
         """
+        # initialize per-account plugin system
+        self._pm = hookspec.PerAccount._make_plugin_manager()
+        self.add_account_plugin(self)
+
         self._dc_context = ffi.gc(
             lib.dc_context_new(ffi.NULL, as_dc_charpointer(os_name)),
             _destroy_dc_context,
         )
-        self._evlogger = EventLogger(self, logid, debug)
-        self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
 
-        # register event call back and initialize plugin system
+        hook = hookspec.Global._get_plugin_manager().hook
-        def _ll_event(ctx, evt_name, data1, data2):
-            assert ctx == self._dc_context
-            self.pluggy.hook.process_low_level_event(
-                account=self, event_name=evt_name, data1=data1, data2=data2
-            )
 
-        self.pluggy = get_plugin_manager()
+        self._threads = iothreads.IOThreads(self)
-        self.pluggy.register(self._evlogger)
+        self._hook_event_queue = queue.Queue()
-        deltachat.set_context_callback(self._dc_context, _ll_event)
+        self._in_use_iter_events = False
+        self._shutdown_event = Event()
 
         # open database
+        self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
         if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
         atexit.register(self.shutdown)
+        hook.dc_account_init(account=self)
+
+    @hookspec.account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        for name, kwargs in self._map_ffi_event(ffi_event):
+            ev = HookEvent(self, name=name, kwargs=kwargs)
+            self._hook_event_queue.put(ev)
 
     # def __del__(self):
     #    self.shutdown()
 
+    def ac_log_line(self, msg):
+        self._pm.hook.ac_log_line(message=msg)
+
     def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
@@ -134,16 +146,15 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def configure(self, **kwargs):
+    def update_config(self, kwargs):
-        """ set config values and configure this account.
+        """ update config values.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
         :returns: None
         """
-        for name, value in kwargs.items():
+        for key, value in kwargs.items():
-            self.set_config(name, value)
+            self.set_config(key, str(value))
-        lib.dc_configure(self._dc_context)
 
     def is_configured(self):
         """ determine if the account is configured already; an initial connection
@@ -151,7 +162,7 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return lib.dc_is_configured(self._dc_context)
+        return bool(lib.dc_is_configured(self._dc_context))
 
     def set_avatar(self, img_path):
         """Set self avatar.
@@ -202,8 +213,7 @@ class Account(object):
 
         :returns: :class:`deltachat.contact.Contact`
         """
-        self.check_is_configured()
+        return Contact(self, const.DC_CONTACT_ID_SELF)
-        return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
 
     def create_contact(self, email, name=None):
         """ create a (new) Contact. If there already is a Contact
@@ -214,11 +224,14 @@ class Account(object):
         :param name: display name for this contact (optional)
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        name = as_dc_charpointer(name)
+        realname, addr = parseaddr(email)
-        email = as_dc_charpointer(email)
+        if name:
-        contact_id = lib.dc_create_contact(self._dc_context, name, email)
+            realname = name
+        realname = as_dc_charpointer(realname)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
-        return Contact(self._dc_context, contact_id)
+        return Contact(self, contact_id)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -231,6 +244,14 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
+    def get_contact_by_addr(self, email):
+        """ get a contact for the email address or None if it's blocked or doesn't exist. """
+        _, addr = parseaddr(email)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
+        if contact_id:
+            return self.get_contact_by_id(contact_id)
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -250,7 +271,15 @@ class Account(object):
             lib.dc_get_contacts(self._dc_context, flags, query),
             lib.dc_array_unref
         )
-        return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x)))
+        return list(iter_array(dc_array, lambda x: Contact(self, x)))
+
+    def get_fresh_messages(self):
+        """ yield all fresh messages from all chats. """
+        dc_array = ffi.gc(
+            lib.dc_get_fresh_msgs(self._dc_context),
+            lib.dc_array_unref
+        )
+        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
     def create_chat_by_contact(self, contact):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
@@ -272,6 +301,9 @@ class Account(object):
         """ create or get an existing chat object for the
         the specified message.
 
+        If this message is in the deaddrop chat then
+        the sender will become an accepted contact.
+
         :param message: messsage id or message instance.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
@@ -324,6 +356,13 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -368,13 +407,18 @@ class Account(object):
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
     def export_self_keys(self, path):
-        """ export public and private keys to the specified directory. """
+        """ export public and private keys to the specified directory.
+
+        Note that the account does not have to be started.
+        """
         return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
+
+        Note that the account does not have to be started.
         """
         export_files = self._export(path, 11)
         if len(export_files) != 1:
@@ -382,7 +426,7 @@ class Account(object):
         return export_files[0]
 
     def _export(self, path, imex_cmd):
-        with ImexTracker(self) as imex_tracker:
+        with self.temp_plugin(ImexTracker()) as imex_tracker:
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             return imex_tracker.wait_finish()
 
@@ -390,6 +434,8 @@ class Account(object):
         """ Import private keys found in the `path` directory.
         The last imported key is made the default keys unless its name
         contains the string legacy. Public keys are not imported.
+
+        Note that the account does not have to be started.
         """
         self._import(path, imex_cmd=2)
 
@@ -402,7 +448,7 @@ class Account(object):
         self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
-        with ImexTracker(self) as imex_tracker:
+        with self.temp_plugin(ImexTracker()) as imex_tracker:
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             imex_tracker.wait_finish()
 
@@ -469,46 +515,6 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def stop_ongoing(self):
-        lib.dc_stop_ongoing_process(self._dc_context)
-
-    #
-    # meta API for start/stop and event based processing
-    #
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.get_message_by_id(ev[2])
-
-    def start_threads(self):
-        """ start IMAP/SMTP threads (and configure account if it hasn't happened).
-
-        :raises: ValueError if 'addr' or 'mail_pw' are not configured.
-        :returns: None
-        """
-        if not self.is_configured():
-            self.configure()
-        self._threads.start()
-
-    def stop_threads(self, wait=True):
-        """ stop IMAP/SMTP threads. """
-        if self._threads.is_started():
-            self.stop_ongoing()
-            self._threads.stop(wait=wait)
-
-    def shutdown(self, wait=True):
-        """ stop threads and close and remove underlying dc_context and callbacks. """
-        if hasattr(self, "_dc_context") and hasattr(self, "_threads"):
-            # print("SHUTDOWN", self)
-            self.stop_threads(wait=False)
-            lib.dc_close(self._dc_context)
-            self.stop_threads(wait=wait)  # to wait for threads
-            deltachat.clear_context_callback(self._dc_context)
-            del self._dc_context
-            atexit.unregister(self.shutdown)
-        self.pluggy.unregister(self._evlogger)
-
     def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
@@ -523,88 +529,122 @@ class Account(object):
         if dc_res == 0:
             raise ValueError("no chat is streaming locations")
 
+    #
+    # meta API for start/stop and event based processing
+    #
 
-class ImexTracker:
+    def add_account_plugin(self, plugin, name=None):
-    def __init__(self, account):
+        """ add an account plugin which implements one or more of
-        self._imex_events = Queue()
+        the :class:`deltachat.hookspec.PerAccount` hooks.
-        self.account = account
+        """
+        self._pm.register(plugin, name=name)
+        self._pm.check_pending()
+        return plugin
 
-    def __enter__(self):
+    @contextmanager
-        self.account.pluggy.register(self)
+    def temp_plugin(self, plugin):
-        return self
+        """ run a with-block with the given plugin temporarily registered. """
+        self._pm.register(plugin)
+        yield plugin
+        self._pm.unregister(plugin)
 
-    def __exit__(self, *args):
+    def stop_ongoing(self):
-        self.account.pluggy.unregister(self)
+        """ Stop ongoing securejoin, configuration or other core jobs. """
+        lib.dc_stop_ongoing_process(self._dc_context)
 
-    @hookimpl
+    def start(self, callback_thread=True):
-    def process_low_level_event(self, account, event_name, data1, data2):
+        """ start this account (activate imap/smtp threads etc.)
-        # there could be multiple accounts instantiated
+        and return immediately.
-        if self.account is not account:
+
+        If this account is not configured, an internal configuration
+        job will be scheduled if config values are sufficiently specified.
+
+        You may call `wait_shutdown` or `shutdown` after the
+        account is in started mode.
+
+        :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
+
+        :returns: None
+        """
+        if not self.is_configured():
+            if not self.get_config("addr") or not self.get_config("mail_pw"):
+                raise MissingCredentials("addr or mail_pwd not set in config")
+            lib.dc_configure(self._dc_context)
+        self._threads.start(callback_thread=callback_thread)
+
+    def wait_shutdown(self):
+        """ wait until shutdown of this account has completed. """
+        self._shutdown_event.wait()
+
+    def shutdown(self, wait=True):
+        """ shutdown account, stop threads and close and remove
+        underlying dc_context and callbacks. """
+        dc_context = self._dc_context
+        if dc_context is None:
             return
-        if event_name == "DC_EVENT_IMEX_PROGRESS":
-            self._imex_events.put(data1)
-        elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(data1)
 
-    def wait_finish(self, progress_timeout=60):
+        if self._threads.is_started():
-        """ Return list of written files, raise ValueError if ExportFailed. """
+            self.stop_ongoing()
-        files_written = []
+            self._threads.stop(wait=False)
-        while True:
+        lib.dc_close(dc_context)
-            ev = self._imex_events.get(timeout=progress_timeout)
+        self._hook_event_queue.put(None)
-            if isinstance(ev, str):
+        self._threads.stop(wait=wait)  # to wait for threads
-                files_written.append(ev)
+        self._dc_context = None
-            elif ev == 0:
+        atexit.unregister(self.shutdown)
-                raise ValueError("export failed, exp-files: {}".format(files_written))
+        self._shutdown_event.set()
-            elif ev == 1000:
+        hook = hookspec.Global._get_plugin_manager().hook
-                return files_written
+        hook.dc_account_after_shutdown(account=self, dc_context=dc_context)
 
+    def _handle_current_events(self):
+        """ handle all currently queued events and then return. """
+        while 1:
+            try:
+                event = self._hook_event_queue.get(block=False)
+            except queue.Empty:
+                break
+            else:
+                event.call_hook()
 
-class IOThreads:
+    def iter_events(self, timeout=None):
-    def __init__(self, dc_context, log_event=lambda *args: None):
+        """ yield hook events until shutdown.
-        self._dc_context = dc_context
-        self._thread_quitflag = False
-        self._name2thread = {}
-        self._log_event = log_event
-        self._log_running = True
 
-        # Make sure the current
+        It is not allowed to call iter_events() from multiple threads.
-        self._start_one_thread("deltachat-log", self.dc_thread_run)
+        """
+        if self._in_use_iter_events:
+            raise RuntimeError("can only call iter_events() from one thread")
+        self._in_use_iter_events = True
+        while 1:
+            event = self._hook_event_queue.get(timeout=timeout)
+            if event is None:
+                break
+            yield event
 
-    def is_started(self):
+    def _map_ffi_event(self, ffi_event):
-        return lib.dc_is_open(self._dc_context) and lib.dc_is_running(self._dc_context)
+        name = ffi_event.name
-
+        if name == "DC_EVENT_CONFIGURE_PROGRESS":
-    def start(self, imap=True, smtp=True, mvbox=False, sentbox=False):
+            data1 = ffi_event.data1
-        assert not self.is_started()
+            if data1 == 0 or data1 == 1000:
-
+                success = data1 == 1000
-        lib.dc_context_run(self._dc_context)
+                yield "ac_configure_completed", dict(success=success)
-
+        elif name == "DC_EVENT_INCOMING_MSG":
-    def _start_one_thread(self, name, func):
+            msg = self.get_message_by_id(ffi_event.data2)
-        self._name2thread[name] = t = threading.Thread(target=func, name=name)
+            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        t.setDaemon(1)
+        elif name == "DC_EVENT_MSGS_CHANGED":
-        t.start()
+            if ffi_event.data2 != 0:
-
+                msg = self.get_message_by_id(ffi_event.data2)
-    def stop(self, wait=False):
+                if msg.is_outgoing():
-        if self.is_started():
+                    res = map_system_message(msg)
-            lib.dc_context_shutdown(self._dc_context)
+                    if res and res[0].startswith("ac_member"):
-
+                        yield res
-    def dc_thread_run(self):
+                    yield "ac_outgoing_message", dict(message=msg)
-        self._log_event("py-bindings-info", 0, "DC LOG THREAD START")
+                elif msg.is_in_fresh():
-
+                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        while self._log_running:
+        elif name == "DC_EVENT_MSG_DELIVERED":
-            if lib.dc_is_open(self._dc_context) and lib.dc_has_next_event(self._dc_context):
+            msg = self.get_message_by_id(ffi_event.data2)
-                event = lib.dc_get_next_event(self._dc_context)
+            yield "ac_message_delivered", dict(message=msg)
-                if event != ffi.NULL:
+        elif name == "DC_EVENT_CHAT_MODIFIED":
-                    deltachat.py_dc_callback(
+            chat = self.get_chat_by_id(ffi_event.data1)
-                        self._dc_context,
+            yield "ac_chat_modified", dict(chat=chat)
-                        lib.dc_event_get_id(event),
-                        lib.dc_event_get_data1(event),
-                        lib.dc_event_get_data2(event)
-                    )
-                    lib.dc_event_unref(event)
-                else:
-                    time.sleep(0.05)
-
-        self._log_event("py-bindings-info", 0, "DC LOG THREAD FINISHED")
 
 
 def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
@@ -631,3 +671,17 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
+
+
+class HookEvent:
+    def __init__(self, account, name, kwargs):
+        assert hasattr(account._pm.hook, name), name
+        self.account = account
+        self.name = name
+        self.kwargs = kwargs
+
+    def call_hook(self):
+        hook = getattr(self.account._pm.hook, self.name, None)
+        if hook is None:
+            raise ValueError("event_name {} unknown".format(self.name))
+        return hook(**self.kwargs)

python/src/deltachat/chat.py

@@ -30,7 +30,7 @@ class Chat(object):
         return not (self == other)
 
     def __repr__(self):
-        return "<Chat id={} name={} dc_context={}>".format(self.id, self.get_name(), self._dc_context)
+        return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
     def _dc_chat(self):
@@ -51,6 +51,16 @@ class Chat(object):
 
     # ------  chat status/metadata API ------------------------------
 
+    def is_group(self):
+        """ return true if this chat is a group chat.
+
+        :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
+        """
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_VERIFIED_GROUP
+        )
+
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
 
@@ -129,7 +139,7 @@ class Chat(object):
         return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
 
     def get_type(self):
-        """ return type of this chat.
+        """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
@@ -353,7 +363,7 @@ class Chat(object):
             lib.dc_array_unref
         )
         return list(iter_array(
-            dc_array, lambda id: Contact(self._dc_context, id))
+            dc_array, lambda id: Contact(self.account, id))
         )
 
     def set_profile_image(self, img_path):
@@ -405,12 +415,6 @@ class Chat(object):
         """
         return lib.dc_chat_get_color(self._dc_chat)
 
-    def get_subtitle(self):
-        """return the subtitle of the chat
-        :returns: the subtitle
-        """
-        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
-
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):

python/src/deltachat/const.py

@@ -11,6 +11,7 @@ from os.path import join as joinpath
 DC_GCL_ARCHIVED_ONLY = 0x01
 DC_GCL_NO_SPECIALS = 0x02
 DC_GCL_ADD_ALLDONE_HINT = 0x04
+DC_GCL_FOR_FORWARDING = 0x08
 DC_GCL_VERIFIED_ONLY = 0x01
 DC_GCL_ADD_SELF = 0x02
 DC_QR_ASK_VERIFYCONTACT = 200
@@ -98,7 +99,6 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
 DC_EVENT_GET_STRING = 2091
@@ -115,8 +115,6 @@ DC_CHAT_VISIBILITY_PINNED = 2
 DC_STR_NOMESSAGES = 1
 DC_STR_SELF = 2
 DC_STR_DRAFT = 3
-DC_STR_MEMBER = 4
-DC_STR_CONTACT = 6
 DC_STR_VOICEMESSAGE = 7
 DC_STR_DEADDROP = 8
 DC_STR_IMAGE = 9
@@ -148,7 +146,6 @@ DC_STR_ARCHIVEDCHATS = 40
 DC_STR_STARREDMSGS = 41
 DC_STR_AC_SETUP_MSG_SUBJECT = 42
 DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_SELFTALK_SUBTITLE = 50
 DC_STR_CANNOT_LOGIN = 60
 DC_STR_SERVER_RESPONSE = 61
 DC_STR_MSGACTIONBYUSER = 62

python/src/deltachat/contact.py

@@ -10,8 +10,9 @@ class Contact(object):
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
-    def __init__(self, dc_context, id):
+    def __init__(self, account, id):
-        self._dc_context = dc_context
+        self.account = account
+        self._dc_context = account._dc_context
         self.id = id
 
     def __eq__(self, other):
@@ -57,3 +58,7 @@ class Contact(object):
         if dc_res == ffi.NULL:
             return None
         return from_dc_charpointer(dc_res)
+
+    def get_chat(self):
+        """return 1:1 chat for this contact. """
+        return self.account.create_chat_by_contact(self)

python/src/deltachat/eventlogger.py

@@ -1,28 +1,88 @@
+import deltachat
 import threading
-import re
 import time
+import re
 from queue import Queue, Empty
-from .hookspec import hookimpl
+from .hookspec import account_hookimpl, global_hookimpl
 
 
-class EventLogger:
+@global_hookimpl
+def dc_account_init(account):
+    # send all FFI events for this account to a plugin hook
+    def _ll_event(ctx, evt_name, data1, data2):
+        assert ctx == account._dc_context
+        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
+        account._pm.hook.ac_process_ffi_event(
+            account=account, ffi_event=ffi_event
+        )
+    deltachat.set_context_callback(account._dc_context, _ll_event)
+
+
+@global_hookimpl
+def dc_account_after_shutdown(dc_context):
+    deltachat.clear_context_callback(dc_context)
+
+
+class FFIEvent:
+    def __init__(self, name, data1, data2):
+        self.name = name
+        self.data1 = data1
+        self.data2 = data2
+
+    def __str__(self):
+        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
+
+
+class FFIEventLogger:
+    """ If you register an instance of this logger with an Account
+    you'll get all ffi-events printed.
+    """
+    # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid=None, debug=True):
+    def __init__(self, account, logid):
+        """
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        """
         self.account = account
-        self._event_queue = Queue()
-        self._debug = debug
-        if logid is None:
-            logid = str(self.account._dc_context).strip(">").split()[-1]
         self.logid = logid
-        self._timeout = None
         self.init_time = time.time()
 
-    @hookimpl
+    @account_hookimpl
-    def process_low_level_event(self, account, event_name, data1, data2):
+    def ac_process_ffi_event(self, ffi_event):
-        if self.account == account:
+        self._log_event(ffi_event)
-            self._log_event(event_name, data1, data2)
+
-            self._event_queue.put((event_name, data1, data2))
+    def _log_event(self, ffi_event):
+        # don't show events that are anyway empty impls now
+        if ffi_event.name == "DC_EVENT_GET_STRING":
+            return
+        self.account.ac_log_line(str(ffi_event))
+
+    @account_hookimpl
+    def ac_log_line(self, message):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        elapsed = time.time() - self.init_time
+        locname = tname
+        if self.logid:
+            locname += "-" + self.logid
+        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+        with self._loglock:
+            print(s, flush=True)
+
+
+class FFIEventTracker:
+    def __init__(self, account, timeout=None):
+        self.account = account
+        self._timeout = timeout
+        self._event_queue = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._event_queue.put(ffi_event)
 
     def set_timeout(self, timeout):
         self._timeout = timeout
@@ -32,10 +92,10 @@ class EventLogger:
             self.get(check_error=check_error)
 
     def get(self, timeout=None, check_error=True):
-        timeout = timeout or self._timeout
+        timeout = timeout if timeout is not None else self._timeout
         ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev[0] == "DC_EVENT_ERROR":
+        if check_error and ev.name == "DC_EVENT_ERROR":
-            raise ValueError("{}({!r},{!r})".format(*ev))
+            raise ValueError(str(ev))
         return ev
 
     def ensure_event_not_queued(self, event_name_regex):
@@ -47,35 +107,31 @@ class EventLogger:
             except Empty:
                 break
             else:
-                assert not rex.match(ev[0]), "event found {}".format(ev)
+                assert not rex.match(ev.name), "event found {}".format(ev)
 
     def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
+        self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex))
         rex = re.compile("(?:{}).*".format(event_name_regex))
         while 1:
             ev = self.get(timeout=timeout, check_error=check_error)
-            if rex.match(ev[0]):
+            if rex.match(ev.name):
                 return ev
 
     def get_info_matching(self, regex):
         rex = re.compile("(?:{}).*".format(regex))
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev[2]):
+            if rex.match(ev.data2):
                 return ev
 
-    def _log_event(self, evt_name, data1, data2):
+    def wait_next_incoming_message(self):
-        # don't show events that are anyway empty impls now
+        """ wait for and return next incoming message. """
-        if evt_name == "DC_EVENT_GET_STRING":
+        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
-            return
+        return self.account.get_message_by_id(ev.data2)
-        if self._debug:
-            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
-            self._log(evpart)
 
-    def _log(self, msg):
+    def wait_next_messages_changed(self):
-        t = threading.currentThread()
+        """ wait for and return next message-changed message or None
-        tname = getattr(t, "name", t)
+        if the event contains no msgid"""
-        if tname == "MainThread":
+        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
-            tname = "MAIN"
+        if ev.data2 > 0:
-        with self._loglock:
+            return self.account.get_message_by_id(ev.data2)
-            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))

python/src/deltachat/hookspec.py

@@ -1,25 +1,92 @@
-""" Hooks for python bindings """
+""" Hooks for Python bindings to Delta Chat Core Rust CFFI"""
 
 import pluggy
 
-name = "deltachat"
 
-hookspec = pluggy.HookspecMarker(name)
+account_spec_name = "deltachat-account"
-hookimpl = pluggy.HookimplMarker(name)
+account_hookspec = pluggy.HookspecMarker(account_spec_name)
-_plugin_manager = None
+account_hookimpl = pluggy.HookimplMarker(account_spec_name)
+
+global_spec_name = "deltachat-global"
+global_hookspec = pluggy.HookspecMarker(global_spec_name)
+global_hookimpl = pluggy.HookimplMarker(global_spec_name)
 
 
-def get_plugin_manager():
+class PerAccount:
-    global _plugin_manager
+    """ per-Account-instance hook specifications.
-    if _plugin_manager is None:
+
-        _plugin_manager = pluggy.PluginManager(name)
+    Except for ac_process_ffi_event all hooks are executed
-        _plugin_manager.add_hookspecs(DeltaChatHookSpecs)
+    in the thread which calls Account.wait_shutdown().
-    return _plugin_manager
+    """
+    @classmethod
+    def _make_plugin_manager(cls):
+        pm = pluggy.PluginManager(account_spec_name)
+        pm.add_hookspecs(cls)
+        return pm
+
+    @account_hookspec
+    def ac_process_ffi_event(self, ffi_event):
+        """ process a CFFI low level events for a given account.
+
+        ffi_event has "name", "data1", "data2" values as specified
+        with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
+
+        DANGER: this hook is executed from the callback invoked by core.
+        Hook implementations need to be short running and can typically
+        not call back into core because this would easily cause recursion issues.
+        """
+
+    @account_hookspec
+    def ac_log_line(self, message):
+        """ log a message related to the account. """
+
+    @account_hookspec
+    def ac_configure_completed(self, success):
+        """ Called when a configure process completed. """
+
+    @account_hookspec
+    def ac_incoming_message(self, message):
+        """ Called on any incoming message (to deaddrop or chat). """
+
+    @account_hookspec
+    def ac_outgoing_message(self, message):
+        """ Called on each outgoing message (both system and "normal")."""
+
+    @account_hookspec
+    def ac_message_delivered(self, message):
+        """ Called when an outgoing message has been delivered to SMTP. """
+
+    @account_hookspec
+    def ac_chat_modified(self, chat):
+        """ Chat was created or modified regarding membership, avatar, title. """
+
+    @account_hookspec
+    def ac_member_added(self, chat, contact, message):
+        """ Called for each contact added to an accepted chat. """
+
+    @account_hookspec
+    def ac_member_removed(self, chat, contact, message):
+        """ Called for each contact removed from a chat.  """
 
 
-class DeltaChatHookSpecs:
+class Global:
-    """ Plugin Hook specifications for Python bindings to Delta Chat CFFI. """
+    """ global hook specifications using a per-process singleton
+    plugin manager instance.
 
-    @hookspec
+    """
-    def process_low_level_event(self, account, event_name, data1, data2):
+    _plugin_manager = None
-        """ process a CFFI low level events for a given account. """
+
+    @classmethod
+    def _get_plugin_manager(cls):
+        if cls._plugin_manager is None:
+            cls._plugin_manager = pm = pluggy.PluginManager(global_spec_name)
+            pm.add_hookspecs(cls)
+        return cls._plugin_manager
+
+    @global_hookspec
+    def dc_account_init(self, account):
+        """ called when `Account::__init__()` function starts executing. """
+
+    @global_hookspec
+    def dc_account_after_shutdown(self, account, dc_context):
+        """ Called after the account has been shutdown. """

python/src/deltachat/iothreads.py

@@ -0,0 +1,106 @@
+
+import threading
+import time
+
+from contextlib import contextmanager
+
+from .capi import lib
+
+
+class IOThreads:
+    def __init__(self, account):
+        self.account = account
+        self._dc_context = account._dc_context
+        self._thread_quitflag = False
+        self._name2thread = {}
+
+    def is_started(self):
+        return len(self._name2thread) > 0
+
+    def start(self, callback_thread):
+        assert not self.is_started()
+        self._start_one_thread("inbox", self.imap_thread_run)
+        self._start_one_thread("smtp", self.smtp_thread_run)
+
+        if callback_thread:
+            self._start_one_thread("cb", self.cb_thread_run)
+
+        if int(self.account.get_config("mvbox_watch")):
+            self._start_one_thread("mvbox", self.mvbox_thread_run)
+
+        if int(self.account.get_config("sentbox_watch")):
+            self._start_one_thread("sentbox", self.sentbox_thread_run)
+
+    def _start_one_thread(self, name, func):
+        self._name2thread[name] = t = threading.Thread(target=func, name=name)
+        t.setDaemon(1)
+        t.start()
+
+    @contextmanager
+    def log_execution(self, message):
+        self.account.ac_log_line(message + " START")
+        yield
+        self.account.ac_log_line(message + " FINISHED")
+
+    def stop(self, wait=False):
+        self._thread_quitflag = True
+
+        # Workaround for a race condition. Make sure that thread is
+        # not in between checking for quitflag and entering idle.
+        time.sleep(0.5)
+
+        lib.dc_interrupt_imap_idle(self._dc_context)
+        lib.dc_interrupt_smtp_idle(self._dc_context)
+        if "mvbox" in self._name2thread:
+            lib.dc_interrupt_mvbox_idle(self._dc_context)
+        if "sentbox" in self._name2thread:
+            lib.dc_interrupt_sentbox_idle(self._dc_context)
+        if wait:
+            for name, thread in self._name2thread.items():
+                if thread != threading.currentThread():
+                    thread.join()
+
+    def cb_thread_run(self):
+        with self.log_execution("CALLBACK THREAD START"):
+            it = self.account.iter_events()
+            while not self._thread_quitflag:
+                try:
+                    ev = next(it)
+                except StopIteration:
+                    break
+                self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs))
+                ev.call_hook()
+
+    def imap_thread_run(self):
+        with self.log_execution("INBOX THREAD START"):
+            while not self._thread_quitflag:
+                lib.dc_perform_imap_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_idle(self._dc_context)
+
+    def mvbox_thread_run(self):
+        with self.log_execution("MVBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_mvbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_idle(self._dc_context)
+
+    def sentbox_thread_run(self):
+        with self.log_execution("SENTBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_sentbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_idle(self._dc_context)
+
+    def smtp_thread_run(self):
+        with self.log_execution("SMTP THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_smtp_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_smtp_idle(self._dc_context)

python/src/deltachat/message.py

@@ -28,7 +28,9 @@ class Message(object):
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
-        return "<Message id={} dc_context={}>".format(self.id, self._dc_context)
+        c = self.get_sender_contact()
+        return "<Message id={} sender={}/{} outgoing={} chat={}/{}>".format(
+            self.id, c.id, c.addr, self.is_outgoing(), self.chat.id, self.chat.get_name())
 
     @classmethod
     def from_db(cls, account, id):
@@ -50,6 +52,16 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
+    def accept_sender_contact(self):
+        """ ensure that the sender is an accepted contact
+        and that the message has a non-deaddrop chat object.
+        """
+        self.account.create_chat_by_message(self)
+        self._dc_msg = ffi.gc(
+                lib.dc_get_msg(self._dc_context, self.id),
+                lib.dc_msg_unref
+        )
+
     @props.with_doc
     def text(self):
         """unicode text of this messages (might be empty if not a text message). """
@@ -81,6 +93,10 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
+    def is_system_message(self):
+        """ return True if this message is a system/info message. """
+        return lib.dc_msg_is_info(self._dc_msg)
+
     def is_setup_message(self):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
@@ -159,6 +175,13 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
+    def get_sender_chat(self):
+        """return the 1:1 chat with the sender of this message.
+
+        :returns: :class:`deltachat.chat.Chat` instance
+        """
+        return self.get_sender_contact().get_chat()
+
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
@@ -166,7 +189,7 @@ class Message(object):
         """
         from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
-        return Contact(self._dc_context, contact_id)
+        return Contact(self.account, contact_id)
 
     #
     # Message State query methods
@@ -207,6 +230,13 @@ class Message(object):
         """
         return self._msgstate == const.DC_STATE_IN_SEEN
 
+    def is_outgoing(self):
+        """Return True if Message is outgoing. """
+        return self._msgstate in (
+            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
+            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
+            const.DC_STATE_OUT_DELIVERED)
+
     def is_out_preparing(self):
         """Return True if Message is outgoing, but its file is being prepared.
         """
@@ -269,6 +299,10 @@ class Message(object):
         """ return True if it's a file message. """
         return self._view_type == const.DC_MSG_FILE
 
+    def mark_seen(self):
+        """ mark this message as seen. """
+        self.account.mark_seen_messages([self.id])
+
 
 # some code for handling DC_MSG_* view types
 
@@ -288,3 +322,29 @@ def get_viewtype_code_from_name(view_type_name):
             return code
     raise ValueError("message typecode not found for {!r}, "
                      "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
+
+
+#
+# some helper code for turning system messages into hook events
+#
+
+def map_system_message(msg):
+    if msg.is_system_message():
+        res = parse_system_add_remove(msg.text)
+        if res:
+            contact = msg.account.get_contact_by_addr(res[1])
+            if contact:
+                d = dict(chat=msg.chat, contact=contact, message=msg)
+                return "ac_member_" + res[0], d
+
+
+def parse_system_add_remove(text):
+    # Member Me (x@y) removed by a@b.
+    # Member x@y removed by a@b
+    text = text.lower()
+    parts = text.split()
+    if parts[0] == "member":
+        if parts[2] in ("removed", "added"):
+            return parts[2], parts[1]
+        if parts[3] in ("removed", "added"):
+            return parts[3], parts[2].strip("()")

python/src/deltachat/testplugin.py

@@ -0,0 +1,381 @@
+from __future__ import print_function
+import os
+import sys
+import subprocess
+import queue
+import threading
+import fnmatch
+import pytest
+import requests
+import time
+from . import Account, const
+from .tracker import ConfigureTracker
+from .capi import lib
+from .eventlogger import FFIEventLogger, FFIEventTracker
+from _pytest.monkeypatch import MonkeyPatch
+from _pytest._code import Source
+
+import tempfile
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--liveconfig", action="store", default=None,
+        help="a file with >=2 lines where each line "
+             "contains NAME=VALUE config settings for one account"
+    )
+    parser.addoption(
+        "--ignored", action="store_true",
+        help="Also run tests marked with the ignored marker",
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
+    )
+    cfg = config.getoption('--liveconfig')
+    if not cfg:
+        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
+        if cfg:
+            config.option.liveconfig = cfg
+
+
+def pytest_runtest_setup(item):
+    if (list(item.iter_markers(name="ignored"))
+            and not item.config.getoption("ignored")):
+        pytest.skip("Ignored tests not requested, use --ignored")
+
+
+def pytest_report_header(config, startdir):
+    summary = []
+
+    t = tempfile.mktemp()
+    m = MonkeyPatch()
+    try:
+        m.setattr(sys.stdout, "write", lambda x: len(x))
+        ac = Account(t)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        m.undo()
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+     )])
+
+    cfg = config.option.liveconfig
+    if cfg:
+        if "?" in cfg:
+            url, token = cfg.split("?", 1)
+            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
+        else:
+            summary.append('Liveconfig file: {}'.format(cfg))
+    return summary
+
+
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+class SessionLiveConfigFromURL:
+    def __init__(self, url):
+        self.configlist = []
+        self.url = url
+
+    def get(self, index):
+        try:
+            return self.configlist[index]
+        except IndexError:
+            assert index == len(self.configlist), index
+            res = requests.post(self.url)
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+            return config
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            return SessionLiveConfigFromURL(liveconfig_opt)
+        else:
+            return SessionLiveConfigFromFile(liveconfig_opt)
+
+
+@pytest.fixture
+def data(request):
+    class Data:
+        def __init__(self):
+            # trying to find test data heuristically
+            # because we are run from a dev-setup with pytest direct,
+            # through tox, and then maybe also from deltachat-binding
+            # users like "deltabot".
+            self.paths = [os.path.normpath(x) for x in [
+                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
+                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
+            ]]
+
+        def get_path(self, bn):
+            """ return path of file or None if it doesn't exist. """
+            for path in self.paths:
+                fn = os.path.join(path, *bn.split("/"))
+                if os.path.exists(fn):
+                    return fn
+            print("WARNING: path does not exist: {!r}".format(fn))
+
+        def read_path(self, bn, mode="r"):
+            fn = self.get_path(bn)
+            if fn is not None:
+                with open(fn, mode) as f:
+                    return f.read()
+
+    return Data()
+
+
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
+
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self.init_time = time.time()
+            self._generated_keys = ["alice", "bob", "charlie",
+                                    "dom", "elena", "fiona"]
+
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
+
+        def make_account(self, path, logid, quiet=False):
+            ac = Account(path)
+            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._configtracker = ac.add_account_plugin(ConfigureTracker())
+            if not quiet:
+                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+            self._finalizers.append(ac.shutdown)
+            return ac
+
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(2)
+            return ac
+
+        def _preconfigure_key(self, account, addr):
+            # Only set a key if we haven't used it yet for another account.
+            if self._generated_keys:
+                keyname = self._generated_keys.pop(0)
+                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
+                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
+                if fname_pub and fname_sec:
+                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
+                    return True
+                else:
+                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
+
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
+
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            self._preconfigure_key(ac, addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
+
+        def get_online_config(self, pre_generated_key=True, quiet=False):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
+
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
+            if pre_generated_key:
+                self._preconfigure_key(ac, configdict['addr'])
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            return ac, dict(configdict)
+
+        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
+                                           pre_generated_key=True, quiet=False, config={}):
+            ac, configdict = self.get_online_config(
+                pre_generated_key=pre_generated_key, quiet=quiet)
+            configdict.update(config)
+            configdict["mvbox_watch"] = str(int(mvbox))
+            configdict["mvbox_move"] = str(int(move))
+            configdict["sentbox_watch"] = str(int(sentbox))
+            ac.update_config(configdict)
+            ac.start()
+            return ac
+
+        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
+            ac1 = self.get_online_configuring_account(
+                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
+            ac1._configtracker.wait_imap_connected()
+            ac1._configtracker.wait_smtp_connected()
+            ac1._configtracker.wait_finish()
+            return ac1
+
+        def get_two_online_accounts(self, move=False, quiet=False):
+            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac2 = self.get_online_configuring_account(quiet=quiet)
+            ac1._configtracker.wait_finish()
+            ac2._configtracker.wait_finish()
+            return ac1, ac2
+
+        def clone_online_account(self, account, pre_generated_key=True):
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            if pre_generated_key:
+                self._preconfigure_key(ac, account.get_config("addr"))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            ac.update_config(dict(
+                addr=account.get_config("addr"),
+                mail_pw=account.get_config("mail_pw"),
+                mvbox_watch=account.get_config("mvbox_watch"),
+                mvbox_move=account.get_config("mvbox_move"),
+                sentbox_watch=account.get_config("sentbox_watch"),
+            ))
+            ac.start()
+            return ac
+
+        def run_bot_process(self, module, ffi=True):
+            fn = module.__file__
+
+            bot_ac, bot_cfg = self.get_online_config()
+
+            args = [
+                sys.executable,
+                "-u",
+                fn,
+                "--email", bot_cfg["addr"],
+                "--password", bot_cfg["mail_pw"],
+                bot_ac.db_path,
+            ]
+            if ffi:
+                args.insert(-1, "--show-ffi")
+            print("$", " ".join(args))
+            popen = subprocess.Popen(
+                args=args,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
+                bufsize=0,                 # line buffering
+                close_fds=True,            # close all FDs other than 0/1/2
+                universal_newlines=True    # give back text
+            )
+            bot = BotProcess(popen, bot_cfg)
+            self._finalizers.append(bot.kill)
+            return bot
+
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
+    return am
+
+
+class BotProcess:
+    def __init__(self, popen, bot_cfg):
+        self.popen = popen
+        self.addr = bot_cfg["addr"]
+
+        # we read stdout as quickly as we can in a thread and make
+        # the (unicode) lines available for readers through a queue.
+        self.stdout_queue = queue.Queue()
+        self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
+        t.setDaemon(1)
+        t.start()
+
+    def _run_stdout_thread(self):
+        try:
+            while 1:
+                line = self.popen.stdout.readline()
+                if not line:
+                    break
+                line = line.strip()
+                self.stdout_queue.put(line)
+        finally:
+            self.stdout_queue.put(None)
+
+    def kill(self):
+        self.popen.kill()
+
+    def wait(self, timeout=30):
+        self.popen.wait(timeout=timeout)
+
+    def fnmatch_lines(self, pattern_lines):
+        patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
+        for next_pattern in patterns:
+            print("+++FNMATCH:", next_pattern)
+            ignored = []
+            while 1:
+                line = self.stdout_queue.get(timeout=15)
+                if line is None:
+                    if ignored:
+                        print("BOT stdout terminated after these lines")
+                        for line in ignored:
+                            print(line)
+                    raise IOError("BOT stdout-thread terminated")
+                if fnmatch.fnmatch(line, next_pattern):
+                    print("+++MATCHED:", line)
+                    break
+                else:
+                    print("+++IGN:", line)
+                    ignored.append(line)
+
+
+@pytest.fixture
+def tmp_db_path(tmpdir):
+    return tmpdir.join("test.db").strpath
+
+
+@pytest.fixture
+def lp():
+    class Printer:
+        def sec(self, msg):
+            print()
+            print("=" * 10, msg, "=" * 10)
+
+        def step(self, msg):
+            print("-" * 5, "step " + msg, "-" * 5)
+    return Printer()

python/src/deltachat/tracker.py

@@ -0,0 +1,76 @@
+
+from queue import Queue
+from threading import Event
+
+from .hookspec import account_hookimpl
+
+
+class ImexFailed(RuntimeError):
+    """ Exception for signalling that import/export operations failed."""
+
+
+class ImexTracker:
+    def __init__(self):
+        self._imex_events = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
+            self._imex_events.put(ffi_event.data1)
+        elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
+            self._imex_events.put(ffi_event.data1)
+
+    def wait_finish(self, progress_timeout=60):
+        """ Return list of written files, raise ValueError if ExportFailed. """
+        files_written = []
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, str):
+                files_written.append(ev)
+            elif ev == 0:
+                raise ImexFailed("export failed, exp-files: {}".format(files_written))
+            elif ev == 1000:
+                return files_written
+
+
+class ConfigureFailed(RuntimeError):
+    """ Exception for signalling that configuration failed."""
+
+
+class ConfigureTracker:
+    ConfigureFailed = ConfigureFailed
+
+    def __init__(self):
+        self._configure_events = Queue()
+        self._smtp_finished = Event()
+        self._imap_finished = Event()
+        self._ffi_events = []
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._ffi_events.append(ffi_event)
+        if ffi_event.name == "DC_EVENT_SMTP_CONNECTED":
+            self._smtp_finished.set()
+        elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
+            self._imap_finished.set()
+
+    @account_hookimpl
+    def ac_configure_completed(self, success):
+        self._configure_events.put(success)
+
+    def wait_smtp_connected(self):
+        """ wait until smtp is configured. """
+        self._smtp_finished.wait()
+
+    def wait_imap_connected(self):
+        """ wait until smtp is configured. """
+        self._imap_finished.wait()
+
+    def wait_finish(self):
+        """ wait until configure is completed.
+
+        Raise Exception if Configure failed
+        """
+        if not self._configure_events.get():
+            content = "\n".join(map(str, self._ffi_events))
+            raise ConfigureFailed(content)

python/tests/conftest.py

@@ -1,320 +1,17 @@
 from __future__ import print_function
-import os
-import sys
-import py
-import pytest
-import requests
-import time
-from deltachat import Account
-from deltachat import const
-from deltachat.capi import lib
-from _pytest.monkeypatch import MonkeyPatch
-import tempfile
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
-    )
-    parser.addoption(
-        "--ignored", action="store_true",
-        help="Also run tests marked with the ignored marker",
-    )
-
-
-def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
-    cfg = config.getoption('--liveconfig')
-    if not cfg:
-        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
-        if cfg:
-            config.option.liveconfig = cfg
-
-
-def pytest_runtest_setup(item):
-    if (list(item.iter_markers(name="ignored"))
-            and not item.config.getoption("ignored")):
-        pytest.skip("Ignored tests not requested, use --ignored")
-
-
-def pytest_report_header(config, startdir):
-    summary = []
-
-    t = tempfile.mktemp()
-    m = MonkeyPatch()
-    try:
-        m.setattr(sys.stdout, "write", lambda x: len(x))
-        ac = Account(t)
-        info = ac.get_info()
-        ac.shutdown()
-    finally:
-        m.undo()
-        os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={}'.format(
-         info['deltachat_core_version'],
-         info['sqlite_version'],
-     )])
-
-    cfg = config.option.liveconfig
-    if cfg:
-        if "#" in cfg:
-            url, token = cfg.split("#", 1)
-            summary.append('Liveconfig provider: {}#<token ommitted>'.format(url))
-        else:
-            summary.append('Liveconfig file: {}'.format(cfg))
-    return summary
-
-
-@pytest.fixture(scope="session")
-def data():
-    class Data:
-        def __init__(self):
-            self.path = os.path.join(os.path.dirname(__file__), "data")
-
-        def get_path(self, bn):
-            fn = os.path.join(self.path, bn)
-            assert os.path.exists(fn)
-            return fn
-    return Data()
-
-
-class SessionLiveConfigFromFile:
-    def __init__(self, fn):
-        self.fn = fn
-        self.configlist = []
-        for line in open(fn):
-            if line.strip() and not line.strip().startswith('#'):
-                d = {}
-                for part in line.split():
-                    name, value = part.split("=")
-                    d[name] = value
-                self.configlist.append(d)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-class SessionLiveConfigFromURL:
-    def __init__(self, url):
-        self.configlist = []
-        self.url = url
-
-    def get(self, index):
-        try:
-            return self.configlist[index]
-        except IndexError:
-            assert index == len(self.configlist), index
-            res = requests.post(self.url)
-            if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
-            d = res.json()
-            config = dict(addr=d["email"], mail_pw=d["password"])
-            self.configlist.append(config)
-            return config
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-@pytest.fixture(scope="session")
-def session_liveconfig(request):
-    liveconfig_opt = request.config.option.liveconfig
-    if liveconfig_opt:
-        if liveconfig_opt.startswith("http"):
-            return SessionLiveConfigFromURL(liveconfig_opt)
-        else:
-            return SessionLiveConfigFromFile(liveconfig_opt)
-
-
-@pytest.fixture(scope='session')
-def datadir():
-    """The py.path.local object of the test-data/ directory."""
-    for path in reversed(py.path.local(__file__).parts()):
-        datadir = path.join('test-data')
-        if datadir.isdir():
-            return datadir
-    else:
-        pytest.skip('test-data directory not found')
-
-
-@pytest.fixture
-def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
-
-    class AccountMaker:
-        def __init__(self):
-            self.live_count = 0
-            self.offline_count = 0
-            self._finalizers = []
-            self.init_time = time.time()
-            self._generated_keys = ["alice", "bob", "charlie",
-                                    "dom", "elena", "fiona"]
-
-        def finalize(self):
-            while self._finalizers:
-                fin = self._finalizers.pop()
-                fin()
-
-        def make_account(self, path, logid):
-            ac = Account(path, logid=logid)
-            self._finalizers.append(ac.shutdown)
-            return ac
-
-        def get_unconfigured_account(self):
-            self.offline_count += 1
-            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(2)
-            return ac
-
-        def _preconfigure_key(self, account, addr):
-            # Only set a key if we haven't used it yet for another account.
-            if self._generated_keys:
-                keyname = self._generated_keys.pop(0)
-                fname_pub = "key/{name}-public.asc".format(name=keyname)
-                fname_sec = "key/{name}-secret.asc".format(name=keyname)
-                account._preconfigure_keypair(addr,
-                                              datadir.join(fname_pub).read(),
-                                              datadir.join(fname_sec).read())
-
-        def get_configured_offline_account(self):
-            ac = self.get_unconfigured_account()
-
-            # do a pseudo-configured account
-            addr = "addr{}@offline.org".format(self.offline_count)
-            ac.set_config("addr", addr)
-            self._preconfigure_key(ac, addr)
-            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
-            ac.set_config("mail_pw", "123")
-            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
-            lib.dc_set_config(ac._dc_context, b"configured", b"1")
-            return ac
-
-        def get_online_config(self, pre_generated_key=True):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
-            configdict = session_liveconfig.get(self.live_count)
-            self.live_count += 1
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
-
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            if pre_generated_key:
-                self._preconfigure_key(ac, configdict['addr'])
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(30)
-            return ac, dict(configdict)
-
-        def get_online_configuring_account(self, mvbox=False, sentbox=False,
-                                           pre_generated_key=True, config={}):
-            ac, configdict = self.get_online_config(
-                pre_generated_key=pre_generated_key)
-            configdict.update(config)
-            ac.configure(**configdict)
-            return ac
-
-        def get_one_online_account(self, pre_generated_key=True):
-            ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key)
-            wait_successful_IMAP_SMTP_connection(ac1)
-            wait_configuration_progress(ac1, 1000)
-            ac1.start_threads()
-            return ac1
-
-        def get_two_online_accounts(self):
-            ac1 = self.get_online_configuring_account()
-            ac2 = self.get_online_configuring_account()
-            wait_successful_IMAP_SMTP_connection(ac1)
-            wait_configuration_progress(ac1, 1000)
-            ac1.start_threads()
-            wait_successful_IMAP_SMTP_connection(ac2)
-            wait_configuration_progress(ac2, 1000)
-            ac2.start_threads()
-
-            return ac1, ac2
-
-        def clone_online_account(self, account, pre_generated_key=True):
-            self.live_count += 1
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            if pre_generated_key:
-                self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(30)
-            ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"))
-
-            return ac
-
-    am = AccountMaker()
-    request.addfinalizer(am.finalize)
-    return am
-
-
-@pytest.fixture
-def tmp_db_path(tmpdir):
-    return tmpdir.join("test.db").strpath
-
-
-@pytest.fixture
-def lp():
-    class Printer:
-        def sec(self, msg):
-            print()
-            print("=" * 10, msg, "=" * 10)
-
-        def step(self, msg):
-            print("-" * 5, "step " + msg, "-" * 5)
-    return Printer()
-
 
 def wait_configuration_progress(account, min_target, max_target=1001, check_error=True):
     min_target = min(min_target, max_target)
     while 1:
-        evt_name, data1, data2 = \
+        event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-            account._evlogger.get_matching("DC_EVENT_CONFIGURE_PROGRESS", check_error=check_error)
+        if event.data1 >= min_target and event.data1 <= max_target:
-        if data1 >= min_target and data1 <= max_target:
             print("** CONFIG PROGRESS {}".format(min_target), account)
             break
 
 
 def wait_securejoin_inviter_progress(account, target):
     while 1:
-        evt_name, data1, data2 = \
+        event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-            account._evlogger.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+        if event.data2 >= target:
-        if data2 >= target:
             print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
             break
-
-
-def wait_successful_IMAP_SMTP_connection(account):
-    imap_ok = smtp_ok = False
-    while not imap_ok or not smtp_ok:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED")
-        if evt_name == "DC_EVENT_IMAP_CONNECTED":
-            imap_ok = True
-            print("** IMAP OK", account)
-        if evt_name == "DC_EVENT_SMTP_CONNECTED":
-            smtp_ok = True
-            print("** SMTP OK", account)
-    print("** IMAP and SMTP logins successful", account)
-
-
-def wait_msgs_changed(account, chat_id, msg_id=None):
-    ev = account._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-    assert ev[1] == chat_id
-    if msg_id is not None:
-        assert ev[2] == msg_id
-    return ev[2]

python/tests/data/key

@@ -0,0 +1 @@
+../../../test-data/key

python/tests/test_increation.py

@@ -6,10 +6,18 @@ import shutil
 import pytest
 from filecmp import cmp
 
-from conftest import wait_configuration_progress, wait_msgs_changed
+from conftest import wait_configuration_progress
 from deltachat import const
 
 
+def wait_msgs_changed(account, chat_id, msg_id=None):
+    ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    assert ev.data1 == chat_id
+    if msg_id is not None:
+        assert ev.data2 == msg_id
+    return ev.data2
+
+
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
@@ -97,22 +105,22 @@ class TestOnlineInCreation:
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
         lp.sec("wait for the messages to be delivered to SMTP")
-        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev[1] == chat.id
+        assert ev.data1 == chat.id
-        assert ev[2] == prepared_original.id
+        assert ev.data2 == prepared_original.id
-        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev[1] == chat2.id
+        assert ev.data1 == chat2.id
-        assert ev[2] == forwarded_id
+        assert ev.data2 == forwarded_id
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1[1] > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1[2])
+        received_original = ac2.get_message_by_id(ev1.data2)
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2[1] > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2[1] != ev1[1]
+        assert ev2.data1 != ev1.data1
-        received_copy = ac2.get_message_by_id(ev2[2])
+        received_copy = ac2.get_message_by_id(ev2.data2)
         assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_lowlevel.py

@@ -1,7 +1,10 @@
 from __future__ import print_function
+
 import threading
 import time
-from deltachat import capi, cutil, const, set_context_callback, clear_context_callback, py_dc_callback
+from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
+from deltachat import register_global_plugin
+from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
 # from deltachat.account import EventLogger
@@ -47,51 +50,37 @@ def test_callback_None2int():
     clear_context_callback(ctx)
 
 
-def test_start_stop_event_thread_basic():
+def test_dc_close_events(tmpdir, acfactory):
-    print("1")
+    ac1 = acfactory.get_unconfigured_account()
-    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
-    print("2")
-    ev_thread = EventThread(ctx)
-    print("3 -- starting event thread")
-    ev_thread.start()
-    print("4 -- stopping event thread")
-    ev_thread.stop()
 
+    # register after_shutdown function
+    shutdowns = []
 
-# FIXME: EventLogger doesn't work without an account anymore
+    class ShutdownPlugin:
-# def test_dc_close_events(tmpdir):
+        @global_hookimpl
-#     ctx = ffi.gc(
+        def dc_account_after_shutdown(self, account):
-#         capi.lib.dc_context_new(ffi.NULL, ffi.NULL),
+            assert account._dc_context is None
-#         lib.dc_context_unref,
+            shutdowns.append(account)
-#     )
+    register_global_plugin(ShutdownPlugin())
-#     evlog = EventLogger(ctx)
+    assert hasattr(ac1, "_dc_context")
-#     evlog.set_timeout(5)
+    ac1.shutdown()
-#     set_context_callback(
+    assert shutdowns == [ac1]
-#         ctx,
-#         lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
-#     )
-#     ev_thread = EventThread(ctx)
-#     ev_thread.start()
 
-#     p = tmpdir.join("hello.db")
+    def find(info_string):
-#     lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
+        evlog = ac1._evtracker
-#     capi.lib.dc_close(ctx)
+        while 1:
+            ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
+            data2 = ev.data2
+            if info_string in data2:
+                return
+            else:
+                print("skipping event", ev)
 
-#     def find(info_string):
+    find("disconnecting inbox-thread")
-#         while 1:
+    find("disconnecting sentbox-thread")
-#             ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
+    find("disconnecting mvbox-thread")
-#             data2 = ev[2]
+    find("disconnecting SMTP")
-#             if info_string in data2:
+    find("Database closed")
-#                 return
-#             else:
-#                 print("skipping event", *ev)
-
-#     find("disconnecting inbox-thread")
-#     find("disconnecting sentbox-thread")
-#     find("disconnecting mvbox-thread")
-#     find("disconnecting SMTP")
-#     find("Database closed")
-#     ev_thread.stop()
 
 
 def test_wrong_db(tmpdir):
@@ -136,10 +125,10 @@ def test_markseen_invalid_message_ids(acfactory):
     contact1 = ac1.create_contact(email="some1@example.com", name="some1")
     chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
-    ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
 
 
 def test_get_special_message_id_returns_empty_message(acfactory):

python/tox.ini

@@ -7,14 +7,14 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs:tests}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
-    DCC_NEW_TMP_EMAIL
+    DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
 deps = 
@@ -41,13 +41,13 @@ deps =
     restructuredtext_lint 
 commands =
     flake8 src/deltachat
-    flake8 tests/
+    flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx==2.2.0
+    sphinx
     breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
@@ -67,7 +67,6 @@ commands =
 
 [pytest]
 addopts = -v -ra --strict-markers
-python_files = tests/test_*.py
 norecursedirs = .tox 
 xfail_strict=true
 timeout = 90

src/aheader.rs

@@ -127,14 +127,10 @@ impl str::FromStr for Aheader {
             .split(';')
             .filter_map(|a| {
                 let attribute: Vec<&str> = a.trim().splitn(2, '=').collect();
-                if attribute.len() < 2 {
+                match &attribute[..] {
-                    return None;
+                    [key, value] => Some((key.trim().to_string(), value.trim().to_string())),
+                    _ => None,
                 }
-
-                Some((
-                    attribute[0].trim().to_string(),
-                    attribute[1].trim().to_string(),
-                ))
             })
             .collect();
 

src/blob.rs

@@ -7,13 +7,13 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use self::image::GenericImageView;
+use image::GenericImageView;
+use thiserror::Error;
+
 use crate::constants::AVATAR_SIZE;
 use crate::context::Context;
 use crate::events::Event;
 
-extern crate image;
-
 /// Represents a file in the blob directory.
 ///
 /// The object has a name, which will always be valid UTF-8.  Having a
@@ -58,7 +58,6 @@ impl<'a> BlobObject<'a> {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
                 cause: err,
-                backtrace: failure::Backtrace::new(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -91,7 +90,6 @@ impl<'a> BlobObject<'a> {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
-                            backtrace: failure::Backtrace::new(),
                         });
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
@@ -104,7 +102,6 @@ impl<'a> BlobObject<'a> {
             blobdir: dir.to_path_buf(),
             blobname: name,
             cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
-            backtrace: failure::Backtrace::new(),
         })
     }
 
@@ -132,7 +129,6 @@ impl<'a> BlobObject<'a> {
                     blobname: String::from(""),
                     src: src.as_ref().to_path_buf(),
                     cause: err,
-                    backtrace: failure::Backtrace::new(),
                 })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
         let (name, mut dst_file) =
@@ -149,7 +145,6 @@ impl<'a> BlobObject<'a> {
                 blobname: name_for_err,
                 src: src.as_ref().to_path_buf(),
                 cause: err,
-                backtrace: failure::Backtrace::new(),
             });
         }
         let blob = BlobObject {
@@ -209,17 +204,14 @@ impl<'a> BlobObject<'a> {
             .map_err(|_| BlobError::WrongBlobdir {
                 blobdir: context.get_blobdir().to_path_buf(),
                 src: path.as_ref().to_path_buf(),
-                backtrace: failure::Backtrace::new(),
             })?;
         if !BlobObject::is_acceptible_blob_name(&rel_path) {
             return Err(BlobError::WrongName {
                 blobname: path.as_ref().to_path_buf(),
-                backtrace: failure::Backtrace::new(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
             blobname: path.as_ref().to_path_buf(),
-            backtrace: failure::Backtrace::new(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -247,7 +239,6 @@ impl<'a> BlobObject<'a> {
         if !BlobObject::is_acceptible_blob_name(&name) {
             return Err(BlobError::WrongName {
                 blobname: PathBuf::from(name),
-                backtrace: failure::Backtrace::new(),
             });
         }
         Ok(BlobObject {
@@ -370,7 +361,6 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
-            backtrace: failure::Backtrace::new(),
         })?;
 
         if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
@@ -383,7 +373,6 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
-            backtrace: failure::Backtrace::new(),
         })?;
 
         Ok(())
@@ -397,98 +386,41 @@ impl<'a> fmt::Display for BlobObject<'a> {
 }
 
 /// Errors for the [BlobObject].
-#[derive(Fail, Debug)]
+#[derive(Debug, Error)]
 pub enum BlobError {
+    #[error("Failed to create blob {blobname} in {}", .blobdir.display())]
     CreateFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[cause]
+        #[source]
         cause: std::io::Error,
-        backtrace: failure::Backtrace,
     },
+    #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())]
     WriteFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[cause]
+        #[source]
         cause: std::io::Error,
-        backtrace: failure::Backtrace,
     },
+    #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
         blobdir: PathBuf,
         blobname: String,
         src: PathBuf,
-        #[cause]
+        #[source]
         cause: std::io::Error,
-        backtrace: failure::Backtrace,
     },
+    #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())]
     RecodeFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[cause]
+        #[source]
         cause: image::ImageError,
-        backtrace: failure::Backtrace,
     },
-    WrongBlobdir {
+    #[error("File path {} is not in {}", .src.display(), .blobdir.display())]
-        blobdir: PathBuf,
+    WrongBlobdir { blobdir: PathBuf, src: PathBuf },
-        src: PathBuf,
+    #[error("Blob has a badname {}", .blobname.display())]
-        backtrace: failure::Backtrace,
+    WrongName { blobname: PathBuf },
-    },
-    WrongName {
-        blobname: PathBuf,
-        backtrace: failure::Backtrace,
-    },
-}
-
-// Implementing Display is done by hand because the failure
-// #[fail(display = "...")] syntax does not allow using
-// `blobdir.display()`.
-impl fmt::Display for BlobError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        // Match on the data rather than kind, they are equivalent for
-        // identifying purposes but contain the actual data we need.
-        match &self {
-            BlobError::CreateFailure {
-                blobdir, blobname, ..
-            } => write!(
-                f,
-                "Failed to create blob {} in {}",
-                blobname,
-                blobdir.display()
-            ),
-            BlobError::WriteFailure {
-                blobdir, blobname, ..
-            } => write!(
-                f,
-                "Failed to write data to blob {} in {}",
-                blobname,
-                blobdir.display()
-            ),
-            BlobError::CopyFailure {
-                blobdir,
-                blobname,
-                src,
-                ..
-            } => write!(
-                f,
-                "Failed to copy data from {} to blob {} in {}",
-                src.display(),
-                blobname,
-                blobdir.display(),
-            ),
-            BlobError::RecodeFailure {
-                blobdir, blobname, ..
-            } => write!(f, "Failed to recode {} in {}", blobname, blobdir.display(),),
-            BlobError::WrongBlobdir { blobdir, src, .. } => write!(
-                f,
-                "File path {} is not in blobdir {}",
-                src.display(),
-                blobdir.display(),
-            ),
-            BlobError::WrongName { blobname, .. } => {
-                write!(f, "Blob has a bad name: {}", blobname.display(),)
-            }
-        }
-    }
 }
 
 #[cfg(test)]

src/chat.rs

@@ -15,7 +15,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::Error;
+use crate::error::{bail, ensure, format_err, Error};
 use crate::events::Event;
 use crate::job::{self, Action};
 use crate::message::{self, InvalidMsgId, Message, MessageState, MsgId};
@@ -340,13 +340,12 @@ impl ChatId {
                     time(),
                     msg.viewtype,
                     MessageState::OutDraft,
-                    msg.text.as_ref().cloned().unwrap_or_default(),
+                    msg.text.as_deref().unwrap_or(""),
                     msg.param.to_string(),
                     1,
                 ],
             )
             .await?;
-
         Ok(())
     }
 
@@ -399,6 +398,62 @@ impl ChatId {
         Ok(self.get_param(context).await?.exists(Param::Devicetalk))
     }
 
+    async fn parent_query<T, F>(
+        self,
+        context: &Context,
+        fields: &str,
+        f: F,
+    ) -> sql::Result<Option<T>>
+    where
+        F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
+    {
+        let sql = &context.sql;
+        let query = format!(
+            "SELECT {} \
+             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) AND NOT hidden \
+             ORDER BY timestamp DESC, id DESC \
+             LIMIT 1;",
+            fields
+        );
+        sql.query_row_optional(
+            query,
+            paramsv![
+                self,
+                MessageState::OutPreparing,
+                MessageState::OutDraft,
+                MessageState::OutPending,
+                MessageState::OutFailed
+            ],
+            f,
+        )
+        .await
+    }
+
+    async fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> {
+        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
+        self.parent_query(
+            context,
+            "rfc724_mid, mime_in_reply_to, mime_references",
+            collect,
+        )
+        .await
+        .ok()
+        .flatten()
+    }
+
+    async fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
+        let collect = |row: &rusqlite::Row| Ok(row.get(0)?);
+        let packed: Option<String> = self.parent_query(context, "param", collect).await?;
+
+        if let Some(ref packed) = packed {
+            let param = packed.parse::<Params>()?;
+            Ok(param.exists(Param::GuaranteeE2ee))
+        } else {
+            // No messages
+            Ok(false)
+        }
+    }
+
     /// Bad evil escape hatch.
     ///
     /// Avoid using this, eventually types should be cleaned up enough
@@ -580,44 +635,6 @@ impl Chat {
         &self.name
     }
 
-    pub async fn get_subtitle(&self, context: &Context) -> String {
-        // returns either the address or the number of chat members
-
-        if self.typ == Chattype::Single && self.param.exists(Param::Selftalk) {
-            return context
-                .stock_str(StockMessage::SelfTalkSubTitle)
-                .await
-                .into();
-        }
-
-        if self.typ == Chattype::Single {
-            return context
-                .sql
-                .query_get_value(
-                    context,
-                    "SELECT c.addr
-                       FROM chats_contacts cc
-                       LEFT JOIN contacts c ON c.id=cc.contact_id
-                      WHERE cc.chat_id=?;",
-                    paramsv![self.id],
-                )
-                .await
-                .unwrap_or_else(|| "Err".into());
-        }
-
-        if self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup {
-            if self.id.is_deaddrop() {
-                return context.stock_str(StockMessage::DeadDrop).await.into();
-            }
-            let cnt = get_chat_contact_cnt(context, self.id).await;
-            return context
-                .stock_string_repl_int(StockMessage::Member, cnt as i32)
-                .await;
-        }
-
-        "Err".to_string()
-    }
-
     fn parent_query(fields: &str) -> String {
         // Check for server_uid guarantees that we don't
         // select a draft or undelivered message.
@@ -717,7 +734,6 @@ impl Chat {
                 .await
                 .map(Into::into)
                 .unwrap_or_else(std::path::PathBuf::new),
-            subtitle: self.get_subtitle(context).await,
             draft,
             is_muted: self.is_muted(),
         })
@@ -873,7 +889,7 @@ impl Chat {
                     }
                 }
 
-                if can_encrypt && (all_mutual || self.parent_is_encrypted(context).await?) {
+                if can_encrypt && (all_mutual || self.id.parent_is_encrypted(context).await?) {
                     msg.param.set_int(Param::GuaranteeE2ee, 1);
                 }
             }
@@ -888,7 +904,7 @@ impl Chat {
             // we do not set In-Reply-To/References in this case.
             if !self.is_self_talk() {
                 if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) =
-                    self.get_parent_mime_headers(context).await
+                    self.id.get_parent_mime_headers(context).await
                 {
                     if !parent_rfc724_mid.is_empty() {
                         new_in_reply_to = parent_rfc724_mid.clone();
@@ -1067,9 +1083,6 @@ pub struct ChatInfo {
     /// currently.
     pub profile_image: std::path::PathBuf,
 
-    /// Subtitle for the chat.
-    pub subtitle: String,
-
     /// The draft message text.
     ///
     /// If the chat has not draft this is an empty string.
@@ -1535,7 +1548,7 @@ pub async fn get_chat_msgs(
     flags: u32,
     marker1before: Option<MsgId>,
 ) -> Vec<MsgId> {
-    match hide_device_expired_messages(context).await {
+    match delete_device_expired_messages(context).await {
         Err(err) => warn!(context, "Failed to delete expired messages: {}", err),
         Ok(messages_deleted) => {
             if messages_deleted {
@@ -1700,11 +1713,11 @@ pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-/// Hides messages which are expired according to "delete_device_after" setting.
+/// Deletes messages which are expired according to "delete_device_after" setting.
 ///
-/// Returns true if any message is hidden, so event can be emitted. If nothing
+/// Returns true if any message is deleted, so event can be emitted. If nothing
-/// has been hidden, returns false.
+/// has been deleted, returns false.
-pub async fn hide_device_expired_messages(context: &Context) -> Result<bool, Error> {
+pub async fn delete_device_expired_messages(context: &Context) -> Result<bool, Error> {
     if let Some(delete_device_after) = context.get_config_delete_device_after().await {
         let threshold_timestamp = time() - delete_device_after;
 
@@ -1717,7 +1730,7 @@ pub async fn hide_device_expired_messages(context: &Context) -> Result<bool, Err
             .unwrap_or_default()
             .0;
 
-        // Hide expired messages
+        // Delete expired messages
         //
         // Only update the rows that have to be updated, to avoid emitting
         // unnecessary "chat modified" events.
@@ -1725,13 +1738,14 @@ pub async fn hide_device_expired_messages(context: &Context) -> Result<bool, Err
             .sql
             .execute(
                 "UPDATE msgs \
-             SET txt = 'DELETED', hidden = 1 \
+             SET txt = 'DELETED', chat_id = ? \
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
              AND chat_id != ? \
              AND NOT hidden",
                 paramsv![
+                    DC_CHAT_ID_TRASH,
                     threshold_timestamp,
                     DC_CHAT_ID_LAST_SPECIAL,
                     self_chat_id,
@@ -1822,17 +1836,17 @@ pub async fn get_next_media(
             msg_type3,
         )
         .await;
-        for i in 0..list.len() {
+        for (i, msg_id) in list.iter().enumerate() {
-            if curr_msg_id == list[i] {
+            if curr_msg_id == *msg_id {
                 match direction {
                     Direction::Forward => {
                         if i + 1 < list.len() {
-                            ret = Some(list[i + 1]);
+                            ret = list.get(i + 1).copied();
                         }
                     }
                     Direction::Backward => {
                         if i >= 1 {
-                            ret = Some(list[i - 1]);
+                            ret = list.get(i - 1).copied();
                         }
                     }
                 }
@@ -1918,38 +1932,56 @@ pub async fn create_group_chat(
     Ok(chat_id)
 }
 
+/// add a contact to the chats_contact table
 pub(crate) async fn add_to_chat_contacts_table(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
 ) -> bool {
-    // add a contact to a chat; the function does not check the type or if any of the record exist or are already
+    match context
-    // added to the chat!
-    context
         .sql
         .execute(
             "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)",
             paramsv![chat_id, contact_id as i32],
         )
         .await
-        .is_ok()
+    {
+        Ok(_) => true,
+        Err(err) => {
+            error!(
+                context,
+                "could not add {} to chat {} table: {}", contact_id, chat_id, err
+            );
+
+            false
+        }
+    }
 }
 
+/// remove a contact from the chats_contact table
 pub(crate) async fn remove_from_chat_contacts_table(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
 ) -> bool {
-    // remove a contact from the chats_contact table unconditionally
+    match context
-    // the function does not check the type or if the record exist
-    context
         .sql
         .execute(
             "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?",
             paramsv![chat_id, contact_id as i32],
         )
         .await
-        .is_ok()
+    {
+        Ok(_) => true,
+        Err(_) => {
+            warn!(
+                context,
+                "could not remove contact {:?} from chat {:?}", contact_id, chat_id
+            );
+
+            false
+        }
+    }
 }
 
 /// Adds a contact to the chat.
@@ -2049,15 +2081,7 @@ pub(crate) async fn add_contact_to_chat_ex(
         msg.param.set(Param::Arg, contact.get_addr());
         msg.param.set_int(Param::Arg2, from_handshake.into());
         msg.id = send_msg(context, chat_id, &mut msg).await?;
-        context.call_cb(Event::MsgsChanged {
-            chat_id,
-            msg_id: msg.id,
-        });
     }
-    context.call_cb(Event::MsgsChanged {
-        chat_id,
-        msg_id: MsgId::new(0),
-    });
     context.call_cb(Event::ChatModified(chat_id));
     Ok(true)
 }
@@ -2211,19 +2235,18 @@ pub async fn set_muted(
     duration: MuteDuration,
 ) -> Result<(), Error> {
     ensure!(!chat_id.is_special(), "Invalid chat ID");
-    if real_group_exists(context, chat_id).await
+    if context
-        && context
+        .sql
-            .sql
+        .execute(
-            .execute(
+            "UPDATE chats SET muted_until=? WHERE id=?;",
-                "UPDATE chats SET muted_until=? WHERE id=?;",
+            paramsv![duration, chat_id],
-                paramsv![duration, chat_id],
+        )
-            )
+        .await
-            .await
+        .is_ok()
-            .is_ok()
     {
         context.call_cb(Event::ChatModified(chat_id));
     } else {
-        bail!("Failed to set name");
+        bail!("Failed to set mute duration, chat might not exist -");
     }
     Ok(())
 }
@@ -2257,8 +2280,7 @@ pub async fn remove_contact_from_chat(
                         "Cannot remove contact from chat; self not in group.".into()
                     )
                 );
-            } else if remove_from_chat_contacts_table(context, chat_id, contact_id).await {
+            } else {
-                /* we should respect this - whatever we send to the group, it gets discarded anyway! */
                 if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
                     if chat.is_promoted() {
                         msg.viewtype = Viewtype::Text;
@@ -2289,14 +2311,20 @@ pub async fn remove_contact_from_chat(
                         msg.param.set_cmd(SystemMessage::MemberRemovedFromGroup);
                         msg.param.set(Param::Arg, contact.get_addr());
                         msg.id = send_msg(context, chat_id, &mut msg).await?;
-                        context.call_cb(Event::MsgsChanged {
-                            chat_id,
-                            msg_id: msg.id,
-                        });
                     }
                 }
+                // we remove the member from the chat after constructing the
+                // to-be-send message. If between send_msg() and here the
+                // process dies the user will have to re-do the action.  It's
+                // better than the other way round: you removed
+                // someone from DB but no peer or device gets to know about it and
+                // group membership is thus different on different devices.
+                // Note also that sending a message needs all recipients
+                // in order to correctly determine encryption so if we
+                // removed it first, it would complicate the
+                // check/encryption logic.
+                success = remove_from_chat_contacts_table(context, chat_id, contact_id).await;
                 context.call_cb(Event::ChatModified(chat_id));
-                success = true;
             }
         }
     }
@@ -2781,7 +2809,6 @@ mod tests {
                 "is_sending_locations": false,
                 "color": 15895624,
                 "profile_image": "",
-                "subtitle": "bob@example.com",
                 "draft": "",
                 "is_muted": false
             }
@@ -3402,4 +3429,18 @@ mod tests {
             false
         );
     }
+
+    #[async_std::test]
+    async fn test_parent_is_encrypted() {
+        let t = dummy_context().await;
+        let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
+            .await
+            .unwrap();
+        assert!(!chat_id.parent_is_encrypted(&t.ctx).await.unwrap());
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("hello".to_string()));
+        chat_id.set_draft(&t.ctx, Some(&mut msg)).await;
+        assert!(!chat_id.parent_is_encrypted(&t.ctx).await.unwrap());
+    }
 }

src/chatlist.rs

@@ -5,7 +5,7 @@ use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
-use crate::error::Result;
+use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock::StockMessage;
@@ -92,9 +92,14 @@ impl Chatlist {
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
+        let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
+        let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
+        let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
+        let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
+
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
-        // messages get hidden to avoid reloading the same chatlist.
+        // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = hide_device_expired_messages(context).await {
+        if let Err(err) = delete_device_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
@@ -111,7 +116,7 @@ impl Chatlist {
                 .map_err(Into::into)
         };
 
-        let skip_id = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+        let skip_id = if flag_for_forwarding {
             chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
                 .await
                 .unwrap_or_default()
@@ -156,7 +161,7 @@ impl Chatlist {
                 process_row,
                 process_rows,
             ).await?
-        } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
+        } else if flag_archived_only {
             // show archived chats
             // (this includes the archived device-chat; we could skip it,
             // however, then the number of archived chats do not match, which might be even more irritating.
@@ -218,7 +223,7 @@ impl Chatlist {
                 .await?
         } else {
             //  show normal chatlist
-            let sort_id_up = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+            let sort_id_up = if flag_for_forwarding {
                 chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
                     .await
                     .unwrap_or_default()
@@ -245,10 +250,10 @@ impl Chatlist {
                 process_row,
                 process_rows,
             ).await?;
-            if 0 == listflags & DC_GCL_NO_SPECIALS {
+            if !flag_no_specials {
                 if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context).await
                 {
-                    if 0 == listflags & DC_GCL_FOR_FORWARDING {
+                    if !flag_for_forwarding {
                         ids.insert(
                             0,
                             (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
@@ -261,7 +266,7 @@ impl Chatlist {
         };
 
         if add_archived_link_item && dc_get_archived_cnt(context).await > 0 {
-            if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
+            if ids.is_empty() && flag_add_alldone_hint {
                 ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
             ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
@@ -284,18 +289,20 @@ impl Chatlist {
     ///
     /// To get the message object from the message ID, use dc_get_chat().
     pub fn get_chat_id(&self, index: usize) -> ChatId {
-        if index >= self.ids.len() {
+        match self.ids.get(index) {
-            return ChatId::new(0);
+            Some((chat_id, _msg_id)) => *chat_id,
+            None => ChatId::new(0),
         }
-        self.ids[index].0
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
     pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
-        ensure!(index < self.ids.len(), "Chatlist index out of range");
+        match self.ids.get(index) {
-        Ok(self.ids[index].1)
+            Some((_chat_id, msg_id)) => Ok(*msg_id),
+            None => bail!("Chatlist index out of range"),
+        }
     }
 
     /// Get a summary for a chatlist index.
@@ -319,25 +326,27 @@ impl Chatlist {
         // Also, sth. as "No messages" would not work if the summary comes from a message.
         let mut ret = Lot::new();
 
-        if index >= self.ids.len() {
+        let (chat_id, lastmsg_id) = match self.ids.get(index) {
-            ret.text2 = Some("ErrBadChatlistIndex".to_string());
+            Some(ids) => ids,
-            return ret;
+            None => {
-        }
+                ret.text2 = Some("ErrBadChatlistIndex".to_string());
+                return ret;
+            }
+        };
 
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0).await {
+        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id).await {
             chat_loaded = chat;
             &chat_loaded
         } else {
             return ret;
         };
 
-        let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id).await {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {

src/config.rs

@@ -63,6 +63,9 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
+    #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
+    MediaQuality,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -251,9 +254,11 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
+    use crate::constants;
     use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
     use image::GenericImageView;
+    use num_traits::FromPrimitive;
     use std::fs::File;
     use std::io::Write;
 
@@ -328,4 +333,48 @@ mod tests {
         assert_eq!(img.width(), AVATAR_SIZE);
         assert_eq!(img.height(), AVATAR_SIZE);
     }
+
+    #[async_std::test]
+    async fn test_selfavatar_copy_without_recode() {
+        let t = dummy_context().await;
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
+        assert!(!avatar_blob.exists().await);
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert_eq!(
+            std::fs::metadata(&avatar_blob).unwrap().len(),
+            avatar_bytes.len() as u64
+        );
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+    }
+
+    #[async_std::test]
+    async fn test_media_quality_config_option() {
+        let t = dummy_context().await;
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        assert_eq!(media_quality, 0);
+        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+        assert_eq!(media_quality, constants::MediaQuality::Balanced);
+
+        t.ctx
+            .set_config(Config::MediaQuality, Some("1"))
+            .await
+            .unwrap();
+
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        assert_eq!(media_quality, 1);
+        assert_eq!(constants::MediaQuality::Worse as i32, 1);
+        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+        assert_eq!(media_quality, constants::MediaQuality::Worse);
+    }
 }

src/configure/auto_mozilla.rs

@@ -1,7 +1,6 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
 //! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::constants::*;
@@ -9,33 +8,7 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-
+use super::Error;
-#[derive(Debug, Fail)]
-pub enum Error {
-    #[fail(display = "Invalid email address: {:?}", _0)]
-    InvalidEmailAddress(String),
-
-    #[fail(display = "XML error at position {}", position)]
-    InvalidXml {
-        position: usize,
-        #[cause]
-        error: quick_xml::Error,
-    },
-
-    #[fail(display = "Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
-    #[fail(display = "Failed to get URL {}", _0)]
-    ReadUrlError(#[cause] super::read_url::Error),
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl From<super::read_url::Error> for Error {
-    fn from(err: super::read_url::Error) -> Error {
-        Error::ReadUrlError(err)
-    }
-}
 
 #[derive(Debug)]
 struct MozAutoconfigure<'a> {
@@ -65,16 +38,16 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam> {
+fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
     let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
     // Split address into local part and domain part.
-    let p = in_emailaddr
+    let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
-        .find('@')
+    let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
-        .ok_or_else(|| Error::InvalidEmailAddress(in_emailaddr.to_string()))?;
+        [domain, local] => (local, domain),
-    let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p);
+        _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
-    let in_emaildomain = &in_emaildomain[1..];
+    };
 
     let mut moz_ac = MozAutoconfigure {
         in_emailaddr,
@@ -125,7 +98,7 @@ pub fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam> {
+) -> Result<LoginParam, Error> {
     let xml_raw = read_url(context, url)?;
 
     let res = parse_xml(&param_in.addr, &xml_raw);

src/configure/auto_outlook.rs

@@ -1,6 +1,5 @@
 //! Outlook's Autodiscover
 
-use quick_xml;
 use quick_xml::events::BytesEnd;
 
 use crate::constants::*;
@@ -8,33 +7,7 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-
+use super::Error;
-#[derive(Debug, Fail)]
-pub enum Error {
-    #[fail(display = "XML error at position {}", position)]
-    InvalidXml {
-        position: usize,
-        #[cause]
-        error: quick_xml::Error,
-    },
-
-    #[fail(display = "Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
-    #[fail(display = "Failed to get URL {}", _0)]
-    ReadUrlError(#[cause] super::read_url::Error),
-
-    #[fail(display = "Number of redirection is exceeded")]
-    RedirectionError,
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl From<super::read_url::Error> for Error {
-    fn from(err: super::read_url::Error) -> Error {
-        Error::ReadUrlError(err)
-    }
-}
 
 struct OutlookAutodiscover {
     pub out: LoginParam,
@@ -52,7 +25,7 @@ enum ParsingResult {
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult> {
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
     let mut outlk_ad = OutlookAutodiscover {
         out: LoginParam::new(),
         out_imap_set: false,
@@ -143,7 +116,7 @@ pub fn outlk_autodiscover(
     context: &Context,
     url: &str,
     _param_in: &LoginParam,
-) -> Result<LoginParam> {
+) -> Result<LoginParam, Error> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {

src/configure/mod.rs

@@ -4,6 +4,7 @@ mod auto_mozilla;
 mod auto_outlook;
 mod read_url;
 
+use anyhow::{bail, ensure, Result};
 use async_std::prelude::*;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
@@ -11,7 +12,6 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::{Error, Result};
 use crate::imap::Imap;
 use crate::login_param::{CertificateChecks, LoginParam};
 use crate::message::Message;
@@ -153,7 +153,7 @@ impl Context {
                 .ok();
 
             progress!(self, 0);
-            Err(Error::Message("Configure failed".to_string()))
+            bail!("Configure failed")
         }
     }
 }
@@ -636,6 +636,28 @@ async fn try_smtp_one_param(context: &Context, param: &LoginParam, smtp: &mut Sm
     Ok(())
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("Invalid email address: {0:?}")]
+    InvalidEmailAddress(String),
+
+    #[error("XML error at position {position}")]
+    InvalidXml {
+        position: usize,
+        #[source]
+        error: quick_xml::Error,
+    },
+
+    #[error("Bad or incomplete autoconfig")]
+    IncompleteAutoconfig(LoginParam),
+
+    #[error("Failed to get URL")]
+    ReadUrlError(#[from] self::read_url::Error),
+
+    #[error("Number of redirection is exceeded")]
+    RedirectionError,
+}
+
 #[cfg(test)]
 mod tests {
 

src/configure/read_url.rs

@@ -1,14 +1,12 @@
 use crate::context::Context;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "URL request error")]
+    #[error("URL request error")]
-    GetError(#[cause] reqwest::Error),
+    GetError(#[from] reqwest::Error),
 }
 
-pub type Result<T> = std::result::Result<T, Error>;
+pub fn read_url(context: &Context, url: &str) -> Result<String, Error> {
-
-pub fn read_url(context: &Context, url: &str) -> Result<String> {
     info!(context, "Requesting URL {}", url);
 
     match reqwest::blocking::Client::new()

src/constants.rs

@@ -57,6 +57,19 @@ impl Default for ShowEmails {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
+pub enum MediaQuality {
+    Balanced = 0,
+    Worse = 1,
+}
+
+impl Default for MediaQuality {
+    fn default() -> Self {
+        MediaQuality::Balanced // also change Config.MediaQuality props(default) on changes
+    }
+}
+
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(u8)]
 pub enum KeyGenType {
@@ -314,8 +327,6 @@ const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 const DC_STR_NOMESSAGES: usize = 1;
 const DC_STR_SELF: usize = 2;
 const DC_STR_DRAFT: usize = 3;
-const DC_STR_MEMBER: usize = 4;
-const DC_STR_CONTACT: usize = 6;
 const DC_STR_VOICEMESSAGE: usize = 7;
 const DC_STR_DEADDROP: usize = 8;
 const DC_STR_IMAGE: usize = 9;
@@ -347,7 +358,6 @@ const DC_STR_ARCHIVEDCHATS: usize = 40;
 const DC_STR_STARREDMSGS: usize = 41;
 const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
 const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
-const DC_STR_SELFTALK_SUBTITLE: usize = 50;
 const DC_STR_CANNOT_LOGIN: usize = 60;
 const DC_STR_SERVER_RESPONSE: usize = 61;
 const DC_STR_MSGACTIONBYUSER: usize = 62;

src/contact.rs

@@ -3,7 +3,6 @@
 use async_std::path::PathBuf;
 use deltachat_derive::*;
 use itertools::Itertools;
-use rusqlite;
 
 use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
@@ -11,10 +10,9 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::e2ee;
+use crate::error::{bail, ensure, format_err, Result};
-use crate::error::{Error, Result};
 use crate::events::Event;
-use crate::key::*;
+use crate::key::{DcKey, Key, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{MessageState, MsgId};
 use crate::mimeparser::AvatarAction;
@@ -22,9 +20,6 @@ use crate::param::*;
 use crate::peerstate::*;
 use crate::stock::StockMessage;
 
-/// Contacts with at least this origin value are shown in the contact list.
-const DC_ORIGIN_MIN_CONTACT_LIST: i32 = 0x100;
-
 /// An object representing a single contact in memory.
 ///
 /// The contact object is not updated.
@@ -92,6 +87,7 @@ pub enum Origin {
     UnhandledQrScan = 0x80,
 
     /// Reply-To: of incoming message of known sender
+    /// Contacts with at least this origin value are shown in the contact list.
     IncomingReplyTo = 0x100,
 
     /// Cc: of incoming message of known sender
@@ -285,7 +281,11 @@ impl Contact {
     ///
     /// To validate an e-mail address independently of the contact database
     /// use `dc_may_be_valid_addr()`.
-    pub async fn lookup_id_by_addr(context: &Context, addr: impl AsRef<str>) -> u32 {
+    pub async fn lookup_id_by_addr(
+        context: &Context,
+        addr: impl AsRef<str>,
+        min_origin: Origin,
+    ) -> u32 {
         if addr.as_ref().is_empty() {
             return 0;
         }
@@ -299,14 +299,13 @@ impl Contact {
         if addr_cmp(addr_normalized, addr_self) {
             return DC_CONTACT_ID_SELF;
         }
-
         context.sql.query_get_value(
             context,
             "SELECT id FROM contacts WHERE addr=?1 COLLATE NOCASE AND id>?2 AND origin>=?3 AND blocked=0;",
             paramsv![
                 addr_normalized,
                 DC_CONTACT_ID_LAST_SPECIAL as i32,
-                DC_ORIGIN_MIN_CONTACT_LIST,
+                min_origin as u32,
             ],
         ).await.unwrap_or_default()
     }
@@ -677,8 +676,6 @@ impl Contact {
             let peerstate = Peerstate::from_addr(context, &contact.addr).await;
             let loginparam = LoginParam::from_database(context, "configured_").await;
 
-            let mut self_key = Key::from_self_public(context, &loginparam.addr, &context.sql).await;
-
             if peerstate.is_some()
                 && peerstate
                     .as_ref()
@@ -694,16 +691,11 @@ impl Contact {
                     })
                     .await;
                 ret += &p;
-                if self_key.is_none() {
+                let self_key = Key::from(SignedPublicKey::load_self(context).await?);
-                    e2ee::ensure_secret_key_exists(context).await?;
-                    self_key = Key::from_self_public(context, &loginparam.addr, &context.sql).await;
-                }
                 let p = context.stock_str(StockMessage::FingerPrints).await;
                 ret += &format!(" {}:", p);
 
-                let fingerprint_self = self_key
+                let fingerprint_self = self_key.formatted_fingerprint();
-                    .map(|k| k.formatted_fingerprint())
-                    .unwrap_or_default();
                 let fingerprint_other_verified = peerstate
                     .peek_key(PeerstateVerifiedStatus::BidirectVerified)
                     .map(|k| k.formatted_fingerprint())
@@ -1160,10 +1152,10 @@ fn cat_fingerprint(
 impl Context {
     /// determine whether the specified addr maps to the/a self addr
     pub async fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        let self_addr = match self.get_config(Config::ConfiguredAddr).await {
+        let self_addr = self
-            Some(s) => s,
+            .get_config(Config::ConfiguredAddr)
-            None => return Err(Error::NotConfigured),
+            .await
-        };
+            .ok_or_else(|| format_err!("Not configured"))?;
 
         Ok(addr_cmp(self_addr, addr))
     }

src/context.rs

@@ -1,9 +1,10 @@
 //! Context module
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
 
+use anyhow::anyhow;
 use async_std::path::{Path, PathBuf};
 use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
 use crossbeam_queue::SegQueue;
@@ -12,16 +13,18 @@ use crate::chat::*;
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
+use crate::dc_tools::duration_to_str;
 use crate::error::*;
 use crate::events::Event;
 use crate::job::{self, Action};
-use crate::key::Key;
+use crate::key::{DcKey, Key, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
 use crate::message::{self, Message, MessengerMessage, MsgId};
 use crate::param::Params;
 use crate::scheduler::Scheduler;
 use crate::sql::Sql;
+use std::time::SystemTime;
 
 #[derive(Clone, Debug)]
 pub struct Context {
@@ -53,6 +56,8 @@ pub struct InnerContext {
     pub(crate) logs: SegQueue<Event>,
 
     pub(crate) scheduler: RwLock<Scheduler>,
+
+    creation_time: SystemTime,
 }
 
 #[derive(Debug)]
@@ -68,8 +73,8 @@ pub struct RunningState {
 /// actual keys and their values which will be present are not
 /// guaranteed.  Calling [Context::get_info] also includes information
 /// about the context on top of the information here.
-pub fn get_info() -> HashMap<&'static str, String> {
+pub fn get_info() -> BTreeMap<&'static str, String> {
-    let mut res = HashMap::new();
+    let mut res = BTreeMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
     res.insert("sqlite_version", rusqlite::version().to_string());
     res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
@@ -115,6 +120,7 @@ impl Context {
             translated_stockstrings: RwLock::new(HashMap::new()),
             logs: SegQueue::new(),
             scheduler: RwLock::new(Scheduler::Stopped),
+            creation_time: std::time::SystemTime::now(),
         };
 
         let ctx = Context {
@@ -168,7 +174,7 @@ impl Context {
     }
 
     pub fn get_next_event(&self) -> Result<Event> {
-        let event = self.logs.pop()?;
+        let event = self.logs.pop().map_err(|err| anyhow!("{}", err))?;
 
         Ok(event)
     }
@@ -237,7 +243,7 @@ impl Context {
      * UI chat/message related API
      ******************************************************************************/
 
-    pub async fn get_info(&self) -> HashMap<&'static str, String> {
+    pub async fn get_info(&self) -> BTreeMap<&'static str, String> {
         let unset = "0";
         let l = LoginParam::from_database(self, "").await;
         let l2 = LoginParam::from_database(self, "configured_").await;
@@ -266,13 +272,10 @@ impl Context {
             .sql
             .query_get_value(self, "SELECT COUNT(*) FROM acpeerstates;", paramsv![])
             .await;
-
+        let fingerprint_str = match SignedPublicKey::load_self(self).await {
-        let fingerprint_str =
+            Ok(key) => Key::from(key).fingerprint(),
-            if let Some(key) = Key::from_self_public(self, &l2.addr, &self.sql).await {
+            Err(err) => format!("<key failure: {}>", err),
-                key.fingerprint()
+        };
-            } else {
-                "<Not yet calculated>".into()
-            };
 
         let inbox_watch = self.get_config_int(Config::InboxWatch).await;
         let sentbox_watch = self.get_config_int(Config::SentboxWatch).await;
@@ -333,6 +336,9 @@ impl Context {
         );
         res.insert("fingerprint", fingerprint_str);
 
+        let elapsed = self.creation_time.elapsed();
+        res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
+
         res
     }
 

src/dc_receive_imf.rs

@@ -2,13 +2,15 @@ use itertools::join;
 use num_traits::FromPrimitive;
 use sha2::{Digest, Sha256};
 
+use mailparse::SingleInfo;
+
 use crate::chat::{self, Chat, ChatId};
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::Result;
+use crate::error::{bail, ensure, Result};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
 use crate::job::{self, Action};
@@ -30,6 +32,10 @@ enum CreateEvent {
 }
 
 /// Receive a message and add it to the database.
+///
+/// Returns an error on recoverable errors, e.g. database errors. In this case,
+/// message parsing should be retried later. If message itself is wrong, logs
+/// the error and returns success.
 pub async fn dc_receive_imf(
     context: &Context,
     imf_raw: &[u8],
@@ -53,10 +59,19 @@ pub async fn dc_receive_imf(
         println!("{}", String::from_utf8_lossy(imf_raw));
     }
 
-    let mut mime_parser = MimeMessage::from_bytes(context, imf_raw).await?;
+    let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw).await {
+        Err(err) => {
+            warn!(context, "dc_receive_imf: can't parse MIME: {}", err);
+            return Ok(());
+        }
+        Ok(mime_parser) => mime_parser,
+    };
 
     // we can not add even an empty record if we have no info whatsoever
-    ensure!(mime_parser.has_headers(), "No Headers Found");
+    if !mime_parser.has_headers() {
+        warn!(context, "dc_receive_imf: no headers found");
+        return Ok(());
+    }
 
     // the function returns the number of created messages in the database
     let mut chat_id = ChatId::new(0);
@@ -95,32 +110,26 @@ pub async fn dc_receive_imf(
     // we do not check Return-Path any more as this is unreliable, see
     // https://github.com/deltachat/deltachat-core/issues/150)
     let (from_id, from_id_blocked, incoming_origin) =
-        if let Some(field_from) = mime_parser.get(HeaderDef::From_) {
+        from_field_to_contact_id(context, &mime_parser.from).await?;
-            from_field_to_contact_id(context, field_from).await?
+
-        } else {
-            (0, false, Origin::Unknown)
-        };
     let incoming = from_id != DC_CONTACT_ID_SELF;
 
     let mut to_ids = ContactIds::new();
-    for header_def in &[HeaderDef::To, HeaderDef::Cc] {
+
-        if let Some(field) = mime_parser.get(header_def.clone()) {
+    to_ids.extend(
-            to_ids.extend(
+        &dc_add_or_lookup_contacts_by_address_list(
-                &dc_add_or_lookup_contacts_by_address_list(
+            context,
-                    context,
+            &mime_parser.recipients,
-                    &field,
+            if !incoming {
-                    if !incoming {
+                Origin::OutgoingTo
-                        Origin::OutgoingTo
+            } else if incoming_origin.is_known() {
-                    } else if incoming_origin.is_known() {
+                Origin::IncomingTo
-                        Origin::IncomingTo
+            } else {
-                    } else {
+                Origin::IncomingUnknownTo
-                        Origin::IncomingUnknownTo
+            },
-                    },
+        )
-                )
+        .await?,
-                .await?,
+    );
-            );
-        }
-    }
 
     // Add parts
 
@@ -238,11 +247,11 @@ pub async fn dc_receive_imf(
 /// Also returns whether it is blocked or not and its origin.
 pub async fn from_field_to_contact_id(
     context: &Context,
-    field_from: &str,
+    from_address_list: &[SingleInfo],
 ) -> Result<(u32, bool, Origin)> {
     let from_ids = dc_add_or_lookup_contacts_by_address_list(
         context,
-        &field_from,
+        from_address_list,
         Origin::IncomingUnknownFrom,
     )
     .await?;
@@ -253,7 +262,7 @@ pub async fn from_field_to_contact_id(
         if from_ids.len() > 1 {
             warn!(
                 context,
-                "mail has more than one From address, only using first: {:?}", field_from
+                "mail has more than one From address, only using first: {:?}", from_address_list
             );
         }
         let from_id = from_ids.get_index(0).cloned().unwrap_or_default();
@@ -266,7 +275,10 @@ pub async fn from_field_to_contact_id(
         }
         Ok((from_id, from_id_blocked, incoming_origin))
     } else {
-        warn!(context, "mail has an empty From header: {:?}", field_from);
+        warn!(
+            context,
+            "mail has an empty From header: {:?}", from_address_list
+        );
         // if there is no from given, from_id stays 0 which is just fine. These messages
         // are very rare, however, we have to add them to the database (they go to the
         // "deaddrop" chat) to avoid a re-download from the server. See also [**]
@@ -308,15 +320,16 @@ async fn add_parts(
     // check, if the mail is already in our database - if so, just update the folder/uid
     // (if the mail was moved around) and finish. (we may get a mail twice eg. if it is
     // moved between folders. make sure, this check is done eg. before securejoin-processing) */
-    if let Ok((old_server_folder, old_server_uid, _)) =
+    if let Some((old_server_folder, old_server_uid, _)) =
-        message::rfc724_mid_exists(context, &rfc724_mid).await
+        message::rfc724_mid_exists(context, &rfc724_mid).await?
     {
         if old_server_folder != server_folder.as_ref() || old_server_uid != server_uid {
             message::update_server_uid(context, &rfc724_mid, server_folder.as_ref(), server_uid)
                 .await;
         }
 
-        bail!("Message already in DB");
+        warn!(context, "Message already in DB");
+        return Ok(());
     }
 
     let mut msgrmsg = if mime_parser.has_chat_version() {
@@ -378,9 +391,11 @@ async fn add_parts(
                 }
                 Err(err) => {
                     *hidden = true;
+
                     context.bob.write().await.status = 0; // secure-join failed
                     context.stop_ongoing().await;
-                    error!(context, "Error in Secure-Join message handling: {}", err);
+                    warn!(context, "Error in Secure-Join message handling: {}", err);
+                    return Ok(());
                 }
             }
         }
@@ -501,7 +516,7 @@ async fn add_parts(
             msgrmsg = MessengerMessage::Yes; // avoid discarding by show_emails setting
             *chat_id = ChatId::new(0);
             allow_creation = true;
-            match observe_securejoin_on_other_device(context, mime_parser, to_id) {
+            match observe_securejoin_on_other_device(context, mime_parser, to_id).await {
                 Ok(securejoin::HandshakeMessage::Done)
                 | Ok(securejoin::HandshakeMessage::Ignore) => {
                     *hidden = true;
@@ -511,7 +526,8 @@ async fn add_parts(
                 }
                 Err(err) => {
                     *hidden = true;
-                    error!(context, "Error in Secure-Join watching: {}", err);
+                    warn!(context, "Error in Secure-Join watching: {}", err);
+                    return Ok(());
                 }
             }
         }
@@ -719,7 +735,7 @@ async fn add_parts(
     );
 
     // check event to send
-    if chat_id.is_trash() {
+    if chat_id.is_trash() || *hidden {
         *create_event_to_send = None;
     } else if incoming && state == MessageState::InFresh {
         if from_id_blocked {
@@ -854,7 +870,6 @@ async fn create_or_lookup_group(
     let mut chat_id_blocked = Blocked::Not;
     let mut recreate_member_list = false;
     let mut send_EVENT_CHAT_MODIFIED = false;
-    let mut X_MrRemoveFromGrp = None;
     let mut X_MrAddToGrp = None;
     let mut X_MrGrpNameChanged = false;
     let mut better_msg: String = From::from("");
@@ -904,25 +919,27 @@ async fn create_or_lookup_group(
     // but we might not know about this group
 
     let grpname = mime_parser.get(HeaderDef::ChatGroupName).cloned();
+    let mut removed_id = 0;
 
-    if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupMemberRemoved).cloned() {
+    if let Some(removed_addr) = mime_parser.get(HeaderDef::ChatGroupMemberRemoved).cloned() {
-        X_MrRemoveFromGrp = Some(optional_field);
+        removed_id = Contact::lookup_id_by_addr(context, &removed_addr, Origin::Unknown).await;
-        mime_parser.is_system_message = SystemMessage::MemberRemovedFromGroup;
+        if removed_id == 0 {
-        let left_group = Contact::lookup_id_by_addr(context, X_MrRemoveFromGrp.as_ref().unwrap())
+            warn!(context, "removed {:?} has no contact_id", removed_addr);
-            .await
+        } else {
-            == from_id as u32;
+            mime_parser.is_system_message = SystemMessage::MemberRemovedFromGroup;
-        better_msg = context
+            better_msg = context
-            .stock_system_msg(
+                .stock_system_msg(
-                if left_group {
+                    if removed_id == from_id as u32 {
-                    StockMessage::MsgGroupLeft
+                        StockMessage::MsgGroupLeft
-                } else {
+                    } else {
-                    StockMessage::MsgDelMember
+                        StockMessage::MsgDelMember
-                },
+                    },
-                X_MrRemoveFromGrp.as_ref().unwrap(),
+                    &removed_addr,
-                "",
+                    "",
-                from_id as u32,
+                    from_id as u32,
-            )
+                )
-            .await
+                .await;
+        }
     } else {
         let field = mime_parser.get(HeaderDef::ChatGroupMemberAdded).cloned();
         if let Some(optional_field) = field {
@@ -1018,7 +1035,7 @@ async fn create_or_lookup_group(
             && !grpid.is_empty()
             && grpname.is_some()
             // otherwise, a pending "quit" message may pop up
-            && X_MrRemoveFromGrp.is_none()
+            && removed_id == 0
             // re-create explicitly left groups only if ourself is re-added
             && (!group_explicitly_left
                 || X_MrAddToGrp.is_some() && addr_cmp(&self_addr, X_MrAddToGrp.as_ref().unwrap()))
@@ -1156,12 +1173,8 @@ async fn create_or_lookup_group(
             }
         }
         send_EVENT_CHAT_MODIFIED = true;
-    } else if let Some(removed_addr) = X_MrRemoveFromGrp {
+    } else if removed_id > 0 {
-        let contact_id = Contact::lookup_id_by_addr(context, removed_addr).await;
+        chat::remove_from_chat_contacts_table(context, chat_id, removed_id).await;
-        if contact_id != 0 {
-            info!(context, "remove {:?} from chat id={}", contact_id, chat_id);
-            chat::remove_from_chat_contacts_table(context, chat_id, contact_id).await;
-        }
         send_EVENT_CHAT_MODIFIED = true;
     }
 
@@ -1270,10 +1283,9 @@ async fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
     // use subject as initial chat name
-    let default_name = context
+    let grpname = mime_parser
-        .stock_string_repl_int(StockMessage::Member, member_ids.len() as i32)
+        .get_subject()
-        .await;
+        .unwrap_or_else(|| "Unnamed group".to_string());
-    let grpname = mime_parser.get_subject().unwrap_or_else(|| default_name);
 
     // create group record
     let new_chat_id: ChatId = create_group_record(
@@ -1593,7 +1605,7 @@ async fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool
         return false;
     }
 
-    if let Ok(ids) = mailparse::msgidparse(mid_list) {
+    if let Ok(ids) = parse_message_ids(mid_list) {
         for id in ids.iter() {
             if is_known_rfc724_mid(context, id).await {
                 return true;
@@ -1643,7 +1655,7 @@ async fn is_reply_to_messenger_message(context: &Context, mime_parser: &MimeMess
 }
 
 pub(crate) async fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
-    if let Ok(ids) = mailparse::msgidparse(mid_list) {
+    if let Ok(ids) = parse_message_ids(mid_list) {
         for id in ids.iter() {
             if is_msgrmsg_rfc724_mid(context, id).await {
                 return true;
@@ -1669,39 +1681,14 @@ async fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
 
 async fn dc_add_or_lookup_contacts_by_address_list(
     context: &Context,
-    addr_list_raw: &str,
+    address_list: &[SingleInfo],
     origin: Origin,
 ) -> Result<ContactIds> {
-    let addrs = match mailparse::addrparse(addr_list_raw) {
-        Ok(addrs) => addrs,
-        Err(err) => {
-            bail!("could not parse {:?}: {:?}", addr_list_raw, err);
-        }
-    };
-
     let mut contact_ids = ContactIds::new();
-    for addr in addrs.iter() {
+    for info in address_list.iter() {
-        match addr {
+        contact_ids.insert(
-            mailparse::MailAddr::Single(info) => {
+            add_or_lookup_contact_by_addr(context, &info.display_name, &info.addr, origin).await?,
-                contact_ids.insert(
+        );
-                    add_or_lookup_contact_by_addr(context, &info.display_name, &info.addr, origin)
-                        .await?,
-                );
-            }
-            mailparse::MailAddr::Group(infos) => {
-                for info in &infos.addrs {
-                    contact_ids.insert(
-                        add_or_lookup_contact_by_addr(
-                            context,
-                            &info.display_name,
-                            &info.addr,
-                            origin,
-                        )
-                        .await?,
-                    );
-                }
-            }
-        }
     }
 
     Ok(contact_ids)
@@ -1748,6 +1735,7 @@ fn dc_create_incoming_rfc724_mid(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chat::ChatVisibility;
     use crate::chatlist::Chatlist;
     use crate::message::Message;
     use crate::test_utils::{dummy_context, TestContext};
@@ -2020,4 +2008,308 @@ mod tests {
         assert_eq!(chat.name, "group with Alice, Bob and Claire");
         assert_eq!(chat::get_chat_contacts(&t.ctx, chat_id).await.len(), 3);
     }
+
+    #[async_std::test]
+    async fn test_read_receipt_and_unarchive() {
+        // create alice's account
+        let t = configured_offline_context().await;
+
+        // create one-to-one with bob, archive one-to-one
+        let bob_id = Contact::create(&t.ctx, "bob", "bob@exampel.org")
+            .await
+            .unwrap();
+        let one2one_id = chat::create_by_contact_id(&t.ctx, bob_id).await.unwrap();
+        one2one_id
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .await
+            .unwrap();
+        let one2one = Chat::load_from_db(&t.ctx, one2one_id).await.unwrap();
+        assert!(one2one.get_visibility() == ChatVisibility::Archived);
+
+        // create a group with bob, archive group
+        let group_id = chat::create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
+            .await
+            .unwrap();
+        chat::add_contact_to_chat(&t.ctx, group_id, bob_id).await;
+        assert_eq!(
+            chat::get_chat_msgs(&t.ctx, group_id, 0, None).await.len(),
+            0
+        );
+        group_id
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .await
+            .unwrap();
+        let group = Chat::load_from_db(&t.ctx, group_id).await.unwrap();
+        assert!(group.get_visibility() == ChatVisibility::Archived);
+
+        // everything archived, chatlist should be empty
+        assert_eq!(
+            Chatlist::try_load(&t.ctx, DC_GCL_NO_SPECIALS, None, None)
+                .await
+                .unwrap()
+                .len(),
+            0
+        );
+
+        // send a message to group with bob
+        dc_receive_imf(
+            &t.ctx,
+            format!(
+                "From: alice@example.org\n\
+                 To: bob@example.org\n\
+                 Subject: foo\n\
+                 Message-ID: <Gr.{}.12345678901@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: {}\n\
+                 Chat-Group-Name: foo\n\
+                 Chat-Disposition-Notification-To: alice@example.org\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+                group.grpid, group.grpid
+            )
+            .as_bytes(),
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+        let msgs = chat::get_chat_msgs(&t.ctx, group_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+        let msg_id = msgs.first().unwrap();
+        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+            .await
+            .unwrap();
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert_eq!(msg.text.unwrap(), "hello");
+        assert_eq!(msg.state, MessageState::OutDelivered);
+        let group = Chat::load_from_db(&t.ctx, group_id).await.unwrap();
+        assert!(group.get_visibility() == ChatVisibility::Normal);
+
+        // bob sends a read receipt to the group
+        dc_receive_imf(
+            &t.ctx,
+            format!(
+                "From: bob@example.org\n\
+                 To: alice@example.org\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: <Mr.12345678902@example.org>\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Reporting-UA: Delta Chat 1.28.0\n\
+                 Original-Recipient: rfc822;bob@example.org\n\
+                 Final-Recipient: rfc822;bob@example.org\n\
+                 Original-Message-ID: <Gr.{}.12345678901@example.org>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n\
+                 \n\
+                 --SNIPP--",
+                group.grpid
+            )
+            .as_bytes(),
+            "INBOX",
+            1,
+            false,
+        )
+        .await.unwrap();
+        assert_eq!(
+            chat::get_chat_msgs(&t.ctx, group_id, 0, None).await.len(),
+            1
+        );
+        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+            .await
+            .unwrap();
+        assert_eq!(msg.state, MessageState::OutMdnRcvd);
+
+        // check, the read-receipt has not unarchived the one2one
+        assert_eq!(
+            Chatlist::try_load(&t.ctx, DC_GCL_NO_SPECIALS, None, None)
+                .await
+                .unwrap()
+                .len(),
+            1
+        );
+        let one2one = Chat::load_from_db(&t.ctx, one2one_id).await.unwrap();
+        assert!(one2one.get_visibility() == ChatVisibility::Archived);
+    }
+
+    #[async_std::test]
+    async fn test_no_from() {
+        // if there is no from given, from_id stays 0 which is just fine. These messages
+        // are very rare, however, we have to add them to the database (they go to the
+        // "deaddrop" chat) to avoid a re-download from the server. See also [**]
+
+        let t = configured_offline_context().await;
+        let context = &t.ctx;
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert!(chats.get_msg_id(0).is_err());
+
+        dc_receive_imf(
+            context,
+            b"To: bob@example.org\n\
+                 Subject: foo\n\
+                 Message-ID: <3924@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        // Check that the message was added to the database:
+        assert!(chats.get_msg_id(0).is_ok());
+    }
+
+    #[async_std::test]
+    async fn test_escaped_from() {
+        let t = configured_offline_context().await;
+        let contact_id = Contact::create(&t.ctx, "foobar", "foobar@example.com")
+            .await
+            .unwrap();
+        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
+            .await
+            .unwrap();
+        dc_receive_imf(
+            &t.ctx,
+            b"From: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
+                 To: alice@example.org\n\
+                 Subject: foo\n\
+                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Disposition-Notification-To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        ).await.unwrap();
+        assert_eq!(
+            Contact::load_from_db(&t.ctx, contact_id)
+                .await
+                .unwrap()
+                .get_authname(),
+            "Фамилия Имя", // The name was "Имя, Фамилия" and ("lastname, firstname") and should be swapped to "firstname, lastname"
+        );
+        let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+        let msg_id = msgs.first().unwrap();
+        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+            .await
+            .unwrap();
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert_eq!(msg.text.unwrap(), "hello");
+        assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1);
+    }
+
+    #[async_std::test]
+    async fn test_escaped_recipients() {
+        let t = configured_offline_context().await;
+        Contact::create(&t.ctx, "foobar", "foobar@example.com")
+            .await
+            .unwrap();
+
+        let carl_contact_id =
+            Contact::add_or_lookup(&t.ctx, "Carl", "carl@host.tld", Origin::IncomingUnknownFrom)
+                .await
+                .unwrap()
+                .0;
+
+        dc_receive_imf(
+            &t.ctx,
+            b"From: Foobar <foobar@example.com>\n\
+                 To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= alice@example.org\n\
+                 Cc: =?utf-8?q?=3Ch2=3E?= <carl@host.tld>\n\
+                 Subject: foo\n\
+                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Disposition-Notification-To: <foobar@example.com>\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+        assert_eq!(
+            Contact::load_from_db(&t.ctx, carl_contact_id)
+                .await
+                .unwrap()
+                .get_name(),
+            "h2"
+        );
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg = Message::load_from_db(&t.ctx, chats.get_msg_id(0).unwrap())
+            .await
+            .unwrap();
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert_eq!(msg.text.unwrap(), "hello");
+        assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1);
+    }
+
+    #[async_std::test]
+    async fn test_cc_to_contact() {
+        let t = configured_offline_context().await;
+        Contact::create(&t.ctx, "foobar", "foobar@example.com")
+            .await
+            .unwrap();
+
+        let carl_contact_id = Contact::add_or_lookup(
+            &t.ctx,
+            "garabage",
+            "carl@host.tld",
+            Origin::IncomingUnknownFrom,
+        )
+        .await
+        .unwrap()
+        .0;
+
+        dc_receive_imf(
+            &t.ctx,
+            b"From: Foobar <foobar@example.com>\n\
+                 To: alice@example.org\n\
+                 Cc: Carl <carl@host.tld>\n\
+                 Subject: foo\n\
+                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Disposition-Notification-To: <foobar@example.com>\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+        assert_eq!(
+            Contact::load_from_db(&t.ctx, carl_contact_id)
+                .await
+                .unwrap()
+                .get_name(),
+            "Carl"
+        );
+    }
 }

src/dc_tools.rs

@@ -5,7 +5,7 @@ use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
 use std::str::FromStr;
-use std::time::SystemTime;
+use std::time::{Duration, SystemTime};
 
 use async_std::path::{Path, PathBuf};
 use async_std::{fs, io};
@@ -13,7 +13,7 @@ use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::context::Context;
-use crate::error::Error;
+use crate::error::{bail, Error};
 use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
@@ -76,6 +76,14 @@ pub fn dc_timestamp_to_str(wanted: i64) -> String {
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
+pub fn duration_to_str(duration: Duration) -> String {
+    let secs = duration.as_secs();
+    let h = secs / 3600;
+    let m = (secs % 3600) / 60;
+    let s = (secs % 3600) % 60;
+    format!("{}h {}m {}s", h, m, s)
+}
+
 pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
@@ -466,6 +474,23 @@ pub(crate) fn time() -> i64 {
         .as_secs() as i64
 }
 
+/// An invalid email address was encountered
+#[derive(Debug, thiserror::Error)]
+#[error("Invalid email address: {message} ({addr})")]
+pub struct InvalidEmailError {
+    message: String,
+    addr: String,
+}
+
+impl InvalidEmailError {
+    fn new(msg: impl Into<String>, addr: impl Into<String>) -> InvalidEmailError {
+        InvalidEmailError {
+            message: msg.into(),
+            addr: addr.into(),
+        }
+    }
+}
+
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -489,7 +514,7 @@ pub struct EmailAddress {
 }
 
 impl EmailAddress {
-    pub fn new(input: &str) -> Result<Self, Error> {
+    pub fn new(input: &str) -> Result<Self, InvalidEmailError> {
         input.parse::<EmailAddress>()
     }
 }
@@ -501,31 +526,55 @@ impl fmt::Display for EmailAddress {
 }
 
 impl FromStr for EmailAddress {
-    type Err = Error;
+    type Err = InvalidEmailError;
 
     /// Performs a dead-simple parse of an email address.
-    fn from_str(input: &str) -> Result<EmailAddress, Error> {
+    fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
-        ensure!(!input.is_empty(), "empty string is not valid");
+        if input.is_empty() {
+            return Err(InvalidEmailError::new("empty string is not valid", input));
+        }
         let parts: Vec<&str> = input.rsplitn(2, '@').collect();
 
-        ensure!(parts.len() > 1, "missing '@' character");
+        let err = |msg: &str| {
-        let local = parts[1];
+            Err(InvalidEmailError {
-        let domain = parts[0];
+                message: msg.to_string(),
+                addr: input.to_string(),
+            })
+        };
+        match &parts[..] {
+            [domain, local] => {
+                if local.is_empty() {
+                    return err("empty string is not valid for local part");
+                }
+                if domain.len() <= 3 {
+                    return err("domain is too short");
+                }
+                let dot = domain.find('.');
+                match dot {
+                    None => {
+                        return err("invalid domain");
+                    }
+                    Some(dot_idx) => {
+                        if dot_idx >= domain.len() - 2 {
+                            return err("invalid domain");
+                        }
+                    }
+                }
+                Ok(EmailAddress {
+                    local: (*local).to_string(),
+                    domain: (*domain).to_string(),
+                })
+            }
+            _ => err("missing '@' character"),
+        }
+    }
+}
 
-        ensure!(
+impl rusqlite::types::ToSql for EmailAddress {
-            !local.is_empty(),
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-            "empty string is not valid for local part"
+        let val = rusqlite::types::Value::Text(self.to_string());
-        );
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        ensure!(domain.len() > 3, "domain is too short");
+        Ok(out)
-
-        let dot = domain.find('.');
-        ensure!(dot.is_some(), "invalid domain");
-        ensure!(dot.unwrap() < domain.len() - 2, "invalid domain");
-
-        Ok(EmailAddress {
-            local: local.to_string(),
-            domain: domain.to_string(),
-        })
     }
 }
 
@@ -833,4 +882,37 @@ mod tests {
         let next = dc_smeared_time(&t.ctx).await;
         assert!((start + count - 1) < next);
     }
+
+    #[test]
+    fn test_duration_to_str() {
+        assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");
+        assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s");
+        assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s");
+        assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s");
+        assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s");
+        assert_eq!(
+            duration_to_str(Duration::from_secs(59 * 60 + 59)),
+            "0h 59m 59s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(59 * 60 + 60)),
+            "1h 0m 0s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)),
+            "2h 59m 59s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)),
+            "3h 0m 0s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)),
+            "3h 0m 59s"
+        );
+        assert_eq!(
+            duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)),
+            "3h 1m 0s"
+        );
+    }
 }

src/dehtml.rs

@@ -3,7 +3,6 @@
 //! A module to remove HTML tags from the email text
 
 use lazy_static::lazy_static;
-use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 lazy_static! {
@@ -35,6 +34,7 @@ pub fn dehtml(buf: &str) -> String {
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
+    reader.check_end_names(false);
 
     let mut buf = Vec::new();
 
@@ -225,4 +225,23 @@ mod tests {
             "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
         );
     }
+
+    #[test]
+    fn test_unclosed_tags() {
+        let input = r##"
+        <!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'
+        'http://www.w3.org/TR/html4/loose.dtd'>
+        <html>
+        <head>
+        <title>Hi</title>
+        <meta http-equiv='Content-Type' content='text/html; charset=iso-8859-1'>						
+        </head>
+        <body>
+        lots of text
+        </body>
+        </html>
+        "##;
+        let txt = dehtml(input);
+        assert_eq!(txt.trim(), "lots of text");
+    }
 }

src/e2ee.rs

@@ -1,19 +1,17 @@
 //! End-to-end encryption support.
 
 use std::collections::HashSet;
-use std::convert::TryFrom;
 
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::config::Config;
-use crate::constants::KeyGenType;
 use crate::context::Context;
-use crate::dc_tools::EmailAddress;
 use crate::error::*;
-use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::headerdef::HeaderDef;
-use crate::key::{self, Key, KeyPairUse, SignedPublicKey};
+use crate::headerdef::HeaderDefMap;
+use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -38,7 +36,7 @@ impl EncryptHelper {
             Some(addr) => addr,
         };
 
-        let public_key = load_or_generate_self_public_key(context, &addr).await?;
+        let public_key = SignedPublicKey::load_self(context).await?;
 
         Ok(EncryptHelper {
             prefer_encrypt,
@@ -108,9 +106,7 @@ impl EncryptHelper {
         }
         let public_key = Key::from(self.public_key.clone());
         keyring.add_ref(&public_key);
-        let sign_key = Key::from_self_private(context, self.addr.clone(), &context.sql)
+        let sign_key = Key::from(SignedSecretKey::load_self(context).await?);
-            .await
-            .ok_or_else(|| format_err!("missing own private key"))?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
@@ -127,8 +123,8 @@ pub async fn try_decrypt(
 ) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
     let from = mail
         .headers
-        .get_header_value(HeaderDef::From_)
+        .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse(&from_addr).ok())
+        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -193,43 +189,6 @@ pub async fn try_decrypt(
     Ok((out_mail, signatures))
 }
 
-/// Load public key from database or generate a new one.
-///
-/// This will load a public key from the database, generating and
-/// storing a new one when one doesn't exist yet.  Care is taken to
-/// only generate one key per context even when multiple threads call
-/// this function concurrently.
-async fn load_or_generate_self_public_key(
-    context: &Context,
-    self_addr: impl AsRef<str>,
-) -> Result<SignedPublicKey> {
-    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql).await {
-        return SignedPublicKey::try_from(key)
-            .map_err(|_| Error::Message("Not a public key".into()));
-    }
-    let _guard = context.generating_key_mutex.lock().await;
-
-    // Check again in case the key was generated while we were waiting for the lock.
-    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql).await {
-        return SignedPublicKey::try_from(key)
-            .map_err(|_| Error::Message("Not a public key".into()));
-    }
-
-    let start = std::time::Instant::now();
-
-    let keygen_type =
-        KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await).unwrap_or_default();
-    info!(context, "Generating keypair with type {}", keygen_type);
-    let keypair = pgp::create_keypair(EmailAddress::new(self_addr.as_ref())?, keygen_type)?;
-    key::store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
-    info!(
-        context,
-        "Keypair generated in {:.3}s.",
-        start.elapsed().as_secs()
-    );
-    Ok(keypair.public)
-}
-
 /// Returns a reference to the encrypted payload and validates the autocrypt structure.
 fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
     ensure!(
@@ -351,6 +310,7 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 ///
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
+// TODO, remove this once deltachat::key::Key no longer exists.
 pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
     let self_addr = context
         .get_config(Config::ConfiguredAddr)
@@ -361,8 +321,7 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
                 "cannot ensure secret key if not configured."
             ))
         })?;
-    load_or_generate_self_public_key(context, &self_addr).await?;
+    SignedPublicKey::load_self(context).await?;
-
     Ok(self_addr)
 }
 
@@ -413,49 +372,6 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         );
     }
 
-    mod load_or_generate_self_public_key {
-        use super::*;
-
-        #[async_std::test]
-        async fn test_existing() {
-            let t = dummy_context().await;
-            let addr = configure_alice_keypair(&t.ctx).await;
-            let key = load_or_generate_self_public_key(&t.ctx, addr).await;
-            assert!(key.is_ok());
-        }
-
-        #[async_std::test]
-        async fn test_generate() {
-            let t = dummy_context().await;
-            let addr = "alice@example.org";
-            let key0 = load_or_generate_self_public_key(&t.ctx, addr).await;
-            assert!(key0.is_ok());
-            let key1 = load_or_generate_self_public_key(&t.ctx, addr).await;
-            assert!(key1.is_ok());
-            assert_eq!(key0.unwrap(), key1.unwrap());
-        }
-
-        #[async_std::test]
-        async fn test_generate_concurrent() {
-            use std::sync::Arc;
-
-            let t = dummy_context().await;
-            let ctx = Arc::new(t.ctx);
-            let ctx0 = Arc::clone(&ctx);
-            let thr0 = async_std::task::spawn(async move {
-                load_or_generate_self_public_key(&ctx0, "alice@example.org").await
-            });
-            let ctx1 = Arc::clone(&ctx);
-            let thr1 = async_std::task::spawn(async move {
-                load_or_generate_self_public_key(&ctx1, "alice@example.org").await
-            });
-
-            let res0 = thr0.await;
-            let res1 = thr1.await;
-            assert_eq!(res0.unwrap(), res1.unwrap());
-        }
-    }
-
     #[test]
     fn test_has_decrypted_pgp_armor() {
         let data = b" -----BEGIN PGP MESSAGE-----";

src/error.rs

@@ -1,198 +1,6 @@
 //! # Error handling
 
-use lettre_email::mime;
+pub use anyhow::{bail, ensure, format_err, Error, Result};
-
-#[derive(Debug, Fail)]
-pub enum Error {
-    #[fail(display = "{:?}", _0)]
-    Failure(failure::Error),
-
-    #[fail(display = "SQL error: {:?}", _0)]
-    SqlError(#[cause] crate::sql::Error),
-
-    #[fail(display = "{:?}", _0)]
-    Io(std::io::Error),
-
-    #[fail(display = "{:?}", _0)]
-    Message(String),
-
-    #[fail(display = "{:?}", _0)]
-    MessageWithCause(String, #[cause] failure::Error, failure::Backtrace),
-
-    #[fail(display = "{:?}", _0)]
-    Image(image_meta::ImageError),
-
-    #[fail(display = "{:?}", _0)]
-    Utf8(std::str::Utf8Error),
-
-    #[fail(display = "PGP: {:?}", _0)]
-    Pgp(pgp::errors::Error),
-
-    #[fail(display = "Base64Decode: {:?}", _0)]
-    Base64Decode(base64::DecodeError),
-
-    #[fail(display = "{:?}", _0)]
-    FromUtf8(std::string::FromUtf8Error),
-
-    #[fail(display = "{}", _0)]
-    BlobError(#[cause] crate::blob::BlobError),
-
-    #[fail(display = "Invalid Message ID.")]
-    InvalidMsgId,
-
-    #[fail(display = "Watch folder not found {:?}", _0)]
-    WatchFolderNotFound(String),
-
-    #[fail(display = "Invalid Email: {:?}", _0)]
-    MailParseError(#[cause] mailparse::MailParseError),
-
-    #[fail(display = "Building invalid Email: {:?}", _0)]
-    LettreError(#[cause] lettre_email::error::Error),
-
-    #[fail(display = "SMTP error: {:?}", _0)]
-    SmtpError(#[cause] async_smtp::error::Error),
-
-    #[fail(display = "FromStr error: {:?}", _0)]
-    FromStr(#[cause] mime::FromStrError),
-
-    #[fail(display = "Not Configured")]
-    NotConfigured,
-
-    #[fail(display = "No event available")]
-    PopError(crossbeam_queue::PopError),
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl From<crate::sql::Error> for Error {
-    fn from(err: crate::sql::Error) -> Error {
-        Error::SqlError(err)
-    }
-}
-
-impl From<crossbeam_queue::PopError> for Error {
-    fn from(err: crossbeam_queue::PopError) -> Error {
-        Error::PopError(err)
-    }
-}
-
-impl From<base64::DecodeError> for Error {
-    fn from(err: base64::DecodeError) -> Error {
-        Error::Base64Decode(err)
-    }
-}
-
-impl From<failure::Error> for Error {
-    fn from(err: failure::Error) -> Error {
-        Error::Failure(err)
-    }
-}
-
-impl From<std::io::Error> for Error {
-    fn from(err: std::io::Error) -> Error {
-        Error::Io(err)
-    }
-}
-
-impl From<std::str::Utf8Error> for Error {
-    fn from(err: std::str::Utf8Error) -> Error {
-        Error::Utf8(err)
-    }
-}
-
-impl From<image_meta::ImageError> for Error {
-    fn from(err: image_meta::ImageError) -> Error {
-        Error::Image(err)
-    }
-}
-
-impl From<pgp::errors::Error> for Error {
-    fn from(err: pgp::errors::Error) -> Error {
-        Error::Pgp(err)
-    }
-}
-
-impl From<std::string::FromUtf8Error> for Error {
-    fn from(err: std::string::FromUtf8Error) -> Error {
-        Error::FromUtf8(err)
-    }
-}
-
-impl From<crate::blob::BlobError> for Error {
-    fn from(err: crate::blob::BlobError) -> Error {
-        Error::BlobError(err)
-    }
-}
-
-impl From<crate::message::InvalidMsgId> for Error {
-    fn from(_err: crate::message::InvalidMsgId) -> Error {
-        Error::InvalidMsgId
-    }
-}
-
-impl From<crate::key::SaveKeyError> for Error {
-    fn from(err: crate::key::SaveKeyError) -> Error {
-        Error::MessageWithCause(format!("{}", err), err.into(), failure::Backtrace::new())
-    }
-}
-
-impl From<crate::pgp::PgpKeygenError> for Error {
-    fn from(err: crate::pgp::PgpKeygenError) -> Error {
-        Error::MessageWithCause(format!("{}", err), err.into(), failure::Backtrace::new())
-    }
-}
-
-impl From<mailparse::MailParseError> for Error {
-    fn from(err: mailparse::MailParseError) -> Error {
-        Error::MailParseError(err)
-    }
-}
-
-impl From<lettre_email::error::Error> for Error {
-    fn from(err: lettre_email::error::Error) -> Error {
-        Error::LettreError(err)
-    }
-}
-
-impl From<mime::FromStrError> for Error {
-    fn from(err: mime::FromStrError) -> Error {
-        Error::FromStr(err)
-    }
-}
-
-#[macro_export]
-macro_rules! bail {
-    ($e:expr) => {
-        return Err($crate::error::Error::Message($e.to_string()));
-    };
-    ($fmt:expr, $($arg:tt)+) => {
-        return Err($crate::error::Error::Message(format!($fmt, $($arg)+)));
-    };
-}
-
-#[macro_export]
-macro_rules! format_err {
-    ($e:expr) => {
-        $crate::error::Error::Message($e.to_string());
-    };
-    ($fmt:expr, $($arg:tt)+) => {
-        $crate::error::Error::Message(format!($fmt, $($arg)+));
-    };
-}
-
-#[macro_export(local_inner_macros)]
-macro_rules! ensure {
-    ($cond:expr, $e:expr) => {
-        if !($cond) {
-            bail!($e);
-        }
-    };
-    ($cond:expr, $fmt:expr, $($arg:tt)+) => {
-        if !($cond) {
-            bail!($fmt, $($arg)+);
-        }
-    };
-}
 
 #[macro_export]
 macro_rules! ensure_eq {

src/events.rs

@@ -201,10 +201,4 @@ pub enum Event {
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
-
-    /// This event is sent out to the inviter when a joiner successfully joined a group.
-    /// @param data1 (int) chat_id
-    /// @param data2 (int) contact_id
-    #[strum(props(id = "2062"))]
-    SecurejoinMemberAdded { chat_id: ChatId, contact_id: u32 },
 }

src/headerdef.rs

@@ -53,12 +53,16 @@ impl HeaderDef {
 
 pub trait HeaderDefMap {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String>;
+    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader>;
 }
 
 impl HeaderDefMap for [MailHeader<'_>] {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String> {
         self.get_first_value(headerdef.get_headername())
     }
+    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader> {
+        self.get_first_header(headerdef.get_headername())
+    }
 }
 
 #[cfg(test)]

src/imap/idle.rs

@@ -11,28 +11,22 @@ use super::session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "IMAP IDLE protocol failed to init/complete")]
+    #[error("IMAP IDLE protocol failed to init/complete")]
-    IdleProtocolFailed(#[cause] async_imap::error::Error),
+    IdleProtocolFailed(#[from] async_imap::error::Error),
 
-    #[fail(display = "IMAP IDLE protocol timed out")]
+    #[error("IMAP IDLE protocol timed out")]
-    IdleTimeout(#[cause] async_std::future::TimeoutError),
+    IdleTimeout(#[from] async_std::future::TimeoutError),
 
-    #[fail(display = "IMAP server does not have IDLE capability")]
+    #[error("IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
-    #[fail(display = "IMAP select folder error")]
+    #[error("IMAP select folder error")]
-    SelectFolderError(#[cause] select_folder::Error),
+    SelectFolderError(#[from] select_folder::Error),
 
-    #[fail(display = "Setup handle error")]
+    #[error("Setup handle error")]
-    SetupHandleError(#[cause] super::Error),
+    SetupHandleError(#[from] super::Error),
-}
-
-impl From<select_folder::Error> for Error {
-    fn from(err: select_folder::Error) -> Error {
-        Error::SelectFolderError(err)
-    }
 }
 
 impl Imap {
@@ -46,10 +40,7 @@ impl Imap {
         if !self.can_idle() {
             return Err(Error::IdleAbilityMissing);
         }
-
+        self.setup_handle_if_needed(context).await?;
-        self.setup_handle_if_needed(context)
-            .await
-            .map_err(Error::SetupHandleError)?;
 
         self.select_folder(context, watch_folder.clone()).await?;
 

src/imap/mod.rs

@@ -23,6 +23,7 @@ use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{self, Action};
 use crate::login_param::{CertificateChecks, LoginParam};
 use crate::message::{self, update_server_uid};
+use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::stock::StockMessage;
@@ -37,78 +38,48 @@ use session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "IMAP Connect without configured params")]
+    #[error("IMAP Connect without configured params")]
     ConnectWithoutConfigure,
 
-    #[fail(display = "IMAP Connection Failed params: {}", _0)]
+    #[error("IMAP Connection Failed params: {0}")]
     ConnectionFailed(String),
 
-    #[fail(display = "IMAP No Connection established")]
+    #[error("IMAP No Connection established")]
     NoConnection,
 
-    #[fail(display = "IMAP Could not get OAUTH token")]
+    #[error("IMAP Could not get OAUTH token")]
     OauthError,
 
-    #[fail(display = "IMAP Could not login as {}", _0)]
+    #[error("IMAP Could not login as {0}")]
     LoginFailed(String),
 
-    #[fail(display = "IMAP Could not fetch")]
+    #[error("IMAP Could not fetch")]
-    FetchFailed(#[cause] async_imap::error::Error),
+    FetchFailed(#[from] async_imap::error::Error),
 
-    #[fail(display = "IMAP operation attempted while it is torn down")]
+    #[error("IMAP operation attempted while it is torn down")]
     InTeardown,
 
-    #[fail(display = "IMAP operation attempted while it is torn down")]
+    #[error("IMAP operation attempted while it is torn down")]
-    SqlError(#[cause] crate::sql::Error),
+    SqlError(#[from] crate::sql::Error),
 
-    #[fail(display = "IMAP got error from elsewhere")]
+    #[error("IMAP got error from elsewhere")]
-    WrappedError(#[cause] crate::error::Error),
+    WrappedError(#[from] crate::error::Error),
 
-    #[fail(display = "IMAP select folder error")]
+    #[error("IMAP select folder error")]
-    SelectFolderError(#[cause] select_folder::Error),
+    SelectFolderError(#[from] select_folder::Error),
 
-    #[fail(display = "Mail parse error")]
+    #[error("Mail parse error")]
-    MailParseError(#[cause] mailparse::MailParseError),
+    MailParseError(#[from] mailparse::MailParseError),
 
-    #[fail(display = "No mailbox selected, folder: {:?}", _0)]
+    #[error("No mailbox selected, folder: {0}")]
     NoMailbox(String),
 
-    #[fail(display = "IMAP other error: {:?}", _0)]
+    #[error("IMAP other error: {0}")]
     Other(String),
 }
 
-impl From<crate::sql::Error> for Error {
-    fn from(err: crate::sql::Error) -> Error {
-        Error::SqlError(err)
-    }
-}
-
-impl From<crate::error::Error> for Error {
-    fn from(err: crate::error::Error) -> Error {
-        Error::WrappedError(err)
-    }
-}
-
-impl From<Error> for crate::error::Error {
-    fn from(err: Error) -> crate::error::Error {
-        crate::error::Error::Message(err.to_string())
-    }
-}
-
-impl From<select_folder::Error> for Error {
-    fn from(err: select_folder::Error) -> Error {
-        Error::SelectFolderError(err)
-    }
-}
-
-impl From<mailparse::MailParseError> for Error {
-    fn from(err: mailparse::MailParseError) -> Error {
-        Error::MailParseError(err)
-    }
-}
-
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
 pub enum ImapActionResult {
     Failed,
@@ -308,7 +279,7 @@ impl Imap {
                     context
                         .stock_string_repl_str2(
                             StockMessage::ServerResponse,
-                            format!("{}:{}", imap_server, imap_port),
+                            format!("IMAP {}:{}", imap_server, imap_port),
                             err.to_string(),
                         )
                         .await
@@ -655,8 +626,13 @@ impl Imap {
             read_cnt += 1;
             let headers = get_fetch_headers(&fetch)?;
             let message_id = prefetch_get_message_id(&headers).unwrap_or_default();
-
+            if let Ok(true) = precheck_imf(context, &message_id, folder.as_ref(), cur_uid)
-            if precheck_imf(context, &message_id, folder.as_ref(), cur_uid).await {
+                .await
+                .map_err(|err| {
+                    warn!(context, "precheck_imf error: {}", err);
+                    err
+                })
+            {
                 // we know the message-id already or don't want the message otherwise.
                 info!(
                     context,
@@ -665,6 +641,9 @@ impl Imap {
                     folder.as_ref(),
                 );
             } else {
+                // we do not know the message-id
+                // or the message-id is missing (in this case, we create one in the further process)
+                // or some other error happened
                 let show = prefetch_should_download(context, &headers, show_emails)
                     .await
                     .map_err(|err| {
@@ -792,13 +771,7 @@ impl Imap {
                 if let Err(err) =
                     dc_receive_imf(context, &body, folder.as_ref(), server_uid, is_seen).await
                 {
-                    warn!(
+                    return Err(Error::Other(format!("dc_receive_imf error: {}", err)));
-                        context,
-                        "dc_receive_imf failed for imap-message {}/{}: {:?}",
-                        folder.as_ref(),
-                        server_uid,
-                        err
-                    );
                 }
             }
         } else {
@@ -1324,9 +1297,9 @@ async fn precheck_imf(
     rfc724_mid: &str,
     server_folder: &str,
     server_uid: u32,
-) -> bool {
+) -> Result<bool> {
-    if let Ok((old_server_folder, old_server_uid, msg_id)) =
+    if let Some((old_server_folder, old_server_uid, msg_id)) =
-        message::rfc724_mid_exists(context, &rfc724_mid).await
+        message::rfc724_mid_exists(context, &rfc724_mid).await?
     {
         if old_server_folder.is_empty() && old_server_uid == 0 {
             info!(
@@ -1383,9 +1356,9 @@ async fn precheck_imf(
         if old_server_folder != server_folder || old_server_uid != server_uid {
             update_server_uid(context, &rfc724_mid, server_folder, server_uid).await;
         }
-        true
+        Ok(true)
     } else {
-        false
+        Ok(false)
     }
 }
 
@@ -1438,12 +1411,8 @@ async fn prefetch_should_download(
         .get_header_value(HeaderDef::AutocryptSetupMessage)
         .is_some();
 
-    let from_field = headers
-        .get_header_value(HeaderDef::From_)
-        .unwrap_or_default();
-
     let (_contact_id, blocked_contact, origin) =
-        from_field_to_contact_id(context, &from_field).await?;
+        from_field_to_contact_id(context, &mimeparser::get_from(headers)).await?;
     let accepted_contact = origin.is_known();
 
     let show = is_autocrypt_setup_message

src/imap/select_folder.rs

@@ -4,25 +4,52 @@ use crate::context::Context;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "IMAP Could not obtain imap-session object.")]
+    #[error("IMAP Could not obtain imap-session object.")]
     NoSession,
 
-    #[fail(display = "IMAP Connection Lost or no connection established")]
+    #[error("IMAP Connection Lost or no connection established")]
     ConnectionLost,
 
-    #[fail(display = "IMAP Folder name invalid: {:?}", _0)]
+    #[error("IMAP Folder name invalid: {0}")]
     BadFolderName(String),
 
-    #[fail(display = "IMAP close/expunge failed: {}", _0)]
+    #[error("IMAP close/expunge failed")]
-    CloseExpungeFailed(#[cause] async_imap::error::Error),
+    CloseExpungeFailed(#[from] async_imap::error::Error),
 
-    #[fail(display = "IMAP other error: {:?}", _0)]
+    #[error("IMAP other error: {0}")]
     Other(String),
 }
 
 impl Imap {
+    /// Issues a CLOSE command to expunge selected folder.
+    ///
+    /// CLOSE is considerably faster than an EXPUNGE, see
+    /// https://tools.ietf.org/html/rfc3501#section-6.4.2
+    async fn close_folder(&mut self, context: &Context) -> Result<()> {
+        if let Some(ref folder) = self.config.selected_folder {
+            info!(context, "Expunge messages in \"{}\".", folder);
+
+            if let Some(ref mut session) = self.session {
+                match session.close().await {
+                    Ok(_) => {
+                        info!(context, "close/expunge succeeded");
+                    }
+                    Err(err) => {
+                        self.trigger_reconnect();
+                        return Err(Error::CloseExpungeFailed(err));
+                    }
+                }
+            } else {
+                return Err(Error::NoSession);
+            }
+        }
+        self.config.selected_folder = None;
+        self.config.selected_folder_needs_expunge = false;
+        Ok(())
+    }
+
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(

src/imex.rs

@@ -16,7 +16,7 @@ use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
-use crate::key::{self, Key};
+use crate::key::{self, DcKey, Key, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
@@ -181,10 +181,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
         passphrase.len() >= 2,
         "Passphrase must be at least 2 chars long."
     );
-    let self_addr = e2ee::ensure_secret_key_exists(context).await?;
+    let private_key = Key::from(SignedSecretKey::load_self(context).await?);
-    let private_key = Key::from_self_private(context, self_addr, &context.sql)
-        .await
-        .ok_or_else(|| format_err!("Failed to get private key."))?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),

src/job.rs

@@ -10,6 +10,10 @@ use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
 
+use async_smtp::smtp::response::Category;
+use async_smtp::smtp::response::Code;
+use async_smtp::smtp::response::Detail;
+
 use crate::blob::BlobObject;
 use crate::chat::{self, ChatId};
 use crate::config::Config;
@@ -17,7 +21,7 @@ use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::{Error, Result};
+use crate::error::{bail, ensure, format_err, Error, Result};
 use crate::events::Event;
 use crate::imap::*;
 use crate::location;
@@ -70,7 +74,19 @@ impl Default for Thread {
     }
 }
 
-#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug,
+    Display,
+    Copy,
+    Clone,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    FromPrimitive,
+    ToPrimitive,
+    FromSql,
+    ToSql,
+)]
 #[repr(i32)]
 pub enum Action {
     Unknown = 0,
@@ -140,32 +156,68 @@ impl fmt::Display for Job {
 }
 
 impl Job {
-    /// Deletes the job from the database.
+    fn new(action: Action, foreign_id: u32, param: Params, delay_seconds: i64) -> Self {
-    async fn delete(&self, context: &Context) -> bool {
+        let timestamp = time();
-        context
+
-            .sql
+        Self {
-            .execute("DELETE FROM jobs WHERE id=?;", paramsv![self.job_id as i32])
+            job_id: 0,
-            .await
+            action,
-            .is_ok()
+            foreign_id,
+            desired_timestamp: timestamp + delay_seconds,
+            added_timestamp: timestamp,
+            tries: 0,
+            param,
+            pending_error: None,
+        }
     }
 
-    /// Updates the job already stored in the database.
+    /// Deletes the job from the database.
+    async fn delete(&self, context: &Context) -> bool {
+        if self.job_id != 0 {
+            context
+                .sql
+                .execute("DELETE FROM jobs WHERE id=?;", paramsv![self.job_id as i32])
+                .await
+                .is_ok()
+        } else {
+            // Already deleted.
+            true
+        }
+    }
+
+    /// Saves the job to the database, creating a new entry if necessary.
     ///
-    /// To add a new job, use [job_add].
+    /// The Job is consumed by this method.
-    async fn update(&self, context: &Context) -> bool {
+    async fn save(self, context: &Context) -> bool {
-        context
+        let thread: Thread = self.action.into();
-            .sql
+
-            .execute(
+        if self.job_id != 0 {
-                "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
+            context
+                .sql
+                .execute(
+                    "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
+                    paramsv![
+                        self.desired_timestamp,
+                        self.tries as i64,
+                        self.param.to_string(),
+                        self.job_id as i32,
+                    ],
+                )
+                .await
+                .is_ok()
+        } else {
+            context.sql.execute(
+                "INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);",
                 paramsv![
-                    self.desired_timestamp,
+                    self.added_timestamp,
-                    self.tries as i64,
+                    thread,
+                    self.action,
+                    self.foreign_id,
                     self.param.to_string(),
-                    self.job_id as i32,
+                    self.desired_timestamp
-                ],
+                ]
-            )
+            ).await.is_ok()
-            .await
+        }
-            .is_ok()
     }
 
     async fn smtp_send<F, Fut>(
@@ -196,8 +248,42 @@ impl Job {
                 self.pending_error = Some(err.to_string());
 
                 let res = match err {
-                    async_smtp::smtp::error::Error::Permanent(_) => {
+                    async_smtp::smtp::error::Error::Permanent(ref response) => {
-                        Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                        match response.code {
+                            // Sometimes servers send a permanent error when actually it is a temporary error
+                            // For documentation see https://tools.ietf.org/html/rfc3463
+
+                            // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
+                            Code {
+                                category: Category::MailSystem,
+                                detail: Detail::Zero,
+                                ..
+                            } => Status::RetryLater,
+
+                            _ => {
+                                // If we do not retry, add an info message to the chat
+                                // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM.
+                                match Message::load_from_db(context, MsgId::new(self.foreign_id))
+                                    .await
+                                {
+                                    Ok(message) => {
+                                        chat::add_info_msg(
+                                            context,
+                                            message.chat_id,
+                                            err.to_string(),
+                                        )
+                                        .await
+                                    }
+                                    Err(e) => warn!(
+                                        context,
+                                        "couldn't load chat_id to inform user about SMTP error: {}",
+                                        e
+                                    ),
+                                };
+
+                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            }
+                        }
                     }
                     async_smtp::smtp::error::Error::Transient(_) => {
                         // We got a transient 4xx response from SMTP server.
@@ -223,7 +309,7 @@ impl Job {
                 // Local error, job is invalid, do not retry.
                 smtp.disconnect().await;
                 warn!(context, "SMTP job is invalid: {}", err);
-                Status::Finished(Err(Error::SmtpError(err)))
+                Status::Finished(Err(err.into()))
             }
             Err(crate::smtp::send::Error::NoTransport) => {
                 // Should never happen.
@@ -756,13 +842,46 @@ async fn add_imap_deletion_jobs(context: &Context) -> sql::Result<()> {
                 Params::new(),
                 0,
             )
-            .await
+            .await;
         }
     }
 
     Ok(())
 }
 
+async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+    if let Some(delete_server_after) = context.get_config_delete_server_after().await {
+        let threshold_timestamp = time() - delete_server_after;
+
+        context
+            .sql
+            .query_row_optional(
+                "SELECT id FROM msgs \
+             WHERE timestamp < ? \
+             AND server_uid != 0",
+                paramsv![threshold_timestamp],
+                |row| row.get::<_, MsgId>(0),
+            )
+            .await
+    } else {
+        Ok(None)
+    }
+}
+
+async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
+    let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
+        Some(Job::new(
+            Action::DeleteMsgOnImap,
+            msg_id.to_u32(),
+            Params::new(),
+            0,
+        ))
+    } else {
+        None
+    };
+    Ok(res)
+}
+
 impl<'a> fmt::Display for Connection<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
@@ -808,7 +927,6 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
                 job.tries = tries;
                 let time_offset = get_backoff_time_offset(tries);
                 job.desired_timestamp = time() + time_offset;
-                job.update(context).await;
                 info!(
                     context,
                     "{}-job #{} not succeeded on try #{}, retry in {} seconds.",
@@ -817,6 +935,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
                     tries,
                     time_offset
                 );
+                job.save(context).await;
             } else {
                 info!(
                     context,
@@ -938,6 +1057,9 @@ pub async fn add(
         return;
     }
 
+    let param_str = param.to_string();
+    let job = Job::new(action, foreign_id as u32, param, delay_seconds);
+    job.save(context).await;
     let timestamp = time();
     let thread: Thread = action.into();
 
@@ -948,26 +1070,28 @@ pub async fn add(
             thread,
             action,
             foreign_id,
-            param.to_string(),
+            param_str,
             (timestamp + delay_seconds as i64)
         ]
     ).await.ok();
 
-    match action {
+    if delay_seconds == 0 {
-        Action::Unknown => unreachable!(),
+        match action {
-        Action::Housekeeping
+            Action::Unknown => unreachable!(),
-        | Action::EmptyServer
+            Action::Housekeeping
-        | Action::OldDeleteMsgOnImap
+            | Action::EmptyServer
-        | Action::DeleteMsgOnImap
+            | Action::OldDeleteMsgOnImap
-        | Action::MarkseenMsgOnImap
+            | Action::DeleteMsgOnImap
-        | Action::MoveMsg => {
+            | Action::MarkseenMsgOnImap
-            context.interrupt_inbox().await;
+            | Action::MoveMsg => {
-        }
+                context.interrupt_inbox().await;
-        Action::MaybeSendLocations
+            }
-        | Action::MaybeSendLocationsEnded
+            Action::MaybeSendLocations
-        | Action::SendMdn
+            | Action::MaybeSendLocationsEnded
-        | Action::SendMsgToSmtp => {
+            | Action::SendMdn
-            context.interrupt_smtp().await;
+            | Action::SendMsgToSmtp => {
+                context.interrupt_smtp().await;
+            }
         }
     }
 }
@@ -1006,7 +1130,7 @@ pub(crate) async fn load_next(
         params_probe
     };
 
-    context
+    let job = context
         .sql
         .query_map(
             query,
@@ -1036,7 +1160,24 @@ pub(crate) async fn load_next(
             },
         )
         .await
-        .unwrap_or_default()
+        .unwrap_or_default();
+
+    if thread == Thread::Imap {
+        if let Some(job) = job {
+            if job.action < Action::DeleteMsgOnImap {
+                load_imap_deletion_job(context)
+                    .await
+                    .unwrap_or_default()
+                    .or(Some(job))
+            } else {
+                Some(job)
+            }
+        } else {
+            load_imap_deletion_job(context).await.unwrap_or_default()
+        }
+    } else {
+        job
+    }
 }
 
 #[cfg(test)]

src/key.rs

@@ -4,38 +4,40 @@ use std::collections::BTreeMap;
 use std::io::Cursor;
 
 use async_std::path::Path;
+use async_trait::async_trait;
+use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
 
+use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_write_file, time, EmailAddress, InvalidEmailError};
-use crate::sql::Sql;
+use crate::sql;
 
 // Re-export key types
 pub use crate::pgp::KeyPair;
 pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
 
 /// Error type for deltachat key handling.
-#[derive(Fail, Debug)]
+#[derive(Debug, thiserror::Error)]
+#[non_exhaustive]
 pub enum Error {
-    #[fail(display = "Could not decode base64")]
+    #[error("Could not decode base64")]
-    Base64Decode(#[cause] base64::DecodeError, failure::Backtrace),
+    Base64Decode(#[from] base64::DecodeError),
-    #[fail(display = "rPGP error: {}", _0)]
+    #[error("rPGP error: {}", _0)]
-    PgpError(#[cause] pgp::errors::Error, failure::Backtrace),
+    Pgp(#[from] pgp::errors::Error),
-}
+    #[error("Failed to generate PGP key: {}", _0)]
-
+    Keygen(#[from] crate::pgp::PgpKeygenError),
-impl From<base64::DecodeError> for Error {
+    #[error("Failed to load key: {}", _0)]
-    fn from(err: base64::DecodeError) -> Error {
+    LoadKey(#[from] sql::Error),
-        Error::Base64Decode(err, failure::Backtrace::new())
+    #[error("Failed to save generated key: {}", _0)]
-    }
+    StoreKey(#[from] SaveKeyError),
-}
+    #[error("No address configured")]
-
+    NoConfiguredAddr,
-impl From<pgp::errors::Error> for Error {
+    #[error("Configured address is invalid: {}", _0)]
-    fn from(err: pgp::errors::Error) -> Error {
+    InvalidConfiguredAddr(#[from] InvalidEmailError),
-        Error::PgpError(err, failure::Backtrace::new())
-    }
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -45,6 +47,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 /// This trait is implemented for rPGP's [SignedPublicKey] and
 /// [SignedSecretKey] types and makes working with them a little
 /// easier in the deltachat world.
+#[async_trait]
 pub trait DcKey: Serialize + Deserializable {
     type KeyType: Serialize + Deserializable;
 
@@ -63,6 +66,9 @@ pub trait DcKey: Serialize + Deserializable {
         Self::from_slice(&bytes)
     }
 
+    /// Load the users' default key from the database.
+    async fn load_self(context: &Context) -> Result<Self::KeyType>;
+
     /// Serialise the key to a base64 string.
     fn to_base64(&self) -> String {
         // Not using Serialize::to_bytes() to make clear *why* it is
@@ -75,12 +81,108 @@ pub trait DcKey: Serialize + Deserializable {
     }
 }
 
+#[async_trait]
 impl DcKey for SignedPublicKey {
     type KeyType = SignedPublicKey;
+
+    async fn load_self(context: &Context) -> Result<Self::KeyType> {
+        match context
+            .sql
+            .query_row(
+                r#"
+            SELECT public_key
+              FROM keypairs
+             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
+               AND is_default=1;
+            "#,
+                paramsv![],
+                |row| row.get::<_, Vec<u8>>(0),
+            )
+            .await
+        {
+            Ok(bytes) => Self::from_slice(&bytes),
+            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
+                let keypair = generate_keypair(context).await?;
+                Ok(keypair.public)
+            }
+            Err(err) => Err(err.into()),
+        }
+    }
 }
 
+#[async_trait]
 impl DcKey for SignedSecretKey {
     type KeyType = SignedSecretKey;
+
+    async fn load_self(context: &Context) -> Result<Self::KeyType> {
+        match context
+            .sql
+            .query_row(
+                r#"
+            SELECT private_key
+              FROM keypairs
+             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
+               AND is_default=1;
+            "#,
+                paramsv![],
+                |row| row.get::<_, Vec<u8>>(0),
+            )
+            .await
+        {
+            Ok(bytes) => Self::from_slice(&bytes),
+            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
+                let keypair = generate_keypair(context).await?;
+                Ok(keypair.secret)
+            }
+            Err(err) => Err(err.into()),
+        }
+    }
+}
+
+async fn generate_keypair(context: &Context) -> Result<KeyPair> {
+    let addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await
+        .ok_or_else(|| Error::NoConfiguredAddr)?;
+    let addr = EmailAddress::new(&addr)?;
+    let _guard = context.generating_key_mutex.lock().await;
+
+    // Check if the key appeared while we were waiting on the lock.
+    match context
+        .sql
+        .query_row(
+            r#"
+        SELECT public_key, private_key
+          FROM keypairs
+         WHERE addr=?1
+           AND is_default=1;
+        "#,
+            paramsv![addr],
+            |row| Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, Vec<u8>>(1)?)),
+        )
+        .await
+    {
+        Ok((pub_bytes, sec_bytes)) => Ok(KeyPair {
+            addr,
+            public: SignedPublicKey::from_slice(&pub_bytes)?,
+            secret: SignedSecretKey::from_slice(&sec_bytes)?,
+        }),
+        Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
+            let start = std::time::Instant::now();
+            let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
+                .unwrap_or_default();
+            info!(context, "Generating keypair with type {}", keytype);
+            let keypair = crate::pgp::create_keypair(addr, keytype)?;
+            store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
+            info!(
+                context,
+                "Keypair generated in {:.3}s.",
+                start.elapsed().as_secs()
+            );
+            Ok(keypair)
+        }
+        Err(err) => Err(err.into()),
+    }
 }
 
 /// Cryptographic key
@@ -197,36 +299,6 @@ impl Key {
         }
     }
 
-    pub async fn from_self_public(
-        context: &Context,
-        self_addr: impl AsRef<str>,
-        sql: &Sql,
-    ) -> Option<Self> {
-        let addr = self_addr.as_ref();
-
-        sql.query_get_value(
-            context,
-            "SELECT public_key FROM keypairs WHERE addr=? AND is_default=1;",
-            paramsv![addr],
-        )
-        .await
-        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Public))
-    }
-
-    pub async fn from_self_private(
-        context: &Context,
-        self_addr: impl AsRef<str>,
-        sql: &Sql,
-    ) -> Option<Self> {
-        sql.query_get_value(
-            context,
-            "SELECT private_key FROM keypairs WHERE addr=? AND is_default=1;",
-            paramsv![self_addr.as_ref()],
-        )
-        .await
-        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Private))
-    }
-
     pub fn to_bytes(&self) -> Vec<u8> {
         match self {
             Key::Public(k) => k.to_bytes().unwrap_or_default(),
@@ -318,21 +390,19 @@ pub enum KeyPairUse {
 }
 
 /// Error saving a keypair to the database.
-#[derive(Fail, Debug)]
+#[derive(Debug, thiserror::Error)]
-#[fail(display = "SaveKeyError: {}", message)]
+#[error("SaveKeyError: {message}")]
 pub struct SaveKeyError {
     message: String,
-    #[cause]
+    #[source]
-    cause: failure::Error,
+    cause: anyhow::Error,
-    backtrace: failure::Backtrace,
 }
 
 impl SaveKeyError {
-    fn new(message: impl Into<String>, cause: impl Into<failure::Error>) -> Self {
+    fn new(message: impl Into<String>, cause: impl Into<anyhow::Error>) -> Self {
         Self {
             message: message.into(),
             cause: cause.into(),
-            backtrace: failure::Backtrace::new(),
         }
     }
 }
@@ -559,6 +629,63 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         }
     }
 
+    #[async_std::test]
+    async fn test_load_self_existing() {
+        let alice = alice_keypair();
+        let t = dummy_context().await;
+        configure_alice_keypair(&t.ctx).await;
+        let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
+        assert_eq!(alice.public, pubkey);
+        let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
+        assert_eq!(alice.secret, seckey);
+    }
+
+    #[async_std::test]
+    #[ignore] // generating keys is expensive
+    async fn test_load_self_generate_public() {
+        let t = dummy_context().await;
+        t.ctx
+            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+            .await
+            .unwrap();
+        let key = SignedPublicKey::load_self(&t.ctx).await;
+        assert!(key.is_ok());
+    }
+
+    #[async_std::test]
+    #[ignore] // generating keys is expensive
+    async fn test_load_self_generate_secret() {
+        let t = dummy_context().await;
+        t.ctx
+            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+            .await
+            .unwrap();
+        let key = SignedSecretKey::load_self(&t.ctx).await;
+        assert!(key.is_ok());
+    }
+
+    #[async_std::test]
+    #[ignore] // generating keys is expensive
+    async fn test_load_self_generate_concurrent() {
+        use std::thread;
+
+        let t = dummy_context().await;
+        t.ctx
+            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+            .await
+            .unwrap();
+        let ctx = t.ctx.clone();
+        let ctx0 = ctx.clone();
+        let thr0 =
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
+        let ctx1 = ctx.clone();
+        let thr1 =
+            thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
+        let res0 = thr0.join().unwrap();
+        let res1 = thr1.join().unwrap();
+        assert_eq!(res0.unwrap(), res1.unwrap());
+    }
+
     #[test]
     fn test_ascii_roundtrip() {
         let public_key = Key::from(KEYPAIR.public.clone());

src/lib.rs

@@ -2,8 +2,6 @@
 #![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
 #![allow(clippy::match_bool)]
 
-#[macro_use]
-extern crate failure_derive;
 #[macro_use]
 extern crate num_derive;
 #[macro_use]

src/location.rs

@@ -1,7 +1,6 @@
 //! Location handling
 
 use bitflags::bitflags;
-use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::chat::{self, ChatId};
@@ -9,7 +8,7 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::*;
 use crate::dc_tools::*;
-use crate::error::Error;
+use crate::error::{ensure, Error};
 use crate::events::Event;
 use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
@@ -122,9 +121,9 @@ impl Kml {
                 }
             } else if self.tag.contains(KmlTag::COORDINATES) {
                 let parts = val.splitn(2, ',').collect::<Vec<_>>();
-                if parts.len() == 2 {
+                if let [longitude, latitude] = &parts[..] {
-                    self.curr.longitude = parts[0].parse().unwrap_or_default();
+                    self.curr.longitude = longitude.parse().unwrap_or_default();
-                    self.curr.latitude = parts[1].parse().unwrap_or_default();
+                    self.curr.latitude = latitude.parse().unwrap_or_default();
                 }
             }
         }

src/lot.rs

@@ -40,11 +40,11 @@ impl Lot {
     }
 
     pub fn get_text1(&self) -> Option<&str> {
-        self.text1.as_ref().map(|s| s.as_str())
+        self.text1.as_deref()
     }
 
     pub fn get_text2(&self) -> Option<&str> {
-        self.text2.as_ref().map(|s| s.as_str())
+        self.text2.as_deref()
     }
 
     pub fn get_text1_meaning(&self) -> Meaning {

src/message.rs

@@ -2,7 +2,6 @@
 
 use async_std::path::{Path, PathBuf};
 use deltachat_derive::{FromSql, ToSql};
-use failure::Fail;
 use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 
@@ -11,7 +10,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
 use crate::dc_tools::*;
-use crate::error::Error;
+use crate::error::{ensure, Error};
 use crate::events::Event;
 use crate::job::{self, Action};
 use crate::lot::{Lot, LotState, Meaning};
@@ -169,7 +168,7 @@ impl rusqlite::types::ToSql for MsgId {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
         if self.0 <= DC_MSG_ID_LAST_SPECIAL {
             return Err(rusqlite::Error::ToSqlConversionFailure(Box::new(
-                InvalidMsgId.compat(),
+                InvalidMsgId,
             )));
         }
         let val = rusqlite::types::Value::Integer(self.0 as i64);
@@ -197,8 +196,8 @@ impl rusqlite::types::FromSql for MsgId {
 /// This usually occurs when trying to use a message ID of
 /// [DC_MSG_ID_LAST_SPECIAL] or below in a situation where this is not
 /// possible.
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
-#[fail(display = "Invalid Message ID.")]
+#[error("Invalid Message ID.")]
 pub struct InvalidMsgId;
 
 #[derive(
@@ -1079,9 +1078,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> bool {
                 if let Err(rusqlite::Error::QueryReturnedNoRows) = query_res {
                     continue;
                 }
-                let (state, blocked) = query_res
+                let (state, blocked) = query_res.map_err(Into::<anyhow::Error>::into)?;
-                    .map_err(|err| Error::SqlError(err.into()))
-                    .expect("query fail");
                 msgs.push((id, state, blocked));
             }
 
@@ -1261,7 +1258,7 @@ pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl
         }
         if let Some(error) = error {
             msg.param.set(Param::Error, error.as_ref());
-            error!(context, "{}", error.as_ref());
+            warn!(context, "Message failed: {}", error.as_ref());
         }
 
         if context
@@ -1504,12 +1501,15 @@ pub async fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> i32 {
 pub(crate) async fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
-) -> Result<(String, u32, MsgId), Error> {
+) -> Result<Option<(String, u32, MsgId)>, Error> {
-    ensure!(!rfc724_mid.is_empty(), "empty rfc724_mid");
+    if rfc724_mid.is_empty() {
+        warn!(context, "Empty rfc724_mid passed to rfc724_mid_exists");
+        return Ok(None);
+    }
 
     let res = context
         .sql
-        .query_row(
+        .query_row_optional(
             "SELECT server_folder, server_uid, id FROM msgs WHERE rfc724_mid=?",
             paramsv![rfc724_mid],
             |row| {

src/mimefactory.rs

@@ -9,12 +9,13 @@ use crate::contact::*;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::*;
 use crate::e2ee::*;
-use crate::error::Error;
+use crate::error::{bail, ensure, format_err, Error};
 use crate::location;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
+use crate::simplify::escape_message_footer_marks;
 use crate::stock::StockMessage;
 
 // attachments of 25 mb brutto should work on the majority of providers
@@ -113,22 +114,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
             let command = msg.param.get_cmd();
 
-            /* for added members, the list is just fine */
-            if command == SystemMessage::MemberRemovedFromGroup {
-                let email_to_remove = msg.param.get(Param::Arg).unwrap_or_default();
-
-                let self_addr = context
-                    .get_config(Config::ConfiguredAddr)
-                    .await
-                    .unwrap_or_default();
-
-                if !email_to_remove.is_empty()
-                    && !addr_cmp(email_to_remove, self_addr)
-                    && !recipients_contain_addr(&recipients, &email_to_remove)
-                {
-                    recipients.push(("".to_string(), email_to_remove.to_string()));
-                }
-            }
             if command != SystemMessage::AutocryptSetupMessage
                 && command != SystemMessage::SecurejoinMessage
                 && context.get_config_bool(Config::MdnsEnabled).await
@@ -865,7 +850,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let message_text = format!(
             "{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
-            &final_text,
+            escape_message_footer_marks(final_text),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -1156,7 +1141,7 @@ async fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
 fn render_rfc724_mid(rfc724_mid: &str) -> String {
     let rfc724_mid = rfc724_mid.trim().to_string();
 
-    if rfc724_mid.chars().nth(0).unwrap_or_default() == '<' {
+    if rfc724_mid.chars().next().unwrap_or_default() == '<' {
         rfc724_mid
     } else {
         format!("<{}>", rfc724_mid)

src/mimeparser.rs

@@ -4,10 +4,9 @@ use std::pin::Pin;
 
 use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
-use mailparse::{DispositionType, MailAddr, MailHeaderMap};
+use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 
 use crate::aheader::Aheader;
-use crate::bail;
 use crate::blob::BlobObject;
 use crate::constants::Viewtype;
 use crate::contact::*;
@@ -15,7 +14,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::dehtml::dehtml;
 use crate::e2ee;
-use crate::error::Result;
+use crate::error::{bail, Result};
 use crate::events::Event;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::location;
@@ -39,6 +38,11 @@ use crate::stock::StockMessage;
 pub struct MimeMessage {
     pub parts: Vec<Part>,
     header: HashMap<String, String>,
+
+    /// Addresses are normalized and lowercased:
+    pub recipients: Vec<SingleInfo>,
+    pub from: Vec<SingleInfo>,
+    pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
     pub signatures: HashSet<String>,
     pub gossipped_addr: HashSet<String>,
@@ -90,9 +94,22 @@ impl MimeMessage {
             .unwrap_or_default();
 
         let mut headers = Default::default();
+        let mut recipients = Default::default();
+        let mut from = Default::default();
+        let mut chat_disposition_notification_to = None;
 
         // init known headers with what mailparse provided us
-        MimeMessage::merge_headers(&mut headers, &mail.headers);
+        MimeMessage::merge_headers(
+            context,
+            &mut headers,
+            &mut recipients,
+            &mut from,
+            &mut chat_disposition_notification_to,
+            &mail.headers,
+        );
+
+        // remove headers that are allowed _only_ in the encrypted part
+        headers.remove("secure-join-fingerprint");
 
         // Memory location for a possible decrypted message.
         let mail_raw;
@@ -118,7 +135,14 @@ impl MimeMessage {
 
                     // let known protected headers from the decrypted
                     // part override the unencrypted top-level
-                    MimeMessage::merge_headers(&mut headers, &decrypted_mail.headers);
+                    MimeMessage::merge_headers(
+                        context,
+                        &mut headers,
+                        &mut recipients,
+                        &mut from,
+                        &mut chat_disposition_notification_to,
+                        &decrypted_mail.headers,
+                    );
 
                     (decrypted_mail, signatures)
                 } else {
@@ -142,6 +166,9 @@ impl MimeMessage {
         let mut parser = MimeMessage {
             parts: Vec::new(),
             header: headers,
+            recipients,
+            from,
+            chat_disposition_notification_to,
             decrypting_failed: false,
 
             // only non-empty if it was a valid autocrypt message
@@ -203,10 +230,8 @@ impl MimeMessage {
     /// Delta Chat sends attachments, such as images, in two-part messages, with the first message
     /// containing an explanation. If such a message is detected, first part can be safely dropped.
     fn squash_attachment_parts(&mut self) {
-        if self.has_chat_version() && self.parts.len() == 2 {
+        if let [textpart, filepart] = &self.parts[..] {
             let need_drop = {
-                let textpart = &self.parts[0];
-                let filepart = &self.parts[1];
                 textpart.typ == Viewtype::Text
                     && (filepart.typ == Viewtype::Image
                         || filepart.typ == Viewtype::Gif
@@ -312,11 +337,9 @@ impl MimeMessage {
 
         // See if an MDN is requested from the other side
         if !self.decrypting_failed && !self.parts.is_empty() {
-            if let Some(ref dn_to_addr) =
+            if let Some(ref dn_to) = self.chat_disposition_notification_to {
-                self.parse_first_addr(context, HeaderDef::ChatDispositionNotificationTo)
+                if let Some(ref from) = self.from.get(0) {
-            {
+                    if from.addr == dn_to.addr {
-                if let Some(ref from_addr) = self.parse_first_addr(context, HeaderDef::From_) {
-                    if compare_addrs(from_addr, dn_to_addr) {
                         if let Some(part) = self.parts.last_mut() {
                             part.param.set_int(Param::WantsMdn, 1);
                         }
@@ -390,20 +413,6 @@ impl MimeMessage {
         self.header.get(headerdef.get_headername())
     }
 
-    fn parse_first_addr(&self, context: &Context, headerdef: HeaderDef) -> Option<MailAddr> {
-        if let Some(value) = self.get(headerdef.clone()) {
-            match mailparse::addrparse(&value) {
-                Ok(ref addrs) => {
-                    return addrs.first().cloned();
-                }
-                Err(err) => {
-                    warn!(context, "header {} parse error: {:?}", headerdef, err);
-                }
-            }
-        }
-        None
-    }
-
     fn parse_mime_recursive<'a>(
         &'a mut self,
         context: &'a Context,
@@ -416,9 +425,9 @@ impl MimeMessage {
             if mail.ctype.params.get("protected-headers").is_some() {
                 if mail.ctype.mimetype == "text/rfc822-headers" {
                     warn!(
-                    context,
+                        context,
-                    "Protected headers found in text/rfc822-headers attachment: Will be ignored.",
+                        "Protected headers found in text/rfc822-headers attachment: Will be ignored.",
-                );
+                    );
                     return Ok(false);
                 }
 
@@ -480,7 +489,9 @@ impl MimeMessage {
             apple mail: "plaintext" as an alternative to "html+PDF attachment") */
             (mime::MULTIPART, "alternative") => {
                 for cur_data in &mail.subparts {
-                    if get_mime_type(cur_data)?.0 == "multipart/mixed" {
+                    if get_mime_type(cur_data)?.0 == "multipart/mixed"
+                        || get_mime_type(cur_data)?.0 == "multipart/related"
+                    {
                         any_part_added = self.parse_mime_recursive(context, cur_data).await?;
                         break;
                     }
@@ -504,15 +515,6 @@ impl MimeMessage {
                     }
                 }
             }
-            (mime::MULTIPART, "related") => {
-                /* add the "root part" - the other parts may be referenced which is
-                not interesting for us (eg. embedded images) we assume he "root part"
-                being the first one, which may not be always true ...
-                however, most times it seems okay. */
-                if let Some(first) = mail.subparts.iter().next() {
-                    any_part_added = self.parse_mime_recursive(context, first).await?;
-                }
-            }
             (mime::MULTIPART, "encrypted") => {
                 // we currently do not try to decrypt non-autocrypt messages
                 // at all. If we see an encrypted part, we set
@@ -761,17 +763,41 @@ impl MimeMessage {
             .and_then(|msgid| parse_message_id(msgid).ok())
     }
 
-    fn merge_headers(headers: &mut HashMap<String, String>, fields: &[mailparse::MailHeader<'_>]) {
+    fn merge_headers(
+        context: &Context,
+        headers: &mut HashMap<String, String>,
+        recipients: &mut Vec<SingleInfo>,
+        from: &mut Vec<SingleInfo>,
+        chat_disposition_notification_to: &mut Option<SingleInfo>,
+        fields: &[mailparse::MailHeader<'_>],
+    ) {
         for field in fields {
             // lowercasing all headers is technically not correct, but makes things work better
             let key = field.get_key().to_lowercase();
             if !headers.contains_key(&key) || // key already exists, only overwrite known types (protected headers)
                     is_known(&key) || key.starts_with("chat-")
             {
-                let value = field.get_value();
+                if key == HeaderDef::ChatDispositionNotificationTo.get_headername() {
-                headers.insert(key.to_string(), value);
+                    match addrparse_header(field) {
+                        Ok(addrlist) => {
+                            *chat_disposition_notification_to = addrlist.extract_single_info();
+                        }
+                        Err(e) => warn!(context, "Could not read {} address: {}", key, e),
+                    }
+                } else {
+                    let value = field.get_value();
+                    headers.insert(key.to_string(), value);
+                }
             }
         }
+        let recipients_new = get_recipients(fields);
+        if !recipients_new.is_empty() {
+            *recipients = recipients_new;
+        }
+        let from_new = get_from(fields);
+        if !from_new.is_empty() {
+            *from = from_new;
+        }
     }
 
     fn process_report(
@@ -840,23 +866,15 @@ async fn update_gossip_peerstates(
     gossip_headers: Vec<String>,
 ) -> Result<HashSet<String>> {
     // XXX split the parsing from the modification part
-    let mut recipients: Option<HashSet<String>> = None;
     let mut gossipped_addr: HashSet<String> = Default::default();
 
     for value in &gossip_headers {
         let gossip_header = value.parse::<Aheader>();
 
         if let Ok(ref header) = gossip_header {
-            if recipients.is_none() {
+            if get_recipients(&mail.headers)
-                recipients = Some(get_recipients(
+                .iter()
-                    mail.headers.iter().map(|v| (v.get_key(), v.get_value())),
+                .any(|info| info.addr == header.addr.to_lowercase())
-                ));
-            }
-
-            if recipients
-                .as_ref()
-                .unwrap()
-                .contains(&header.addr.to_lowercase())
             {
                 let mut peerstate = Peerstate::from_addr(context, &header.addr).await;
                 if let Some(ref mut peerstate) = peerstate {
@@ -894,14 +912,29 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
-pub(crate) fn parse_message_id(value: &str) -> crate::error::Result<String> {
+pub(crate) fn parse_message_ids(ids: &str) -> Result<Vec<String>> {
-    let ids = mailparse::msgidparse(value)
+    // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>`
-        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
+    let mut msgids = Vec::new();
+    for id in ids.split_whitespace() {
+        let mut id = id.to_string();
+        if id.starts_with('<') {
+            id = id[1..].to_string();
+        }
+        if id.ends_with('>') {
+            id = id[..id.len() - 1].to_string();
+        }
+        if !id.is_empty() {
+            msgids.push(id);
+        }
+    }
+    Ok(msgids)
+}
 
-    if let Some(id) = ids.first() {
+pub(crate) fn parse_message_id(ids: &str) -> Result<String> {
+    if let Some(id) = parse_message_ids(ids)?.first() {
         Ok(id.to_string())
     } else {
-        bail!("could not parse message_id: {}", value);
+        bail!("could not parse message_id: {}", ids);
     }
 }
 
@@ -1002,6 +1035,11 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String
     let desired_filename =
         desired_filename.or_else(|| ct.params.get("name").map(|s| s.to_string()));
 
+    // MS Outlook is known to specify filename in the "name" attribute of
+    // Content-Type and omit Content-Disposition.
+    let desired_filename =
+        desired_filename.or_else(|| mail.ctype.params.get("name").map(|s| s.to_string()));
+
     // If there is no filename, but part is an attachment, guess filename
     if ct.disposition == DispositionType::Attachment && desired_filename.is_none() {
         if let Some(subtype) = mail.ctype.mimetype.split('/').nth(1) {
@@ -1017,50 +1055,50 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String
     }
 }
 
-// returned addresses are normalized and lowercased.
+/// Returned addresses are normalized and lowercased.
-fn get_recipients<S: AsRef<str>, T: Iterator<Item = (S, S)>>(headers: T) -> HashSet<String> {
+fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
-    let mut recipients: HashSet<String> = Default::default();
+    get_all_addresses_from_header(headers, |header_key| {
+        header_key == "to" || header_key == "cc"
+    })
+}
 
-    for (hkey, hvalue) in headers {
+/// Returned addresses are normalized and lowercased.
-        let hkey = hkey.as_ref().to_lowercase();
+pub(crate) fn get_from(headers: &[MailHeader]) -> Vec<SingleInfo> {
-        let hvalue = hvalue.as_ref();
+    get_all_addresses_from_header(headers, |header_key| header_key == "from")
-        if hkey == "to" || hkey == "cc" {
+}
-            if let Ok(addrs) = mailparse::addrparse(hvalue) {
+
-                for addr in addrs.iter() {
+fn get_all_addresses_from_header<F>(headers: &[MailHeader], pred: F) -> Vec<SingleInfo>
-                    match addr {
+where
-                        mailparse::MailAddr::Single(ref info) => {
+    F: Fn(String) -> bool,
-                            recipients.insert(addr_normalize(&info.addr).to_lowercase());
+{
-                        }
+    let mut result: Vec<SingleInfo> = Default::default();
-                        mailparse::MailAddr::Group(ref infos) => {
+
-                            for info in &infos.addrs {
+    headers
-                                recipients.insert(addr_normalize(&info.addr).to_lowercase());
+        .iter()
-                            }
+        .filter(|header| pred(header.get_key().to_lowercase()))
+        .filter_map(|header| mailparse::addrparse_header(header).ok())
+        .for_each(|addrs| {
+            for addr in addrs.iter() {
+                match addr {
+                    mailparse::MailAddr::Single(ref info) => {
+                        result.push(SingleInfo {
+                            addr: addr_normalize(&info.addr).to_lowercase(),
+                            display_name: info.display_name.clone(),
+                        });
+                    }
+                    mailparse::MailAddr::Group(ref infos) => {
+                        for info in &infos.addrs {
+                            result.push(SingleInfo {
+                                addr: addr_normalize(&info.addr).to_lowercase(),
+                                display_name: info.display_name.clone(),
+                            });
                         }
                     }
                 }
             }
-        }
+        });
-    }
 
-    recipients
+    result
-}
-
-/// Check if the only addrs match, ignoring names.
-fn compare_addrs(a: &mailparse::MailAddr, b: &mailparse::MailAddr) -> bool {
-    match a {
-        mailparse::MailAddr::Group(group_a) => match b {
-            mailparse::MailAddr::Group(group_b) => group_a
-                .addrs
-                .iter()
-                .zip(group_b.addrs.iter())
-                .all(|(a, b)| a.addr == b.addr),
-            _ => false,
-        },
-        mailparse::MailAddr::Single(single_a) => match b {
-            mailparse::MailAddr::Single(single_b) => single_a.addr == single_b.addr,
-            _ => false,
-        },
-    }
 }
 
 #[cfg(test)]
@@ -1113,16 +1151,13 @@ mod tests {
         assert_eq!(mimeparser.get_rfc724_mid(), None);
     }
 
-    #[async_std::test]
+    #[test]
-    async fn test_get_recipients() {
+    fn test_get_recipients() {
-        let context = dummy_context().await;
         let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
-        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
+        let mail = mailparse::parse_mail(&raw[..]).unwrap();
-            .await
+        let recipients = get_recipients(&mail.headers);
-            .unwrap();
+        assert!(recipients.iter().any(|info| info.addr == "abc@bcd.com"));
-        let recipients = get_recipients(mimeparser.header.iter());
+        assert!(recipients.iter().any(|info| info.addr == "def@def.de"));
-        assert!(recipients.contains("abc@bcd.com"));
-        assert!(recipients.contains("def@def.de"));
         assert_eq!(recipients.len(), 2);
     }
 
@@ -1179,14 +1214,10 @@ mod tests {
             .await
             .unwrap();
 
-        let of = mimeparser
+        let of = &mimeparser.from[0];
-            .parse_first_addr(&context.ctx, HeaderDef::From_)
+        assert_eq!(of.addr, "hello@one.org");
-            .unwrap();
-        assert_eq!(of, mailparse::addrparse("hello@one.org").unwrap()[0]);
 
-        let of =
+        assert!(mimeparser.chat_disposition_notification_to.is_none());
-            mimeparser.parse_first_addr(&context.ctx, HeaderDef::ChatDispositionNotificationTo);
-        assert!(of.is_none());
     }
 
     #[async_std::test]
@@ -1196,14 +1227,16 @@ mod tests {
                     Content-Type: multipart/mixed; boundary=\"==break==\";\n\
                     Subject: outer-subject\n\
                     Secure-Join-Group: no\n\
-                    Test-Header: Bar\nChat-Version: 0.0\n\
+                    Secure-Join-Fingerprint: 123456\n\
+                    Test-Header: Bar\n\
+                    chat-VERSION: 0.0\n\
                     \n\
                     --==break==\n\
                     Content-Type: text/plain; protected-headers=\"v1\";\n\
                     Subject: inner-subject\n\
                     SecureBar-Join-Group: yes\n\
                     Test-Header: Xy\n\
-                    Chat-Version: 1.0\n\
+                    chat-VERSION: 1.0\n\
                     \n\
                     test1\n\
                     \n\
@@ -1224,12 +1257,17 @@ mod tests {
 
         // the following fields would bubble up
         // if the test would really use encryption for the protected part
-        // however, as this is not the case, the outer things stay valid
+        // however, as this is not the case, the outer things stay valid.
+        // for Chat-Version, also the case-insensivity is tested.
         assert_eq!(mimeparser.get_subject(), Some("outer-subject".into()));
 
         let of = mimeparser.get(HeaderDef::ChatVersion).unwrap();
         assert_eq!(of, "0.0");
         assert_eq!(mimeparser.parts.len(), 1);
+
+        // make sure, headers that are only allowed in the encrypted part
+        // cannot be set from the outer part
+        assert!(mimeparser.get(HeaderDef::SecureJoinFingerprint).is_none());
     }
 
     #[async_std::test]
@@ -1533,6 +1571,266 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
             Some("Mail with inline attachment".to_string())
         );
 
-        assert_eq!(message.parts.len(), 2);
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::File);
+        assert_eq!(message.parts[0].msg, "Hello!");
+    }
+
+    #[async_std::test]
+    async fn parse_inline_image() {
+        let context = dummy_context().await;
+        let raw = br#"Message-ID: <foobar@example.org>
+From: foo <foo@example.org>
+Subject: example
+To: bar@example.org
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="--11019878869865180"
+
+----11019878869865180
+Content-Type: text/plain; charset=utf-8
+
+Test
+
+----11019878869865180
+Content-Type: image/jpeg;
+ name="JPEG_filename.jpg"
+Content-Transfer-Encoding: base64
+Content-Disposition: inline;
+ filename="JPEG_filename.jpg"
+
+ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG
+acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT
+/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal
+n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C
+xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2
+rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU
+8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI
+K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4
+CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
+
+
+----11019878869865180--
+"#;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(message.get_subject(), Some("example".to_string()));
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Image);
+        assert_eq!(message.parts[0].msg, "Test");
+    }
+
+    #[async_std::test]
+    async fn parse_thunderbird_html_embedded_image() {
+        let context = dummy_context().await;
+        let raw = br#"To: Alice <alice@example.org>
+From: Bob <bob@example.org>
+Subject: Test subject
+Message-ID: <foobarbaz@example.org>
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.7.0
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+ boundary="------------779C1631600DF3DB8C02E53A"
+Content-Language: en-US
+
+This is a multi-part message in MIME format.
+--------------779C1631600DF3DB8C02E53A
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+
+Test
+
+
+--------------779C1631600DF3DB8C02E53A
+Content-Type: multipart/related;
+ boundary="------------10CC6C2609EB38DA782C5CA9"
+
+
+--------------10CC6C2609EB38DA782C5CA9
+Content-Type: text/html; charset=utf-8
+Content-Transfer-Encoding: 7bit
+
+<html>
+<head>
+<meta http-equiv="content-type" content="text/html; charset=UTF-8">
+</head>
+<body>
+Test<br>
+<p><img moz-do-not-send="false" src="cid:part1.9DFA679B.52A88D69@example.org" alt=""></p>
+</body>
+</html>
+
+--------------10CC6C2609EB38DA782C5CA9
+Content-Type: image/png;
+ name="1.png"
+Content-Transfer-Encoding: base64
+Content-ID: <part1.9DFA679B.52A88D69@example.org>
+Content-Disposition: inline;
+ filename="1.png"
+
+ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG
+acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT
+/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal
+n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C
+xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2
+rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU
+8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI
+K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4
+CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
+--------------10CC6C2609EB38DA782C5CA9--
+
+--------------779C1631600DF3DB8C02E53A--"#;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(message.get_subject(), Some("Test subject".to_string()));
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Image);
+        assert_eq!(message.parts[0].msg, "Test");
+    }
+
+    // Outlook specifies filename in the "name" attribute of Content-Type
+    #[async_std::test]
+    async fn parse_outlook_html_embedded_image() {
+        let context = dummy_context().await;
+        let raw = br##"From: Anonymous <anonymous@example.org>
+To: Anonymous <anonymous@example.org>
+Subject: Delta Chat is great stuff!
+Date: Tue, 5 May 2020 01:23:45 +0000
+MIME-Version: 1.0
+Content-Type: multipart/related;
+	boundary="----=_NextPart_000_0003_01D622B3.CA753E60"
+X-Mailer: Microsoft Outlook 15.0
+
+This is a multipart message in MIME format.
+
+------=_NextPart_000_0003_01D622B3.CA753E60
+Content-Type: multipart/alternative;
+	boundary="----=_NextPart_001_0004_01D622B3.CA753E60"
+
+
+------=_NextPart_001_0004_01D622B3.CA753E60
+Content-Type: text/plain;
+	charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+
+
+
+------=_NextPart_001_0004_01D622B3.CA753E60
+Content-Type: text/html;
+	charset="us-ascii"
+Content-Transfer-Encoding: quoted-printable
+
+<html>
+<body>
+<p>
+Test<img src="cid:image001.jpg@01D622B3.C9D8D750">
+</p>
+</body>
+</html>
+------=_NextPart_001_0004_01D622B3.CA753E60--
+
+------=_NextPart_000_0003_01D622B3.CA753E60
+Content-Type: image/jpeg;
+	name="image001.jpg"
+Content-Transfer-Encoding: base64
+Content-ID: <image001.jpg@01D622B3.C9D8D750>
+
+ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG
+acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT
+/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal
+n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C
+xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2
+rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU
+8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI
+K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4
+CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
+
+------=_NextPart_000_0003_01D622B3.CA753E60--
+"##;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(
+            message.get_subject(),
+            Some("Delta Chat is great stuff!".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Image);
+        assert_eq!(message.parts[0].msg, "Test");
+    }
+
+    #[test]
+    fn test_parse_message_id() {
+        let test = parse_message_id("<foobar>");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foobar");
+
+        let test = parse_message_id("<foo> <bar>");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("  < foo > <bar>");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("foo");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id(" foo ");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("foo bar");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("  foo  bar ");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("");
+        assert!(test.is_err());
+
+        let test = parse_message_id(" ");
+        assert!(test.is_err());
+
+        let test = parse_message_id("<>");
+        assert!(test.is_err());
+
+        let test = parse_message_id("<> bar");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "bar");
+    }
+
+    #[test]
+    fn test_parse_message_ids() {
+        let test = parse_message_ids("  foo  bar <foobar>").unwrap();
+        assert_eq!(test.len(), 3);
+        assert_eq!(test[0], "foo");
+        assert_eq!(test[1], "bar");
+        assert_eq!(test[2], "foobar");
+
+        let test = parse_message_ids("  < foobar >").unwrap();
+        assert_eq!(test.len(), 1);
+        assert_eq!(test[0], "foobar");
+
+        let test = parse_message_ids("").unwrap();
+        assert!(test.is_empty());
+
+        let test = parse_message_ids(" ").unwrap();
+        assert!(test.is_empty());
+
+        let test = parse_message_ids("  < ").unwrap();
+        assert!(test.is_empty());
     }
 }

src/param.rs

@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
-use crate::error;
+use crate::error::{self, bail, ensure};
 use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 

src/pgp.rs

@@ -18,7 +18,7 @@ use rand::{thread_rng, CryptoRng, Rng};
 
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
-use crate::error::Result;
+use crate::error::{bail, ensure, format_err, Result};
 use crate::key::*;
 use crate::keyring::*;
 
@@ -117,21 +117,19 @@ pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap<String, Str
 ///
 /// Most of these are likely coding errors rather than user errors
 /// since all variability is hardcoded.
-#[derive(Fail, Debug)]
+#[derive(Debug, thiserror::Error)]
-#[fail(display = "PgpKeygenError: {}", message)]
+#[error("PgpKeygenError: {message}")]
-pub(crate) struct PgpKeygenError {
+pub struct PgpKeygenError {
     message: String,
-    #[cause]
+    #[source]
-    cause: failure::Error,
+    cause: anyhow::Error,
-    backtrace: failure::Backtrace,
 }
 
 impl PgpKeygenError {
-    fn new(message: impl Into<String>, cause: impl Into<failure::Error>) -> Self {
+    fn new(message: impl Into<String>, cause: impl Into<anyhow::Error>) -> Self {
         Self {
             message: message.into(),
             cause: cause.into(),
-            backtrace: failure::Backtrace::new(),
         }
     }
 }
@@ -189,7 +187,7 @@ pub(crate) fn create_keypair(
                 .unwrap(),
         )
         .build()
-        .map_err(|err| PgpKeygenError::new("invalid key params", failure::err_msg(err)))?;
+        .map_err(|err| PgpKeygenError::new("invalid key params", format_err!(err)))?;
     let key = key_params
         .generate()
         .map_err(|err| PgpKeygenError::new("invalid params", err))?;

src/provider/data.rs

@@ -268,7 +268,14 @@ lazy_static::lazy_static! {
     };
 
     // yandex.ru.md: yandex.ru, yandex.com
-    // - skipping provider with status OK and no special things to do
+    static ref P_YANDEX_RU: Provider = Provider {
+        status: Status::PREPARATION,
+        before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/yandex-ru",
+        server: vec![
+        ],
+    };
 
     // ziggo.nl.md: ziggo.nl
     static ref P_ZIGGO_NL: Provider = Provider {
@@ -360,6 +367,8 @@ lazy_static::lazy_static! {
         ("ymail.com", &*P_YAHOO),
         ("rocketmail.com", &*P_YAHOO),
         ("yahoodns.net", &*P_YAHOO),
+        ("yandex.ru", &*P_YANDEX_RU),
+        ("yandex.com", &*P_YANDEX_RU),
         ("ziggo.nl", &*P_ZIGGO_NL),
     ].iter().copied().collect();
 }

src/qr.rs

@@ -2,20 +2,20 @@
 
 use lazy_static::lazy_static;
 use percent_encoding::percent_decode_str;
+use reqwest::Url;
+use serde::Deserialize;
 
 use crate::chat;
 use crate::config::*;
 use crate::constants::Blocked;
 use crate::contact::*;
 use crate::context::Context;
-use crate::error::Error;
+use crate::error::{bail, ensure, format_err, Error};
 use crate::key::dc_format_fingerprint;
 use crate::key::dc_normalize_fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::param::*;
 use crate::peerstate::*;
-use reqwest::Url;
-use serde::Deserialize;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
@@ -37,6 +37,10 @@ impl Into<Lot> for Error {
     }
 }
 
+fn starts_with_ignore_case(string: &str, pattern: &str) -> bool {
+    string.to_lowercase().starts_with(&pattern.to_lowercase())
+}
+
 /// Check a scanned QR code.
 /// The function should be called after a QR code is scanned.
 /// The function takes the raw text scanned and checks what can be done with it.
@@ -45,9 +49,9 @@ pub async fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
 
     info!(context, "Scanned QR code: {}", qr);
 
-    if qr.starts_with(OPENPGP4FPR_SCHEME) {
+    if starts_with_ignore_case(qr, OPENPGP4FPR_SCHEME) {
         decode_openpgp(context, qr).await
-    } else if qr.starts_with(DCACCOUNT_SCHEME) {
+    } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
         decode_account(context, qr)
     } else if qr.starts_with(MAILTO_SCHEME) {
         decode_mailto(context, qr).await
@@ -224,27 +228,20 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error
 
     let response = reqwest::blocking::Client::new().post(url_str).send();
     if response.is_err() {
-        return Err(format_err!(
+        bail!("Cannot create account, request to {} failed", url_str);
-            "Cannot create account, request to {} failed",
-            url_str
-        ));
     }
     let response = response.unwrap();
     if !response.status().is_success() {
-        return Err(format_err!(
+        bail!("Request to {} unsuccessful: {:?}", url_str, response);
-            "Request to {} unsuccessful: {:?}",
-            url_str,
-            response
-        ));
     }
 
     let parsed: reqwest::Result<CreateAccountResponse> = response.json();
     if parsed.is_err() {
-        return Err(format_err!(
+        bail!(
             "Failed to parse JSON response from {}: error: {:?}",
             url_str,
             parsed
-        ));
+        );
     }
     println!("response: {:?}", &parsed);
     let parsed = parsed.unwrap();
@@ -296,7 +293,6 @@ async fn decode_smtp(context: &Context, qr: &str) -> Lot {
         Ok(addr) => addr,
         Err(err) => return err.into(),
     };
-
     let name = "".to_string();
     Lot::from_address(context, name, addr).await
 }
@@ -534,6 +530,17 @@ mod tests {
         assert_ne!(res.get_id(), 0);
         assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
+        // Test it again with lowercased "openpgp4fpr:" uri scheme
+        let res = check_qr(
+            &ctx.ctx,
+            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+        ).await;
+
+        println!("{:?}", res);
+        assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
+        assert_ne!(res.get_id(), 0);
+        assert_eq!(res.get_text1().unwrap(), "test ? test !");
+
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
     }
@@ -551,6 +558,16 @@ mod tests {
         assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
         assert_ne!(res.get_id(), 0);
 
+        // Test it again with lowercased "openpgp4fpr:" uri scheme
+        let res = check_qr(
+            &ctx.ctx,
+            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
+        ).await;
+
+        println!("{:?}", res);
+        assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
+        assert_ne!(res.get_id(), 0);
+
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).await.unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
         assert_eq!(contact.get_name(), "Jörn P. P.");
@@ -572,6 +589,20 @@ mod tests {
         );
         assert_eq!(res.get_id(), 0);
 
+        // Test it again with lowercased "openpgp4fpr:" uri scheme
+
+        let res = check_qr(
+            &ctx.ctx,
+            "openpgp4fpr:1234567890123456789012345678901234567890",
+        )
+        .await;
+        assert_eq!(res.get_state(), LotState::QrFprWithoutAddr);
+        assert_eq!(
+            res.get_text1().unwrap(),
+            "1234 5678 9012 3456 7890\n1234 5678 9012 3456 7890"
+        );
+        assert_eq!(res.get_id(), 0);
+
         let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890").await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert_eq!(res.get_id(), 0);
@@ -588,6 +619,15 @@ mod tests {
         .await;
         assert_eq!(res.get_state(), LotState::QrAccount);
         assert_eq!(res.get_text1().unwrap(), "example.org");
+
+        // Test it again with lowercased "dcaccount:" uri scheme
+        let res = check_qr(
+            &ctx.ctx,
+            "dcaccount:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+        )
+        .await;
+        assert_eq!(res.get_state(), LotState::QrAccount);
+        assert_eq!(res.get_text1().unwrap(), "example.org");
     }
 
     #[async_std::test]
@@ -600,5 +640,14 @@ mod tests {
         .await;
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
+
+        // Test it again with lowercased "dcaccount:" uri scheme
+        let res = check_qr(
+            &ctx.ctx,
+            "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
+        )
+        .await;
+        assert_eq!(res.get_state(), LotState::QrError);
+        assert!(res.get_text1().is_some());
     }
 }

src/securejoin.rs

@@ -10,10 +10,10 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::e2ee::*;
-use crate::error::Error;
+use crate::error::{bail, Error};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
-use crate::key::{dc_normalize_fingerprint, Key};
+use crate::key::{dc_normalize_fingerprint, DcKey, Key, SignedPublicKey};
 use crate::lot::LotState;
 use crate::message::Message;
 use crate::mimeparser::*;
@@ -140,12 +140,13 @@ pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> O
 }
 
 async fn get_self_fingerprint(context: &Context) -> Option<String> {
-    if let Some(self_addr) = context.get_config(Config::ConfiguredAddr).await {
+    match SignedPublicKey::load_self(context).await {
-        if let Some(key) = Key::from_self_public(context, self_addr, &context.sql).await {
+        Ok(key) => Some(Key::from(key).fingerprint()),
-            return Some(key.fingerprint());
+        Err(_) => {
+            warn!(context, "get_self_fingerprint(): failed to load key");
+            None
         }
     }
-    None
 }
 
 async fn cleanup(
@@ -374,24 +375,21 @@ async fn fingerprint_equals_sender(
     }
     false
 }
-#[derive(Fail, Debug)]
+#[derive(Debug, thiserror::Error)]
 pub(crate) enum HandshakeError {
-    #[fail(display = "Can not be called with special contact ID")]
+    #[error("Can not be called with special contact ID")]
     SpecialContactId,
-    #[fail(display = "Not a Secure-Join message")]
+    #[error("Not a Secure-Join message")]
     NotSecureJoinMsg,
-    #[fail(
+    #[error("Failed to look up or create chat for contact #{contact_id}")]
-        display = "Failed to look up or create chat for contact #{}",
-        contact_id
-    )]
     NoChat {
         contact_id: u32,
-        #[cause]
+        #[source]
         cause: Error,
     },
-    #[fail(display = "Chat for group {} not found", group)]
+    #[error("Chat for group {group} not found")]
     ChatNotFound { group: String },
-    #[fail(display = "No configured self address found")]
+    #[error("No configured self address found")]
     NoSelfAddr,
 }
 
@@ -424,8 +422,6 @@ pub(crate) async fn handle_securejoin_handshake(
     mime_message: &MimeMessage,
     contact_id: u32,
 ) -> Result<HandshakeMessage, HandshakeError> {
-    let own_fingerprint: String;
-
     if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return Err(HandshakeError::SpecialContactId);
     }
@@ -546,7 +542,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
-            own_fingerprint = get_self_fingerprint(context).await.unwrap();
+            let own_fingerprint = get_self_fingerprint(context).await.unwrap();
             joiner_progress!(context, contact_id, 400);
             context.bob.write().await.expects = DC_VC_CONTACT_CONFIRM;
 
@@ -666,11 +662,18 @@ pub(crate) async fn handle_securejoin_handshake(
                 }
             } else {
                 // Alice -> Bob
-                send_handshake_msg(context, contact_chat_id, "vc-contact-confirm", "", None, "")
+                send_handshake_msg(
-                    .await;
+                    context,
+                    contact_chat_id,
+                    "vc-contact-confirm",
+                    "",
+                    Some(fingerprint.clone()),
+                    "",
+                )
+                .await;
                 inviter_progress!(context, contact_id, 1000);
             }
-            Ok(HandshakeMessage::Done)
+            Ok(HandshakeMessage::Ignore) // "Done" would delete the message and break multi-device (the key from Autocrypt-header is needed)
         }
         "vg-member-added" | "vc-contact-confirm" => {
             /*=======================================================
@@ -762,27 +765,31 @@ pub(crate) async fn handle_securejoin_handshake(
             }
             secure_connection_established(context, contact_chat_id).await;
             context.bob.write().await.expects = 0;
-            if join_vg {
+
-                // Bob -> Alice
+            // Bob -> Alice
-                send_handshake_msg(
+            send_handshake_msg(
-                    context,
+                context,
-                    contact_chat_id,
+                contact_chat_id,
-                    "vg-member-added-received",
+                if join_vg {
-                    "",
+                    "vg-member-added-received"
-                    None,
+                } else {
-                    "",
+                    "vc-contact-confirm-received" // only for observe_securejoin_on_other_device()
-                )
+                },
-                .await;
+                "",
-            }
+                Some(scanned_fingerprint_of_alice),
+                "",
+            )
+            .await;
+
             context.bob.write().await.status = 1;
             context.stop_ongoing().await;
             Ok(if join_vg {
                 HandshakeMessage::Propagate
             } else {
-                HandshakeMessage::Done
+                HandshakeMessage::Ignore // "Done" deletes the message and breaks multi-device
             })
         }
-        "vg-member-added-received" => {
+        "vg-member-added-received" | "vc-contact-confirm-received" => {
             /*==========================================================
             ====              Alice - the inviter side              ====
             ====  Step 8 in "Out-of-band verified groups" protocol  ====
@@ -790,30 +797,26 @@ pub(crate) async fn handle_securejoin_handshake(
 
             if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
                 if contact.is_verified(context).await == VerifiedStatus::Unverified {
-                    warn!(context, "vg-member-added-received invalid.",);
+                    warn!(context, "{} invalid.", step);
                     return Ok(HandshakeMessage::Ignore);
                 }
-                inviter_progress!(context, contact_id, 800);
+                if join_vg {
-                inviter_progress!(context, contact_id, 1000);
+                    inviter_progress!(context, contact_id, 800);
-                let field_grpid = mime_message
+                    inviter_progress!(context, contact_id, 1000);
-                    .get(HeaderDef::SecureJoinGroup)
+                    let field_grpid = mime_message
-                    .map(|s| s.as_str())
+                        .get(HeaderDef::SecureJoinGroup)
-                    .unwrap_or_else(|| "");
+                        .map(|s| s.as_str())
-                let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid)
+                        .unwrap_or_else(|| "");
-                    .await
+                    if let Err(err) = chat::get_chat_id_by_grpid(context, &field_grpid).await {
-                    .map_err(|err| {
                         warn!(context, "Failed to lookup chat_id from grpid: {}", err);
-                        HandshakeError::ChatNotFound {
+                        return Err(HandshakeError::ChatNotFound {
                             group: field_grpid.to_string(),
-                        }
+                        });
-                    })?;
+                    }
-                context.call_cb(Event::SecurejoinMemberAdded {
+                }
-                    chat_id: group_chat_id,
+                Ok(HandshakeMessage::Ignore) // "Done" deletes the message and breaks multi-device
-                    contact_id,
-                });
-                Ok(HandshakeMessage::Done)
             } else {
-                warn!(context, "vg-member-added-received invalid.",);
+                warn!(context, "{} invalid.", step);
                 Ok(HandshakeMessage::Ignore)
             }
         }
@@ -825,8 +828,6 @@ pub(crate) async fn handle_securejoin_handshake(
 }
 
 /// observe_securejoin_on_other_device() must be called when a self-sent securejoin message is seen.
-/// currently, the message is only ignored, in the future,
-/// we may mark peers as verified accross devices:
 ///
 /// in a multi-device-setup, there may be other devices that "see" the handshake messages.
 /// if the seen messages seen are self-sent messages encrypted+signed correctly with our key,
@@ -843,17 +844,82 @@ pub(crate) async fn handle_securejoin_handshake(
 ///   the joining device has marked the peer as verified on vg-member-added/vc-contact-confirm
 ///   before sending vg-member-added-received - so, if we observe vg-member-added-received,
 ///   we can mark the peer as verified as well.
-///
+pub(crate) async fn observe_securejoin_on_other_device(
-/// to make this work, (a) some messages must not be deleted,
+    context: &Context,
-/// (b) we need a vc-contact-confirm-received message if bcc_self is set,
+    mime_message: &MimeMessage,
-/// (c) we should make sure, we do not only rely on the unencrypted To:-header for identifying the peer
+    contact_id: u32,
-/// (in handle_securejoin_handshake() we have the oob information for that)
-pub(crate) fn observe_securejoin_on_other_device(
-    _context: &Context,
-    _mime_message: &MimeMessage,
-    _contact_id: u32,
 ) -> Result<HandshakeMessage, HandshakeError> {
-    Ok(HandshakeMessage::Ignore)
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+        return Err(HandshakeError::SpecialContactId);
+    }
+    let step = mime_message
+        .get(HeaderDef::SecureJoin)
+        .ok_or(HandshakeError::NotSecureJoinMsg)?;
+    info!(context, "observing secure-join message \'{}\'", step);
+
+    let contact_chat_id =
+        match chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not).await {
+            Ok((chat_id, blocked)) => {
+                if blocked != Blocked::Not {
+                    chat_id.unblock(context).await;
+                }
+                chat_id
+            }
+            Err(err) => {
+                return Err(HandshakeError::NoChat {
+                    contact_id,
+                    cause: err,
+                });
+            }
+        };
+
+    match step.as_str() {
+        "vg-member-added"
+        | "vc-contact-confirm"
+        | "vg-member-added-received"
+        | "vc-contact-confirm-received" => {
+            if !encrypted_and_signed(
+                context,
+                mime_message,
+                get_self_fingerprint(context).await.unwrap_or_default(),
+            ) {
+                could_not_establish_secure_connection(
+                    context,
+                    contact_chat_id,
+                    "Message not encrypted correctly.",
+                )
+                .await;
+                return Ok(HandshakeMessage::Ignore);
+            }
+            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
+                Some(fp) => fp,
+                None => {
+                    could_not_establish_secure_connection(
+                        context,
+                        contact_chat_id,
+                        "Fingerprint not provided, please update Delta Chat on all your devices.",
+                    )
+                    .await;
+                    return Ok(HandshakeMessage::Ignore);
+                }
+            };
+            if mark_peer_as_verified(context, fingerprint).await.is_err() {
+                could_not_establish_secure_connection(
+                    context,
+                    contact_chat_id,
+                    format!("Fingerprint mismatch on observing {}.", step).as_ref(),
+                )
+                .await;
+                return Ok(HandshakeMessage::Ignore);
+            }
+            Ok(if step.as_str() == "vg-member-added" {
+                HandshakeMessage::Propagate
+            } else {
+                HandshakeMessage::Ignore
+            })
+        }
+        _ => Ok(HandshakeMessage::Ignore),
+    }
 }
 
 async fn secure_connection_established(context: &Context, contact_chat_id: ChatId) {

src/simplify.rs

@@ -1,13 +1,41 @@
+// protect lines starting with `--` against being treated as a footer.
+// for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B);
+// this should be invisible on most systems and there is no need to unescape it again
+// (which won't be done by non-deltas anyway)
+//
+// this escapes a bit more than actually needed by delta (eg. also lines as "-- footer"),
+// but for non-delta-compatibility, that seems to be better.
+// (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced)
+pub fn escape_message_footer_marks(text: &str) -> String {
+    if text.starts_with("--") {
+        "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-")
+    } else {
+        text.replace("\n--", "\n-\u{200B}-")
+    }
+}
+
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
 fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
+    let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
-        // quoted-printable may encode `-- ` to `-- =20` which is converted
-        // back to `--  `
         match line {
+            // some providers encode `-- ` to `-- =20` which results in `--  `
             "-- " | "--  " => return &lines[..ix],
+            // some providers encode `-- ` to `=2D-` which results in only `--`;
+            // use that only when no other footer is found
+            // and if the line before is empty and the line after is not empty
+            "--" => {
+                if (ix == 0 || lines[ix - 1] == "") && ix != lines.len() - 1 && lines[ix + 1] != ""
+                {
+                    nearly_standard_footer = Some(ix);
+                }
+            }
             _ => (),
         }
     }
+    if let Some(ix) = nearly_standard_footer {
+        return &lines[..ix];
+    }
     lines
 }
 
@@ -41,25 +69,27 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
     let lines = split_lines(&input);
     let (lines, is_forwarded) = skip_forward_header(&lines);
 
-    let lines = remove_message_footer(lines);
+    let original_lines = &lines;
-    let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
-    let (lines, has_bottom_quote) = if !is_chat_message {
-        remove_bottom_quote(lines)
-    } else {
-        (lines, false)
-    };
-    let (lines, has_top_quote) = if !is_chat_message {
-        remove_top_quote(lines)
-    } else {
-        (lines, false)
-    };
 
-    // re-create buffer from the remaining lines
+    let lines = remove_message_footer(lines);
-    let text = render_message(
+
-        lines,
+    let text = if is_chat_message {
-        has_top_quote,
+        render_message(lines, false, false)
-        has_nonstandard_footer || has_bottom_quote,
+    } else {
-    );
+        let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
+        let (lines, has_bottom_quote) = remove_bottom_quote(lines);
+        let (lines, has_top_quote) = remove_top_quote(lines);
+
+        if lines.iter().all(|it| it.trim().is_empty()) {
+            render_message(original_lines, false, false)
+        } else {
+            render_message(
+                lines,
+                has_top_quote,
+                has_nonstandard_footer || has_bottom_quote,
+            )
+        }
+    };
     (text, is_forwarded)
 }
 
@@ -67,14 +97,13 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
 /// Returns message body lines and a boolean indicating whether
 /// a message is forwarded or not.
 fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
-    if lines.len() >= 3
+    match lines {
-        && lines[0] == "---------- Forwarded message ----------"
+        ["---------- Forwarded message ----------", first_line, "", rest @ ..]
-        && lines[1].starts_with("From: ")
+            if first_line.starts_with("From: ") =>
-        && lines[2].is_empty()
+        {
-    {
+            (rest, true)
-        (&lines[3..], true)
+        }
-    } else {
+        _ => (lines, false),
-        (lines, false)
     }
 }
 
@@ -155,7 +184,8 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) ->
     if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
         ret += " [...]";
     }
-    ret
+    // redo escaping done by escape_message_footer_marks()
+    ret.replace("\u{200B}", "")
 }
 
 /**
@@ -204,6 +234,25 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_dont_remove_whole_message() {
+        let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
+        let (plain, is_forwarded) = simplify(input, false);
+        assert_eq!(
+            plain,
+            "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
+        );
+        assert!(!is_forwarded);
+    }
+
+    #[test]
+    fn test_chat_message() {
+        let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
+        let (plain, is_forwarded) = simplify(input, true);
+        assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
+        assert!(!is_forwarded);
+    }
+
     #[test]
     fn test_simplify_trim() {
         let input = "line1\n\r\r\rline2".to_string();
@@ -248,4 +297,46 @@ mod tests {
         assert_eq!(lines, &["not a quote", "> first", "> second"]);
         assert!(!has_top_quote);
     }
+
+    #[test]
+    fn test_escape_message_footer_marks() {
+        let esc = escape_message_footer_marks("--\n--text --in line");
+        assert_eq!(esc, "-\u{200B}-\n-\u{200B}-text --in line");
+
+        let esc = escape_message_footer_marks("--\r\n--text");
+        assert_eq!(esc, "-\u{200B}-\r\n-\u{200B}-text");
+    }
+
+    #[test]
+    fn test_remove_message_footer() {
+        let input = "text\n--\nno footer".to_string();
+        let (plain, _) = simplify(input, true);
+        assert_eq!(plain, "text\n--\nno footer");
+
+        let input = "text\n\n--\n\nno footer".to_string();
+        let (plain, _) = simplify(input, true);
+        assert_eq!(plain, "text\n\n--\n\nno footer");
+
+        let input = "text\n\n-- no footer\n\n".to_string();
+        let (plain, _) = simplify(input, true);
+        assert_eq!(plain, "text\n\n-- no footer");
+
+        let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
+        let (plain, _) = simplify(input, true);
+        assert_eq!(plain, "text\n\n--\nno footer");
+
+        let input = "text\n\n--\ntreated as footer when unescaped".to_string();
+        let (plain, _) = simplify(input.clone(), true);
+        assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
+        let escaped = escape_message_footer_marks(&input);
+        let (plain, _) = simplify(escaped, true);
+        assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
+
+        let input = "--\ntreated as footer when unescaped".to_string();
+        let (plain, _) = simplify(input.clone(), true);
+        assert_eq!(plain, ""); // see remove_message_footer() for some explanations
+        let escaped = escape_message_footer_marks(&input);
+        let (plain, _) = simplify(escaped, true);
+        assert_eq!(plain, "--\ntreated as footer when unescaped");
+    }
 }

src/smtp/mod.rs

@@ -12,39 +12,34 @@ use crate::context::Context;
 use crate::events::Event;
 use crate::login_param::{dc_build_tls, LoginParam};
 use crate::oauth2::*;
+use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "Bad parameters")]
+    #[error("Bad parameters")]
     BadParameters,
 
-    #[fail(display = "Invalid login address {}: {}", address, error)]
+    #[error("Invalid login address {address}: {error}")]
     InvalidLoginAddress {
         address: String,
-        #[cause]
+        #[source]
         error: error::Error,
     },
 
-    #[fail(display = "SMTP failed to connect: {:?}", _0)]
+    #[error("SMTP: failed to connect: {0:?}")]
-    ConnectionFailure(#[cause] smtp::error::Error),
+    ConnectionFailure(#[source] smtp::error::Error),
 
-    #[fail(display = "SMTP: failed to setup connection {:?}", _0)]
+    #[error("SMTP: failed to setup connection {0:?}")]
-    ConnectionSetupFailure(#[cause] smtp::error::Error),
+    ConnectionSetupFailure(#[source] smtp::error::Error),
 
-    #[fail(display = "SMTP: oauth2 error {:?}", _0)]
+    #[error("SMTP: oauth2 error {address}")]
     Oauth2Error { address: String },
 
-    #[fail(display = "TLS error")]
+    #[error("TLS error")]
-    Tls(#[cause] async_native_tls::Error),
+    Tls(#[from] async_native_tls::Error),
-}
-
-impl From<async_native_tls::Error> for Error {
-    fn from(err: async_native_tls::Error) -> Error {
-        Error::Tls(err)
-    }
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -172,7 +167,18 @@ impl Smtp {
             .timeout(Some(Duration::from_secs(SMTP_TIMEOUT)));
 
         let mut trans = client.into_transport();
-        trans.connect().await.map_err(Error::ConnectionFailure)?;
+        if let Err(err) = trans.connect().await {
+            let message = context
+                .stock_string_repl_str2(
+                    StockMessage::ServerResponse,
+                    format!("SMTP {}:{}", domain, port),
+                    err.to_string(),
+                )
+                .await;
+
+            emit_event!(context, Event::ErrorNetwork(message));
+            return Err(Error::ConnectionFailure(err));
+        }
 
         self.transport = Some(trans);
         self.last_success = Some(Instant::now());

src/smtp/send.rs

@@ -8,15 +8,15 @@ use crate::events::Event;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "Envelope error: {}", _0)]
+    #[error("Envelope error: {}", _0)]
-    EnvelopeError(#[cause] async_smtp::error::Error),
+    EnvelopeError(#[from] async_smtp::error::Error),
 
-    #[fail(display = "Send error: {}", _0)]
+    #[error("Send error: {}", _0)]
-    SendError(#[cause] async_smtp::smtp::error::Error),
+    SendError(#[from] async_smtp::smtp::error::Error),
 
-    #[fail(display = "SMTP has no transport")]
+    #[error("SMTP has no transport")]
     NoTransport,
 }
 

src/sql.rs

@@ -27,50 +27,28 @@ macro_rules! paramsv {
     };
 }
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "Sqlite Error: {:?}", _0)]
+    #[error("Sqlite Error: {0:?}")]
-    Sql(#[cause] rusqlite::Error),
+    Sql(#[from] rusqlite::Error),
-    #[fail(display = "Sqlite Connection Pool Error: {:?}", _0)]
+    #[error("Sqlite Connection Pool Error: {0:?}")]
-    ConnectionPool(#[cause] r2d2::Error),
+    ConnectionPool(#[from] r2d2::Error),
-    #[fail(display = "Sqlite: Connection closed")]
+    #[error("Sqlite: Connection closed")]
     SqlNoConnection,
-    #[fail(display = "Sqlite: Already open")]
+    #[error("Sqlite: Already open")]
     SqlAlreadyOpen,
-    #[fail(display = "Sqlite: Failed to open")]
+    #[error("Sqlite: Failed to open")]
     SqlFailedToOpen,
-    #[fail(display = "{:?}", _0)]
+    #[error("{0}")]
-    Io(#[cause] std::io::Error),
+    Io(#[from] std::io::Error),
-    #[fail(display = "{:?}", _0)]
+    #[error("{0:?}")]
-    BlobError(#[cause] crate::blob::BlobError),
+    BlobError(#[from] crate::blob::BlobError),
+    #[error("{0}")]
+    Other(#[from] crate::error::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-impl From<rusqlite::Error> for Error {
-    fn from(err: rusqlite::Error) -> Error {
-        Error::Sql(err)
-    }
-}
-
-impl From<r2d2::Error> for Error {
-    fn from(err: r2d2::Error) -> Error {
-        Error::ConnectionPool(err)
-    }
-}
-
-impl From<std::io::Error> for Error {
-    fn from(err: std::io::Error) -> Error {
-        Error::Io(err)
-    }
-}
-
-impl From<crate::blob::BlobError> for Error {
-    fn from(err: crate::blob::BlobError) -> Error {
-        Error::BlobError(err)
-    }
-}
-
 /// A wrapper around the underlying Sqlite3 object.
 #[derive(DebugStub)]
 pub struct Sql {
@@ -107,11 +85,13 @@ impl Sql {
     pub async fn open<T: AsRef<Path>>(&self, context: &Context, dbfile: T, readonly: bool) -> bool {
         match open(context, self, dbfile, readonly).await {
             Ok(_) => true,
-            Err(crate::error::Error::SqlError(Error::SqlAlreadyOpen)) => false,
+            Err(err) => match err.downcast_ref::<Error>() {
-            Err(_) => {
+                Some(Error::SqlAlreadyOpen) => false,
-                self.close().await;
+                _ => {
-                false
+                    self.close().await;
-            }
+                    false
+                }
+            },
         }
     }
 
@@ -257,6 +237,28 @@ impl Sql {
         .await
     }
 
+    /// Execute a query which is expected to return zero or one row.
+    pub async fn query_row_optional<T, F>(
+        &self,
+        sql: impl AsRef<str>,
+        params: Vec<&dyn crate::ToSql>,
+        f: F,
+    ) -> Result<Option<T>>
+    where
+        F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
+    {
+        match self.query_row(sql, params, f).await {
+            Ok(res) => Ok(Some(res)),
+            Err(Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => Ok(None),
+            Err(Error::Sql(rusqlite::Error::InvalidColumnType(
+                _,
+                _,
+                rusqlite::types::Type::Null,
+            ))) => Ok(None),
+            Err(err) => Err(err),
+        }
+    }
+
     /// Executes a query which is expected to return one row and one
     /// column. If the query does not return a value or returns SQL
     /// `NULL`, returns `Ok(None)`.
@@ -268,19 +270,8 @@ impl Sql {
     where
         T: rusqlite::types::FromSql,
     {
-        match self
+        self.query_row_optional(query, params, |row| row.get::<_, T>(0))
-            .query_row(query, params, |row| row.get::<_, T>(0))
             .await
-        {
-            Ok(res) => Ok(Some(res)),
-            Err(Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => Ok(None),
-            Err(Error::Sql(rusqlite::Error::InvalidColumnType(
-                _,
-                _,
-                rusqlite::types::Type::Null,
-            ))) => Ok(None),
-            Err(err) => Err(err),
-        }
     }
 
     /// Not resultified version of `query_get_value_result`. Returns
@@ -297,7 +288,7 @@ impl Sql {
         match self.query_get_value_result(query, params).await {
             Ok(res) => res,
             Err(err) => {
-                error!(context, "sql: Failed query_row: {}", err);
+                warn!(context, "sql: Failed query_row: {}", err);
                 None
             }
         }
@@ -1332,6 +1323,8 @@ async fn open(
     Ok(())
 }
 
+/// Removes from the database locally deleted messages that also don't
+/// have a server UID.
 async fn prune_tombstones(context: &Context) -> Result<()> {
     context
         .sql

src/stock.rs

@@ -10,7 +10,7 @@ use crate::chat;
 use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::contact::*;
 use crate::context::Context;
-use crate::error::Error;
+use crate::error::{bail, Error};
 use crate::message::Message;
 use crate::param::Param;
 use crate::stock::StockMessage::{DeviceMessagesHint, WelcomeMessage};
@@ -35,12 +35,6 @@ pub enum StockMessage {
     #[strum(props(fallback = "Draft"))]
     Draft = 3,
 
-    #[strum(props(fallback = "%1$s member(s)"))]
-    Member = 4,
-
-    #[strum(props(fallback = "%1$s contact(s)"))]
-    Contact = 6,
-
     #[strum(props(fallback = "Voice message"))]
     VoiceMessage = 7,
 
@@ -136,9 +130,6 @@ pub enum StockMessage {
     ))]
     AcSetupMsgBody = 43,
 
-    #[strum(props(fallback = "Messages I sent to myself"))]
-    SelfTalkSubTitle = 50,
-
     #[strum(props(fallback = "Cannot login as %1$s."))]
     CannotLogin = 60,
 
@@ -317,7 +308,8 @@ impl Context {
         from_id: u32,
     ) -> String {
         let insert1 = if id == StockMessage::MsgAddMember || id == StockMessage::MsgDelMember {
-            let contact_id = Contact::lookup_id_by_addr(self, param1.as_ref()).await;
+            let contact_id =
+                Contact::lookup_id_by_addr(self, param1.as_ref(), Origin::Unknown).await;
             if contact_id != 0 {
                 Contact::get_by_id(self, contact_id)
                     .await
@@ -449,9 +441,9 @@ mod tests {
         // uses %1$s substitution
         assert_eq!(
             t.ctx
-                .stock_string_repl_str(StockMessage::Member, "42")
+                .stock_string_repl_str(StockMessage::MsgAddMember, "Foo")
                 .await,
-            "42 member(s)"
+            "Member Foo added."
         );
         // We have no string using %1$d to test...
     }
@@ -460,8 +452,10 @@ mod tests {
     async fn test_stock_string_repl_int() {
         let t = dummy_context().await;
         assert_eq!(
-            t.ctx.stock_string_repl_int(StockMessage::Member, 42).await,
+            t.ctx
-            "42 member(s)"
+                .stock_string_repl_int(StockMessage::MsgAddMember, 42)
+                .await,
+            "Member 42 added."
         );
     }