Add some information about mailing list managers.

Remove additional chat type MailingList, make them GroupLists with a special param, make can_send() return false, code style and deduplication

.circleci/config.yml

@@ -7,10 +7,6 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
-  python:
-    docker:
-      - image: 3.7.7-stretch
-            
 
 restore-workspace: &restore-workspace
   attach_workspace:

CHANGELOG.md

@@ -1,81 +1,5 @@
 # Changelog 
 
-## 1.33.0
-
-- let `dc_set_muted()` also mute one-to-one chats #1470
-
-- fix a bug that led to load and traffic if the server does not use sent-folder
-  #1472
-
-
-## 1.32.0
-
-- fix endless loop when trying to download messages with bad RFC Message-ID,
-  also be more reliable on similar errors #1463 #1466 #1462
-
-- fix bug with comma in contact request #1438
-
-- do not refer to hidden messages on replies #1459
-
-- improve error handling #1468 #1465 #1464
-
-
-## 1.31.0
-
-- always describe the context of the displayed error #1451
-
-- do not emit `DC_EVENT_ERROR` when message sending fails;
-  `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451
-
-- new config-option `media_quality` #1449
-
-- try over if writing message to database fails #1447
-
-
-## 1.30.0
-
-- expunge deleted messages #1440
-
-- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439
-
-
-## 1.29.0
-
-- new config options `delete_device_after` and `delete_server_after`,
-  each taking an amount of seconds after which messages
-  are deleted from the device and/or the server #1310 #1335 #1411 #1417 #1423
-
-- new api `dc_estimate_deletion_cnt()` to estimate the effect
-  of `delete_device_after` and `delete_server_after`
-
-- use Ed25519 keys by default, these keys are much shorter
-  than RSA keys, which results in saving traffic and speed improvements #1362
-
-- improve message ellipsizing #1397 #1430
-
-- emit `DC_EVENT_ERROR_NETWORK` also on smtp-errors #1378
-
-- do not show badly formatted non-delta-messages as empty #1384
-
-- try over SMTP on potentially recoverable error 5.5.0 #1379
-
-- remove device-chat from forward-to-chat-list #1367
-
-- improve group-handling #1368
-
-- `dc_get_info()` returns uptime (how long the context is in use)
-
-- python improvements and adaptions #1408 #1415
-
-- log to the stdout and stderr in tests #1416
-
-- refactoring, code improvements #1363 #1365 #1366 #1370 #1375 #1389 #1390 #1418 #1419
-
-- removed api: `dc_chat_get_subtitle()`, `dc_get_version_str()`, `dc_array_add_id()`
-
-- removed events: `DC_EVENT_MEMBER_ADDED`, `DC_EVENT_MEMBER_REMOVED`
-
-
 ## 1.28.0
 
 - new flag DC_GCL_FOR_FORWARDING for dc_get_chatlist()

Cargo.lock

@@ -629,7 +629,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.33.0"
+version = "1.28.0"
 dependencies = [
  "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -695,10 +695,10 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.33.0"
+version = "1.28.0"
 dependencies = [
  "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
- "deltachat 1.33.0",
+ "deltachat 1.28.0",
  "human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.33.0"
+version = "1.28.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"

README.md

@@ -17,9 +17,8 @@ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-cargo run --example repl -- ~/deltachat-db
+cargo run --example repl -- /path/to/db
 ```
-where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Configure your account (if not already configured):
 

ci_scripts/ci_upload.sh

@@ -37,7 +37,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
-pip3 install -U pip setuptools
+pip3 install -U pip
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN

ci_scripts/cleanup_devpi_indices.py

@@ -48,7 +48,7 @@ def run():
         projectnames = get_projectnames(baseurl, username, indexname)
         if indexname == "master" or not indexname:
             continue
-        clear_index = not projectnames
+        assert projectnames == ["deltachat"]
         for projectname in projectnames:
             dates = get_release_dates(baseurl, username, indexname, projectname)
             if not dates:
@@ -60,11 +60,8 @@ def run():
                 date = datetime.datetime(*max(dates))
             if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
                 assert username and indexname
-                clear_index = True
-                break
-        if clear_index:
-            url = baseurl + username + "/" + indexname
-            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+                url = baseurl + username + "/" + indexname
+                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
 
 
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.33.0"
+version = "1.28.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -41,7 +41,7 @@ typedef struct _dc_provider dc_provider_t;
  * uintptr_t event_handler_func(dc_context_t* context, int event,
  *                              uintptr_t data1, uintptr_t data2)
  * {
- *     return 0;
+ *     return 0; // for unhandled events, it is always safe to return 0
  * }
  *
  * dc_context_t* context = dc_context_new(event_handler_func, NULL, NULL);
@@ -208,7 +208,7 @@ typedef struct _dc_provider dc_provider_t;
  * @param event one of the @ref DC_EVENT constants
  * @param data1 depends on the event parameter
  * @param data2 depends on the event parameter
- * @return events do not expect a return value, just always return 0
+ * @return return 0 unless stated otherwise in the event parameter documentation
  */
 typedef uintptr_t (*dc_callback_t) (dc_context_t* context, int event, uintptr_t data1, uintptr_t data2);
 
@@ -229,7 +229,7 @@ typedef uintptr_t (*dc_callback_t) (dc_context_t* context, int event, uintptr_t
  *       otherwise!
  *     - The callback SHOULD return _fast_, for GUI updates etc. you should
  *       post yourself an asynchronous message to your GUI thread, if needed.
- *     - events do not expect a return value, just always return 0.
+ *     - If not mentioned otherweise, the callback should return 0.
  * @param userdata can be used by the client for any purpuse.  He finds it
  *     later in dc_get_userdata().
  * @param os_name is only for decorative use
@@ -342,8 +342,7 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `displayname`  = Own name to use when sending messages.  MUAs are allowed to spread this way eg. using CC, defaults to empty
  * - `selfstatus`   = Own status to display eg. in email footers, defaults to a standard text
- * - `selfavatar`   = File containing avatar. Will immediately be copied to the 
- *                    `blobdir`; the original image will not be needed anymore.
+ * - `selfavatar`   = File containing avatar. Will be copied to blob directory.
  *                    NULL to remove the avatar.
  *                    It is planned for future versions
  *                    to send this image together with the next messages.
@@ -378,21 +377,10 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    >=1=seconds, after which messages are deleted automatically from the device.
  *                    Messages in the "saved messages" chat (see dc_chat_is_self_talk()) are skipped.
  *                    Messages are deleted whether they were seen or not, the UI should clearly point that out.
- *                    See also dc_estimate_deletion_cnt().
  * - `delete_server_after` = 0=do not delete messages from server automatically (default),
- *                    1=delete messages directly after receiving from server, mvbox is skipped.
- *                    >1=seconds, after which messages are deleted automatically from the server, mvbox is used as defined.
+ *                    >=1=seconds, after which messages are deleted automatically from the server.
  *                    "Saved messages" are deleted from the server as well as
  *                    emails matching the `show_emails` settings above, the UI should clearly point that out.
- *                    See also dc_estimate_deletion_cnt().
- * - `media_quality` = DC_MEDIA_QUALITY_BALANCED (0) =
- *                    good outgoing images/videos/voice quality at reasonable sizes (default)
- *                    DC_MEDIA_QUALITY_WORSE (1)
- *                    allow worse images/videos/voice quality to gain smaller sizes,
- *                    suitable for providers or areas known to have a bad connection.
- *                    In contrast to other options, the implementation of this option is currently up to the UIs;
- *                    this may change in future, however,
- *                    having the option in the core allows provider-specific-defaults already today.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -960,10 +948,8 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
  *       if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and
  *       the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are _any_ archived
  *       chats
- *     - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
- *       and hides the "Device chat" and the deaddrop.
+ *     - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist,
  *       typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
- *       to also hide the archive link.
  *     - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
  *       to the list (may be used eg. for selecting chats on forwarding, the flag is
  *       not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -1323,7 +1309,6 @@ int             dc_get_fresh_msg_cnt         (dc_context_t* context, uint32_t ch
  * by the dc_set_config()-options `delete_device_after` or `delete_server_after`.
  * This is typically used to show the estimated impact to the user before actually enabling ephemeral messages.
  *
- * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param from_server 1=Estimate deletion count for server, 0=Estimate deletion count for device
  * @param seconds Count messages older than the given number of seconds.
@@ -1627,10 +1612,8 @@ int             dc_set_chat_name             (dc_context_t* context, uint32_t ch
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
  * @param chat_id The chat ID to set the image for.
- * @param image Full path of the image to use as the group image. The image will immediately be copied to the 
- *     `blobdir`; the original image will not be needed anymore.
- *      If you pass NULL here, the group image is deleted (for promoted groups, all members are informed about 
- *      this change anyway).
+ * @param image Full path of the image to use as the group image.  If you pass NULL here,
+ *     the group image is deleted (for promoted groups, all members are informed about this change anyway).
  * @return 1=success, 0=error
  */
 int             dc_set_chat_profile_image    (dc_context_t* context, uint32_t chat_id, const char* image);
@@ -1640,7 +1623,7 @@ int             dc_set_chat_profile_image    (dc_context_t* context, uint32_t ch
 /**
  * Set mute duration of a chat.
  *
- * The ui can then call dc_chat_is_muted() when receiving a new message to decide whether it should trigger an notification.
+ * This value can be checked by the ui upon receiving a new message to decide whether it should trigger an notification.
  *
  * Sends out #DC_EVENT_CHAT_MODIFIED.
  *
@@ -1803,7 +1786,7 @@ int             dc_may_be_valid_addr         (const char* addr);
 
 /**
  * Check if an e-mail address belongs to a known and unblocked contact.
- * To get a list of all known and unblocked contacts, use dc_get_contacts().
+ * Known and unblocked contacts will be returned by dc_get_contacts().
  *
  * To validate an e-mail address independently of the contact database
  * use dc_may_be_valid_addr().
@@ -1811,8 +1794,7 @@ int             dc_may_be_valid_addr         (const char* addr);
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @param addr The e-mail-address to check.
- * @return Contact ID of the contact belonging to the e-mail-address
- *     or 0 if there is no contact that is or was introduced by an accepted contact.  
+ * @return 1=address is a contact in use, 0=address is not a contact in use.
  */
 uint32_t        dc_lookup_contact_id_by_addr (dc_context_t* context, const char* addr);
 
@@ -2845,6 +2827,19 @@ int             dc_chat_get_type             (const dc_chat_t* chat);
 char*           dc_chat_get_name             (const dc_chat_t* chat);
 
 
+/*
+ * Get a subtitle for a chat.  The subtitle is eg. the email-address or the
+ * number of group members.
+ *
+ * Deprecated function. Subtitles should be created in the ui
+ * where plural forms and other specials can be handled more gracefully.
+ *
+ * @param chat The chat object to calulate the subtitle for.
+ * @return Subtitle as a string. Must be released using dc_str_unref() after usage. Never NULL.
+ */
+char*           dc_chat_get_subtitle         (const dc_chat_t* chat);
+
+
 /**
  * Get the chat's profile image.
  * For groups, this is the image set by any group member
@@ -2978,7 +2973,7 @@ int             dc_chat_is_sending_locations (const dc_chat_t* chat);
 
 
 /**
- * Check whether the chat is currently muted (can be changed by dc_set_chat_mute_duration()).
+ * Check whether the chat is currently muted
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -4165,6 +4160,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * These constants are used as events
  * reported to the callback given to dc_context_new().
+ * If you do not want to handle an event, it is always safe to return 0,
+ * so there is no need to add a "case" for every event.
  *
  * @addtogroup DC_EVENT
  * @{
@@ -4179,6 +4176,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_INFO                     100
 
@@ -4189,6 +4187,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_SMTP_CONNECTED           101
 
@@ -4199,6 +4198,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_IMAP_CONNECTED           102
 
@@ -4208,6 +4208,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_SMTP_MESSAGE_SENT        103
 
@@ -4217,6 +4218,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_IMAP_MESSAGE_DELETED   104
 
@@ -4226,6 +4228,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
@@ -4235,6 +4238,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) folder name.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_IMAP_FOLDER_EMPTIED  106
 
@@ -4244,6 +4248,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) path name
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_NEW_BLOB_FILE 150
 
@@ -4253,6 +4258,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) path name
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_DELETED_BLOB_FILE 151
 
@@ -4265,6 +4271,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Warning string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_WARNING                  300
 
@@ -4287,6 +4294,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     Some error strings are taken from dc_set_stock_translation(),
  *     however, most error strings will be in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_ERROR                    400
 
@@ -4310,6 +4318,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     0=subsequent network error, should be logged only
  * @param data2 (const char*) Error string, always set, never NULL.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_ERROR_NETWORK            401
 
@@ -4325,6 +4334,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified
  *     and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_ERROR_SELF_NOT_IN_GROUP  410
 
@@ -4338,6 +4348,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id for single added messages
  * @param data2 (int) msg_id for single added messages
+ * @return 0
  */
 #define DC_EVENT_MSGS_CHANGED             2000
 
@@ -4350,6 +4361,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
+ * @return 0
  */
 #define DC_EVENT_INCOMING_MSG             2005
 
@@ -4360,6 +4372,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
+ * @return 0
  */
 #define DC_EVENT_MSG_DELIVERED            2010
 
@@ -4370,6 +4383,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
+ * @return 0
  */
 #define DC_EVENT_MSG_FAILED               2012
 
@@ -4380,6 +4394,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
+ * @return 0
  */
 #define DC_EVENT_MSG_READ                 2015
 
@@ -4392,6 +4407,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_CHAT_MODIFIED            2020
 
@@ -4401,6 +4417,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) If not 0, this is the contact_id of an added contact that should be selected.
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_CONTACTS_CHANGED         2030
 
@@ -4413,6 +4430,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     If the locations of several contacts have been changed,
  *     eg. after calling dc_delete_all_locations(), this parameter is set to 0.
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_LOCATION_CHANGED         2035
 
@@ -4422,6 +4440,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_CONFIGURE_PROGRESS       2041
 
@@ -4431,6 +4450,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_IMEX_PROGRESS            2051
 
@@ -4445,6 +4465,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 (const char*) Path and file name.
  *     Must not be unref'd or modified and is valid only until the callback returns.
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_IMEX_FILE_WRITTEN        2052
 
@@ -4462,6 +4483,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified".
  *     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
  *     1000=Protocol finished for this contact.
+ * @return 0
  */
 #define DC_EVENT_SECUREJOIN_INVITER_PROGRESS      2060
 
@@ -4477,9 +4499,29 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data2 (int) Progress as:
  *     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
  *     (Bob has verified alice and waits until Alice does the same for him)
+ * @return 0
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
+
+/**
+ * This event is sent for each member that gets added to a (verified or unverified) chat. 
+ *
+ * @param data1 (int) chat_id
+ * @param data2 (int) contact_id
+ * @return 0
+ */
+#define DC_EVENT_MEMBER_ADDED 2062
+
+/**
+ * This event is sent for each member that gets removed from a (verified or unverified) chat. 
+ *
+ * @param data1 (int) chat_id
+ * @param data2 (int) contact_id
+ * @return 0
+ */
+#define DC_EVENT_MEMBER_REMOVED 2063
+
 /**
  * @}
  */
@@ -4495,6 +4537,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EVENT_DATA2_IS_STRING(e)  ((e)>=100 && (e)<=499)
 #define DC_EVENT_RETURNS_INT(e)      ((e)==DC_EVENT_IS_OFFLINE) // not used anymore
 #define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING) // not used anymore
+char*           dc_get_version_str           (void); // deprecated
+void            dc_array_add_id              (dc_array_t*, uint32_t); // deprecated
 #define dc_archive_chat(a,b,c)  dc_set_chat_visibility((a), (b), (c)? 1 : 0) // not used anymore
 #define dc_chat_get_archived(a) (dc_chat_get_visibility((a))==1? 1 : 0)      // not used anymore
 
@@ -4506,14 +4550,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_SHOW_EMAILS_ACCEPTED_CONTACTS 1
 #define DC_SHOW_EMAILS_ALL               2
 
-
-/*
- * Values for dc_get|set_config("media_quality")
- */
-#define DC_MEDIA_QUALITY_BALANCED 0
-#define DC_MEDIA_QUALITY_WORSE    1
-
-
 /*
  * Values for dc_get|set_config("key_gen_type")
  */
@@ -4632,6 +4668,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_STR_NOMESSAGES                 1
 #define DC_STR_SELF                       2
 #define DC_STR_DRAFT                      3
+#define DC_STR_MEMBER                     4
+#define DC_STR_CONTACT                    6
 #define DC_STR_VOICEMESSAGE               7
 #define DC_STR_DEADDROP                   8
 #define DC_STR_IMAGE                      9
@@ -4663,6 +4701,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_STR_STARREDMSGS                41
 #define DC_STR_AC_SETUP_MSG_SUBJECT       42
 #define DC_STR_AC_SETUP_MSG_BODY          43
+#define DC_STR_SELFTALK_SUBTITLE          50
 #define DC_STR_CANNOT_LOGIN               60
 #define DC_STR_SERVER_RESPONSE            61
 #define DC_STR_MSGACTIONBYUSER            62

deltachat-ffi/src/lib.rs

@@ -13,7 +13,7 @@ extern crate human_panic;
 extern crate num_traits;
 extern crate serde_json;
 
-use std::collections::BTreeMap;
+use std::collections::HashMap;
 use std::convert::TryInto;
 use std::ffi::CString;
 use std::fmt::Write;
@@ -196,6 +196,25 @@ impl ContextWrapper {
                         progress as uintptr_t,
                     );
                 }
+                Event::SecurejoinMemberAdded {
+                    chat_id,
+                    contact_id,
+                }
+                | Event::MemberAdded {
+                    chat_id,
+                    contact_id,
+                }
+                | Event::MemberRemoved {
+                    chat_id,
+                    contact_id,
+                } => {
+                    ffi_cb(
+                        self,
+                        event_id,
+                        chat_id.to_u32() as uintptr_t,
+                        contact_id as uintptr_t,
+                    );
+                }
             }
         }
     }
@@ -434,7 +453,7 @@ pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c
 }
 
 fn render_info(
-    info: BTreeMap<&'static str, String>,
+    info: HashMap<&'static str, String>,
 ) -> std::result::Result<String, std::fmt::Error> {
     let mut res = String::new();
     for (key, value) in &info {
@@ -465,6 +484,11 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
         .unwrap_or_else(|_| ptr::null_mut())
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_version_str() -> *mut libc::c_char {
+    context::get_version_str().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_configure(context: *mut dc_context_t) {
     if context.is_null() {
@@ -2080,6 +2104,16 @@ pub unsafe extern "C" fn dc_array_unref(a: *mut dc_array::dc_array_t) {
     Box::from_raw(a);
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_array_add_id(array: *mut dc_array_t, item: libc::c_uint) {
+    if array.is_null() {
+        eprintln!("ignoring careless call to dc_array_add_id()");
+        return;
+    }
+
+    (*array).add_id(item);
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_cnt(array: *const dc_array_t) -> libc::size_t {
     if array.is_null() {
@@ -2407,6 +2441,19 @@ pub unsafe extern "C" fn dc_chat_get_name(chat: *mut dc_chat_t) -> *mut libc::c_
     ffi_chat.chat.get_name().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_get_subtitle(chat: *mut dc_chat_t) -> *mut libc::c_char {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_get_subtitle()");
+        return "".strdup();
+    }
+    let ffi_chat = &*chat;
+    let ffi_context: &ContextWrapper = &*ffi_chat.context;
+    ffi_context
+        .with_inner(|ctx| ffi_chat.chat.get_subtitle(ctx).strdup())
+        .unwrap_or_else(|_| "".strdup())
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_chat_get_profile_image(chat: *mut dc_chat_t) -> *mut libc::c_char {
     if chat.is_null() {

draft/mailing_list.rst

@@ -0,0 +1,19 @@
+2. Mailing lists are now only detected by the ListId header, because how should DC sort mailing lists if the ListId header is unset and only the precedence header says it's a mailing list (?). We could just hide these emails (with "junk" or "list" precedence), as we did before, if there is a reason to do so. 
+
+   I do not actually understand why such emails were hidden in the first place, though; if there is an automatic answer stating that someone is out of office or if an email could not be delivered (these are the usual reasons why such emails are sent), I would want to know about this as a user.
+   
+  Björn: to the ListId-header: it is already an improvement to use only that and to keep ignoring mails with Precedence-header. historically, i ignored all these mails to reduce noise that is created from them (eg. contacts should not be added to the suggestion list...) and to be able to concentrate on other things first.
+   
+   
+3. for `List-Id: foo <bla>`, bla now is the id and foo is the name. For GitHub and GitLab notifications this is bad because all notifications will go into one chat. Maybe we should instead take the "in-reply-to" header to find out what messages belong together. For normal mailing lists, of course, this will create a second chat if someone does not use the "Reply" button to write to that mailing list.
+
+  Björn: well, having all notifications in one chat is not that bad. i would just follow the guidelines for now, use the ID and not the Name.
+
+
+4. Currently a mailing list is shown as an empty group (ChatType `Group`) ("0 members"). 
+
+   Maybe we should change the ChatType to `Single` because this way, the UI would fit better. Disadvantage: We can't show the different senders of the messages. 
+   
+   Or we introduce a `ChatType` `MailingList`. Very big disadvantage: We would have to adapt all UI project and it would be nice if we could keep the changes within the core.
+
+5. Contacts from mailings lists stay "unknown" now and are not shown in the contacts suggestion list..

draft/mailing_list_current_state.rst

@@ -0,0 +1,13 @@
+1. Mailing lists with Precedence-header and without List-Id are shown as normal messages.
+   
+2. for `List-Id: foo <bla>`, bla now is the id and foo is the name. If foo is not present, bla is taken as the name as well. For GitHub and GitLab notifications all notifications will go into one chat. To distinguish, all subjects are shown in mailing lists.
+
+3. Currently a mailing list is shown as a group with SELF and the List-Id. I could not find any stable way to get a mail address that could be called the "mailing list address". To get this to work, I disabled the may_be_valid_addr check. (to be discussed...)
+
+4. Contacts from mailings lists stay "unknown" and are not shown in the contacts suggestion list.
+
+5. In general, only the From: address is taken as the author. If the domain matches with the List-Id domain (like `deltachat.github.com` and `Hocuri <notifications@notification.github.com>`) then the display name is added to the email address -> `Hocuri - notifications@notification.github.com`. This is not really nice, but as the UIs distinguish bettween contacts only based on the address this was the only way I could find without changing the API. 
+
+   TODO: Prevent the user from sending emails to such contacts (like Hocuri - notifications@notification.github.com)
+
+6. You can't send to mailing lists.

draft/mailing_list_managers.md

@@ -0,0 +1,71 @@
+# Mailing list integration of Delta Chat
+
+A collection of information to help with the integration of mailing lists into Delta Chat.
+
+## Chat name
+
+How should the chat be titled?
+
+It could be taken from the email header `List-Id`:
+
+### Mailman 
+
+* Schema: `${list-description} <${list-local-part}.${list-domain}>`
+* $list-description could be many words, should be truncated. Could also be empty, though.
+
+### Schleuder
+* Schema: `<${list-local-part}.${list-domain}>`
+* No name available.
+* Could be absent: <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L357-358>.
+
+
+### Sympa
+* Schema: `<${list-local-part}.${list-domain}>`
+* No name available.
+* Always present <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/Spindle/TransformOutgoing.pm#L110-L118>, <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/List.pm#L6832-L6833>.
+
+
+## Sender name
+
+What's the name of the person that actually sent the message?
+
+Some MLM change the sender information to avoid problems with DMARC.
+
+### Mailman
+
+* Since v2.1.16 the `From` depends on the list's configuration and possibly on the sender-domain's DMARC-configuration — i.e. messages from the same list can arrive one of the Variants A-D!
+* Documentation of config options:
+ * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-544>,
+ * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-163>,
+ * <https://wiki.list.org/DEV/DMARC>
+* Variant A: `From` is unchanged.
+* Variant B. `From` is mangled like this: `${sender-name} via ${list-name} <${list-addr-spec}>`. The original sender is put into `Reply-To`.
+* Variant C. `From` is mangled like this: `${sender-name} <${encoded-sender-addr-spec}@${list-domain}>`. The original sender is put into `Reply-To`.
+* Variant D: `From` is set to ${list-name-addr}, the originally sent message is included as mime-part.
+* Variant E: `From` is set to ${list-name-addr}, original sender information is removed.
+
+### Schleuder
+
+* Visible only in mime-body (which is possibly encrypted), or not at all.
+* The first `text/plain` mime-part may include the information, as taken from the original message: `From: ${sender-name-addr}`.
+
+### Sympa
+* Depends on the list's configuration.
+* Documentation:
+ * <https://sympa-community.github.io/manual/customize/dmarc-protection.html>,
+ * <https://sympa-community.github.io/gpldoc/man/sympa.conf.5.html#dmarc-protection>.
+* Variant A: `From` is unchanged.
+* Variant B: `From` is mangled like Mailman Variant B, but the original sender information is put into `X-Original-From`.
+
+
+## Autocrypt
+
+### Mailman
+* Not supported.
+
+### Schleuder
+* A list's key is included in sent messages (as of version 3.5.0) (optional, by default active): <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L349-355>.
+* Incoming keys are not yet looked at (that feature is planned: <https://0xacab.org/schleuder/schleuder/issues/435>).
+
+### Sympa
+* Not supported.

python/examples/group_tracking.py

@@ -17,31 +17,19 @@ class GroupTrackingPlugin:
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))
 
-    @account_hookimpl
-    def ac_outgoing_message(self, message):
-        print("ac_outgoing_message:", message)
-
     @account_hookimpl
     def ac_configure_completed(self, success):
-        print("ac_configure_completed:", success)
+        print("*** ac_configure_completed:", success)
 
     @account_hookimpl
-    def ac_chat_modified(self, chat):
-        print("ac_chat_modified:", chat.id, chat.get_name())
+    def ac_member_added(self, chat, contact):
+        print("*** ac_member_added", contact.addr, "from", chat)
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, message):
-        print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_removed(self, chat, contact, message):
-        print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+    def ac_member_removed(self, chat, contact):
+        print("*** ac_member_removed", contact.addr, "from", chat)
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -33,25 +33,25 @@ def test_echo_quit_plugin(acfactory):
 
 def test_group_tracking_plugin(acfactory, lp):
     lp.sec("creating one group-tracking bot and two temp accounts")
-    botproc = acfactory.run_bot_process(group_tracking, ffi=False)
+    botproc = acfactory.run_bot_process(group_tracking)
 
     ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
 
     botproc.fnmatch_lines("""
-        *ac_configure_completed*
+        *ac_configure_completed: True*
     """)
     ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
     ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
 
-    lp.sec("creating bot test group with bot")
+    lp.sec("creating bot test group with all three")
     bot_contact = ac1.create_contact(botproc.addr)
     ch = ac1.create_group_chat("bot test group")
     ch.add_contact(bot_contact)
     ch.send_text("hello")
 
     botproc.fnmatch_lines("""
-        *ac_chat_modified*bot test group*
-    """)
+        *ac_member_added {}*
+    """.format(ac1.get_config("addr")))
 
     lp.sec("adding third member {}".format(ac2.get_config("addr")))
     contact3 = ac1.create_contact(ac2.get_config("addr"))

python/src/deltachat/account.py

@@ -13,7 +13,7 @@ from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
-from .message import Message, map_system_message
+from .message import Message
 from .contact import Contact
 from .tracker import ImexTracker
 from . import hookspec, iothreads
@@ -30,7 +30,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True):
+    def __init__(self, db_path, os_name=None):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -39,7 +39,6 @@ class Account(object):
         """
         # initialize per-account plugin system
         self._pm = hookspec.PerAccount._make_plugin_manager()
-        self._logging = logging
         self.add_account_plugin(self)
 
         self._dc_context = ffi.gc(
@@ -64,17 +63,10 @@ class Account(object):
         atexit.register(self.shutdown)
         hook.dc_account_init(account=self)
 
-    def disable_logging(self):
-        """ disable logging. """
-        self._logging = False
-
-    def enable_logging(self):
-        """ re-enable logging. """
-        self._logging = True
-
     @hookspec.account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
-        for name, kwargs in self._map_ffi_event(ffi_event):
+        name, kwargs = self._map_ffi_event(ffi_event)
+        if name is not None:
             ev = HookEvent(self, name=name, kwargs=kwargs)
             self._hook_event_queue.put(ev)
 
@@ -82,8 +74,7 @@ class Account(object):
     #    self.shutdown()
 
     def ac_log_line(self, msg):
-        if self._logging:
-            self._pm.hook.ac_log_line(message=msg)
+        self._pm.hook.ac_log_line(message=msg)
 
     def _check_config_key(self, name):
         if name not in self._configkeys:
@@ -254,14 +245,6 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email):
-        """ get a contact for the email address or None if it's blocked or doesn't exist. """
-        _, addr = parseaddr(email)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
-        if contact_id:
-            return self.get_contact_by_id(contact_id)
-
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -283,14 +266,6 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_fresh_messages(self):
-        """ yield all fresh messages from all chats. """
-        dc_array = ffi.gc(
-            lib.dc_get_fresh_msgs(self._dc_context),
-            lib.dc_array_unref
-        )
-        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
-
     def create_chat_by_contact(self, contact):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
@@ -639,26 +614,27 @@ class Account(object):
             data1 = ffi_event.data1
             if data1 == 0 or data1 == 1000:
                 success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
+                return "ac_configure_completed", dict(success=success)
         elif name == "DC_EVENT_INCOMING_MSG":
             msg = self.get_message_by_id(ffi_event.data2)
-            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+            return "ac_incoming_message", dict(message=msg)
         elif name == "DC_EVENT_MSGS_CHANGED":
             if ffi_event.data2 != 0:
                 msg = self.get_message_by_id(ffi_event.data2)
-                if msg.is_outgoing():
-                    res = map_system_message(msg)
-                    if res and res[0].startswith("ac_member"):
-                        yield res
-                    yield "ac_outgoing_message", dict(message=msg)
-                elif msg.is_in_fresh():
-                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+                if msg.is_in_fresh():
+                    return "ac_incoming_message", dict(message=msg)
         elif name == "DC_EVENT_MSG_DELIVERED":
             msg = self.get_message_by_id(ffi_event.data2)
-            yield "ac_message_delivered", dict(message=msg)
-        elif name == "DC_EVENT_CHAT_MODIFIED":
+            return "ac_message_delivered", dict(message=msg)
+        elif name == "DC_EVENT_MEMBER_ADDED":
             chat = self.get_chat_by_id(ffi_event.data1)
-            yield "ac_chat_modified", dict(chat=chat)
+            contact = self.get_contact_by_id(ffi_event.data2)
+            return "ac_member_added", dict(chat=chat, contact=contact)
+        elif name == "DC_EVENT_MEMBER_REMOVED":
+            chat = self.get_chat_by_id(ffi_event.data1)
+            contact = self.get_contact_by_id(ffi_event.data2)
+            return "ac_member_removed", dict(chat=chat, contact=contact)
+        return None, {}
 
 
 def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):

python/src/deltachat/chat.py

@@ -30,7 +30,7 @@ class Chat(object):
         return not (self == other)
 
     def __repr__(self):
-        return "<Chat id={} name={}>".format(self.id, self.get_name())
+        return "<Chat id={} name={} dc_context={}>".format(self.id, self.get_name(), self._dc_context)
 
     @property
     def _dc_chat(self):
@@ -415,6 +415,12 @@ class Chat(object):
         """
         return lib.dc_chat_get_color(self._dc_chat)
 
+    def get_subtitle(self):
+        """return the subtitle of the chat
+        :returns: the subtitle
+        """
+        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
+
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):

python/src/deltachat/const.py

@@ -11,7 +11,6 @@ from os.path import join as joinpath
 DC_GCL_ARCHIVED_ONLY = 0x01
 DC_GCL_NO_SPECIALS = 0x02
 DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
 DC_GCL_VERIFIED_ONLY = 0x01
 DC_GCL_ADD_SELF = 0x02
 DC_QR_ASK_VERIFYCONTACT = 200
@@ -99,6 +98,9 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
+DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
+DC_EVENT_MEMBER_ADDED = 2063
+DC_EVENT_MEMBER_REMOVED = 2064
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
 DC_EVENT_GET_STRING = 2091
@@ -115,6 +117,8 @@ DC_CHAT_VISIBILITY_PINNED = 2
 DC_STR_NOMESSAGES = 1
 DC_STR_SELF = 2
 DC_STR_DRAFT = 3
+DC_STR_MEMBER = 4
+DC_STR_CONTACT = 6
 DC_STR_VOICEMESSAGE = 7
 DC_STR_DEADDROP = 8
 DC_STR_IMAGE = 9
@@ -146,6 +150,7 @@ DC_STR_ARCHIVEDCHATS = 40
 DC_STR_STARREDMSGS = 41
 DC_STR_AC_SETUP_MSG_SUBJECT = 42
 DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_SELFTALK_SUBTITLE = 50
 DC_STR_CANNOT_LOGIN = 60
 DC_STR_SERVER_RESPONSE = 61
 DC_STR_MSGACTIONBYUSER = 62

python/src/deltachat/hookspec.py

@@ -48,25 +48,17 @@ class PerAccount:
     def ac_incoming_message(self, message):
         """ Called on any incoming message (to deaddrop or chat). """
 
-    @account_hookspec
-    def ac_outgoing_message(self, message):
-        """ Called on each outgoing message (both system and "normal")."""
-
     @account_hookspec
     def ac_message_delivered(self, message):
         """ Called when an outgoing message has been delivered to SMTP. """
 
     @account_hookspec
-    def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title. """
+    def ac_member_added(self, chat, contact):
+        """ Called for each contact added to a chat. """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, message):
-        """ Called for each contact added to an accepted chat. """
-
-    @account_hookspec
-    def ac_member_removed(self, chat, contact, message):
-        """ Called for each contact removed from a chat.  """
+    def ac_member_removed(self, chat, contact):
+        """ Called for each contact removed from a chat. """
 
 
 class Global:

python/src/deltachat/message.py

@@ -28,9 +28,7 @@ class Message(object):
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
-        c = self.get_sender_contact()
-        return "<Message id={} sender={}/{} outgoing={} chat={}/{}>".format(
-            self.id, c.id, c.addr, self.is_outgoing(), self.chat.id, self.chat.get_name())
+        return "<Message id={} dc_context={}>".format(self.id, self._dc_context)
 
     @classmethod
     def from_db(cls, account, id):
@@ -93,10 +91,6 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
-    def is_system_message(self):
-        """ return True if this message is a system/info message. """
-        return lib.dc_msg_is_info(self._dc_msg)
-
     def is_setup_message(self):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
@@ -230,13 +224,6 @@ class Message(object):
         """
         return self._msgstate == const.DC_STATE_IN_SEEN
 
-    def is_outgoing(self):
-        """Return True if Message is outgoing. """
-        return self._msgstate in (
-            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
-            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
-            const.DC_STATE_OUT_DELIVERED)
-
     def is_out_preparing(self):
         """Return True if Message is outgoing, but its file is being prepared.
         """
@@ -322,29 +309,3 @@ def get_viewtype_code_from_name(view_type_name):
             return code
     raise ValueError("message typecode not found for {!r}, "
                      "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
-
-
-#
-# some helper code for turning system messages into hook events
-#
-
-def map_system_message(msg):
-    if msg.is_system_message():
-        res = parse_system_add_remove(msg.text)
-        if res:
-            contact = msg.account.get_contact_by_addr(res[1])
-            if contact:
-                d = dict(chat=msg.chat, contact=contact, message=msg)
-                return "ac_member_" + res[0], d
-
-
-def parse_system_add_remove(text):
-    # Member Me (x@y) removed by a@b.
-    # Member x@y removed by a@b
-    text = text.lower()
-    parts = text.split()
-    if parts[0] == "member":
-        if parts[2] in ("removed", "added"):
-            return parts[2], parts[1]
-        if parts[3] in ("removed", "added"):
-            return parts[3], parts[2].strip("()")

python/src/deltachat/testplugin.py

@@ -5,13 +5,9 @@ import subprocess
 import queue
 import threading
 import fnmatch
-import time
-import weakref
-import tempfile
-
 import pytest
 import requests
-
+import time
 from . import Account, const
 from .tracker import ConfigureTracker
 from .capi import lib
@@ -19,7 +15,7 @@ from .eventlogger import FFIEventLogger, FFIEventTracker
 from _pytest.monkeypatch import MonkeyPatch
 from _pytest._code import Source
 
-import deltachat
+import tempfile
 
 
 def pytest_addoption(parser):
@@ -44,55 +40,11 @@ def pytest_configure(config):
         if cfg:
             config.option.liveconfig = cfg
 
-    # Make sure we don't get garbled output because threads keep running
-    # collect all ever created accounts in a weakref-set (so we don't
-    # keep objects unneccessarily alive) and enable/disable logging
-    # for each pytest test phase # (setup/call/teardown).
-    # Additionally make the acfactory use a logging/no-logging default.
 
-    class LoggingAspect:
-        def __init__(self):
-            self._accounts = weakref.WeakSet()
-
-        @deltachat.global_hookimpl
-        def dc_account_init(self, account):
-            self._accounts.add(account)
-
-        def disable_logging(self, item):
-            for acc in self._accounts:
-                acc.disable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(False)
-
-        def enable_logging(self, item):
-            for acc in self._accounts:
-                acc.enable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(True)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_setup(self, item):
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_pyfunc_call(self, pyfuncitem):
-            self.enable_logging(pyfuncitem)
-            yield
-            self.disable_logging(pyfuncitem)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_teardown(self, item):
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-    la = LoggingAspect()
-    config.pluginmanager.register(la)
-    deltachat.register_global_plugin(la)
+def pytest_runtest_setup(item):
+    if (list(item.iter_markers(name="ignored"))
+            and not item.config.getoption("ignored")):
+        pytest.skip("Ignored tests not requested, use --ignored")
 
 
 def pytest_report_header(config, startdir):
@@ -212,35 +164,24 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
-            self._accounts = []
             self.init_time = time.time()
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
-            self.set_logging_default(False)
 
         def finalize(self):
             while self._finalizers:
                 fin = self._finalizers.pop()
                 fin()
 
-            while self._accounts:
-                acc = self._accounts.pop()
-                acc.shutdown()
-                acc.disable_logging()
-
         def make_account(self, path, logid, quiet=False):
-            ac = Account(path, logging=self._logging)
+            ac = Account(path)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
             ac._configtracker = ac.add_account_plugin(ConfigureTracker())
-            ac.addr = ac.get_self_contact().addr
             if not quiet:
                 ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
-            self._accounts.append(ac)
+            self._finalizers.append(ac.shutdown)
             return ac
 
-        def set_logging_default(self, logging):
-            self._logging = bool(logging)
-
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
@@ -321,13 +262,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac2._configtracker.wait_finish()
             return ac1, ac2
 
-        def get_many_online_accounts(self, num, move=True, quiet=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=quiet)
-                        for i in range(num)]
-            for acc in accounts:
-                acc._configtracker.wait_finish()
-            return accounts
-
         def clone_online_account(self, account, pre_generated_key=True):
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
@@ -346,28 +280,26 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac.start()
             return ac
 
-        def run_bot_process(self, module, ffi=True):
+        def run_bot_process(self, module):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
             args = [
                 sys.executable,
-                "-u",
                 fn,
+                "--show-ffi",
                 "--email", bot_cfg["addr"],
                 "--password", bot_cfg["mail_pw"],
                 bot_ac.db_path,
             ]
-            if ffi:
-                args.insert(-1, "--show-ffi")
             print("$", " ".join(args))
             popen = subprocess.Popen(
                 args=args,
                 stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-                bufsize=0,                 # line buffering
+                bufsize=1,                 # line buffering
                 close_fds=True,            # close all FDs other than 0/1/2
                 universal_newlines=True    # give back text
             )
@@ -413,21 +345,15 @@ class BotProcess:
         patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
         for next_pattern in patterns:
             print("+++FNMATCH:", next_pattern)
-            ignored = []
             while 1:
                 line = self.stdout_queue.get(timeout=15)
                 if line is None:
-                    if ignored:
-                        print("BOT stdout terminated after these lines")
-                        for line in ignored:
-                            print(line)
                     raise IOError("BOT stdout-thread terminated")
                 if fnmatch.fnmatch(line, next_pattern):
                     print("+++MATCHED:", line)
                     break
                 else:
                     print("+++IGN:", line)
-                    ignored.append(line)
 
 
 @pytest.fixture

python/tests/stress_test_db.py

@@ -1,105 +0,0 @@
-import time
-import os
-from queue import Queue
-
-import deltachat
-
-
-def test_db_busy_error(acfactory, tmpdir):
-    starttime = time.time()
-
-    def log(string):
-        print("%3.2f %s" % (time.time() - starttime, string))
-
-    # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(5, quiet=False)
-    log("created %s accounts" % len(accounts))
-
-    # put a bigfile into each account
-    for acc in accounts:
-        acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
-        with open(acc.bigfile, "wb") as f:
-            f.write(b"01234567890"*1000_000)
-    log("created %s bigfiles" % len(accounts))
-
-    contact_addrs = [acc.get_self_contact().addr for acc in accounts]
-    chat = accounts[0].create_group_chat("stress-group")
-    for addr in contact_addrs[1:]:
-        chat.add_contact(chat.account.create_contact(addr))
-
-    # setup auto-responder bots which report back failures/actions
-    report_queue = Queue()
-
-    def report_func(replier, report_type, *report_args):
-        report_queue.put((replier, report_type, report_args))
-
-    # each replier receives all events and sends report events to receive_queue
-    repliers = []
-    for acc in accounts:
-        replier = AutoReplier(acc, num_send=1000, num_bigfiles=0, report_func=report_func)
-        acc.add_account_plugin(replier)
-        repliers.append(replier)
-
-    # kick off message sending
-    # after which repliers will reply to each other
-    chat.send_text("hello")
-
-    alive_count = len(accounts)
-    while alive_count > 0:
-        replier, report_type, report_args = report_queue.get(10)
-        addr = replier.account.get_self_contact().addr
-        assert addr
-        if report_type == ReportType.exit:
-            alive_count -= 1
-            log("{} EXIT -- remaining: {}".format(addr, alive_count))
-            replier.account.shutdown(wait=True)
-        elif report_type == ReportType.message_sent:
-            log("{} sent message: {}".format(addr, report_args[0].text))
-        elif report_type == ReportType.message_incoming:
-            log("{} incoming message: {}".format(addr, report_args[0].text))
-        elif report_type == ReportType.ffi_error:
-            log("{} ERROR: {}".format(addr, report_args[0]))
-            replier.account.shutdown(wait=True)
-            alive_count -= 1
-
-
-class ReportType:
-    exit = "exit"
-    message_sent = "message-sent"
-    ffi_error = "ffi-error"
-    message_incoming = "message-incoming"
-
-
-class AutoReplier:
-    def __init__(self, account, report_func, num_send, num_bigfiles):
-        self.account = account
-        self.report_func = report_func
-        self.num_send = num_send
-        self.num_bigfiles = num_bigfiles
-        self.current_sent = 0
-
-    @deltachat.account_hookimpl
-    def ac_incoming_message(self, message):
-        if self.current_sent >= self.num_send:
-            return
-        message.accept_sender_contact()
-        message.mark_seen()
-        self.report_func(self, ReportType.message_incoming, message)
-
-        self.current_sent += 1
-        # we are still alive, let's send a reply
-        if self.num_bigfiles and self.current_sent % self.num_bigfiles == 0:
-            message.chat.send_text("send big file as reply to: {}".format(message.text))
-            msg = message.chat.send_file(self.account.bigfile)
-        else:
-            msg = message.chat.send_text("got message id {}, small text reply".format(message.id))
-            assert msg.text
-        self.report_func(self, ReportType.message_sent, msg)
-        if self.current_sent >= self.num_send:
-            self.report_func(self, ReportType.exit)
-            return
-
-    @deltachat.account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        if ffi_event.name == "DC_EVENT_ERROR":
-            self.report_func(self, ReportType.ffi_error, ffi_event)

python/tests/test_account.py

@@ -11,17 +11,6 @@ from conftest import (wait_configuration_progress,
                       wait_securejoin_inviter_progress)
 
 
-@pytest.mark.parametrize("msgtext,res", [
-    ("Member Me (tmp1@x.org) removed by tmp2@x.org.", ("removed", "tmp1@x.org")),
-    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org")),
-])
-def test_parse_system_add_remove(msgtext, res):
-    from deltachat.message import parse_system_add_remove
-
-    out = parse_system_add_remove(msgtext)
-    assert out == res
-
-
 class TestOfflineAccountBasic:
     def test_wrong_db(self, tmpdir):
         p = tmpdir.join("hello.db")
@@ -181,6 +170,35 @@ class TestOfflineChat:
         else:
             pytest.fail("could not find chat")
 
+    def test_add_member_event(self, ac1):
+        chat = ac1.create_group_chat(name="title1")
+        assert chat.is_group()
+        contact1 = ac1.create_contact("some1@hello.com", name="some1")
+
+        chat.add_contact(contact1)
+        for ev in ac1.iter_events(timeout=1):
+            if ev.name == "ac_member_added":
+                assert ev.kwargs["chat"] == chat
+                if ev.kwargs["contact"] == ac1.get_self_contact():
+                    continue
+                assert ev.kwargs["contact"] == contact1
+                break
+
+    def test_remove_member_event(self, ac1):
+        chat = ac1.create_group_chat(name="title1")
+        assert chat.is_group()
+        contact1 = ac1.create_contact("some1@hello.com", name="some1")
+        chat.add_contact(contact1)
+        ac1._handle_current_events()
+        chat.remove_contact(contact1)
+        for ev in ac1.iter_events(timeout=1):
+            if ev.name == "ac_member_removed":
+                assert ev.kwargs["chat"] == chat
+                if ev.kwargs["contact"] == ac1.get_self_contact():
+                    continue
+                assert ev.kwargs["contact"] == contact1
+                break
+
     def test_group_chat_creation(self, ac1):
         contact1 = ac1.create_contact("some1@hello.com", name="some1")
         contact2 = ac1.create_contact("some2@hello.com", name="some2")
@@ -203,6 +221,7 @@ class TestOfflineChat:
         # assert d["param"] == chat.param
         assert d["color"] == chat.get_color()
         assert d["profile_image"] == "" if chat.get_profile_image() is None else chat.get_profile_image()
+        assert d["subtitle"] == chat.get_subtitle()
         assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft()
 
     def test_group_chat_creation_with_translation(self, ac1):
@@ -478,7 +497,7 @@ class TestOfflineChat:
         # perform plugin hooks
         ac1._handle_current_events()
 
-        assert len(in_list) == 10
+        assert len(in_list) == 11
         chat_contacts = chat.get_contacts()
         for in_cmd, in_chat, in_contact in in_list:
             assert in_cmd == "added"
@@ -486,23 +505,19 @@ class TestOfflineChat:
             assert in_contact in chat_contacts
             chat_contacts.remove(in_contact)
 
-        assert chat_contacts[0].id == 1  # self contact
-
-        in_list[:] = []
-
         lp.sec("ac1: removing two contacts and checking things are right")
         chat.remove_contact(contacts[9])
         chat.remove_contact(contacts[3])
         assert len(chat.get_contacts()) == 9
 
         ac1._handle_current_events()
-        assert len(in_list) == 2
-        assert in_list[0][0] == "removed"
-        assert in_list[0][1] == chat
-        assert in_list[0][2] == contacts[9]
-        assert in_list[1][0] == "removed"
-        assert in_list[1][1] == chat
-        assert in_list[1][2] == contacts[3]
+        assert len(in_list) == 13
+        assert in_list[-2][0] == "removed"
+        assert in_list[-2][1] == chat
+        assert in_list[-2][2] == contacts[9]
+        assert in_list[-1][0] == "removed"
+        assert in_list[-1][1] == chat
+        assert in_list[-1][2] == contacts[3]
 
 
 class TestOnlineAccount:
@@ -928,13 +943,6 @@ class TestOnlineAccount:
         assert msg_back.text == "message-back"
         assert msg_back.is_encrypted()
 
-        # test get_fresh_messages
-        fresh_msgs = list(ac1.get_fresh_messages())
-        assert len(fresh_msgs) == 1
-        assert fresh_msgs[0] == msg_back
-        msg_back.mark_seen()
-        assert not list(ac1.get_fresh_messages())
-
         # Test that we do not gossip peer keys in 1-to-1 chat,
         # as it makes no sense to gossip to peers their own keys.
         # Gossip is only sent in encrypted messages,
@@ -1059,17 +1067,12 @@ class TestOnlineAccount:
                 message_queue.put(message)
 
         delivered = queue.Queue()
-        out = queue.Queue()
 
         class OutPlugin:
             @account_hookimpl
             def ac_message_delivered(self, message):
                 delivered.put(message)
 
-            @account_hookimpl
-            def ac_outgoing_message(self, message):
-                out.put(message)
-
         ac1.add_account_plugin(OutPlugin())
         ac2.add_account_plugin(InPlugin())
 
@@ -1080,8 +1083,6 @@ class TestOnlineAccount:
         assert ev.data1 == chat.id
         assert ev.data2 == msg_out.id
         assert msg_out.is_out_delivered()
-        m = out.get()
-        assert m == msg_out
         m = delivered.get()
         assert m == msg_out
 
@@ -1209,6 +1210,11 @@ class TestOnlineAccount:
         ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
         ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
         wait_securejoin_inviter_progress(ac1, 1000)
+        ac1._evtracker.get_matching("DC_EVENT_MEMBER_ADDED")
+
+        ch.remove_contact(ac1.get_self_contact())
+        ac2._evtracker.get_matching("DC_EVENT_MEMBER_REMOVED")
+        ac1._evtracker.get_matching("DC_EVENT_MEMBER_REMOVED")
 
     def test_qr_verified_group_and_chatting(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1220,6 +1226,7 @@ class TestOnlineAccount:
         chat2 = ac2.qr_join_chat(qr)
         assert chat2.id >= 10
         wait_securejoin_inviter_progress(ac1, 1000)
+        ac1._evtracker.get_matching("DC_EVENT_MEMBER_ADDED")
 
         lp.sec("ac2: read member added message")
         msg = ac2._evtracker.wait_next_incoming_message()
@@ -1291,77 +1298,48 @@ class TestOnlineAccount:
 
     def test_add_remove_member_remote_events(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-        ac1_addr = ac1.get_config("addr")
-        ac2_addr = ac2.get_config("addr")
         # activate local plugin for ac2
         in_list = queue.Queue()
 
-        class EventHolder:
-            def __init__(self, **kwargs):
-                self.__dict__.update(kwargs)
-
         class InPlugin:
             @account_hookimpl
-            def ac_incoming_message(self, message):
-                # we immediately accept the sender because
-                # otherwise we won't see member_added contacts
-                message.accept_sender_contact()
+            def ac_member_added(self, chat, contact):
+                in_list.put(("added", chat, contact))
 
             @account_hookimpl
-            def ac_chat_modified(self, chat):
-                in_list.put(EventHolder(action="chat-modified", chat=chat))
-
-            @account_hookimpl
-            def ac_member_added(self, chat, contact, message):
-                in_list.put(EventHolder(action="added", chat=chat, contact=contact, message=message))
-
-            @account_hookimpl
-            def ac_member_removed(self, chat, contact, message):
-                in_list.put(EventHolder(action="removed", chat=chat, contact=contact, message=message))
+            def ac_member_removed(self, chat, contact):
+                in_list.put(("removed", chat, contact))
 
         ac2.add_account_plugin(InPlugin())
 
         lp.sec("ac1: create group chat with ac2")
         chat = ac1.create_group_chat("hello")
-        contact = ac1.create_contact(email=ac2_addr)
+        contact = ac1.create_contact(email=ac2.get_config("addr"))
         chat.add_contact(contact)
 
         lp.sec("ac1: send a message to group chat to promote the group")
         chat.send_text("afterwards promoted")
-        ev = in_list.get(timeout=10)
-        assert ev.action == "chat-modified"
-        assert chat.is_promoted()
-        assert sorted(x.addr for x in chat.get_contacts()) == \
-            sorted(x.addr for x in ev.chat.get_contacts())
+        ev1 = in_list.get()
+        ev2 = in_list.get()
+        assert ev1[2] == ac2.get_self_contact()
+        assert ev2[2].addr == ac1.get_config("addr")
 
         lp.sec("ac1: add address2")
-        # note that if the above accept_sender_contact() would not
-        # happen we would not receive a proper member_added event
-        contact2 = ac1.create_contact(email="notexistingaccountihope@testrun.org")
+        contact2 = ac1.create_contact(email="not@example.org")
         chat.add_contact(contact2)
-        ev = in_list.get(timeout=10)
-        assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
-        assert ev.action == "added"
-        assert ev.message.get_sender_contact().addr == ac1_addr
-        assert ev.contact.addr == "notexistingaccountihope@testrun.org"
+        ev1 = in_list.get()
+        assert ev1[2].addr == contact2.addr
 
         lp.sec("ac1: remove address2")
         chat.remove_contact(contact2)
-        ev = in_list.get(timeout=10)
-        assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
-        assert ev.action == "removed"
-        assert ev.contact.addr == contact2.addr
-        assert ev.message.get_sender_contact().addr == ac1_addr
+        ev1 = in_list.get()
+        assert ev1[0] == "removed"
+        assert ev1[2].addr == contact2.addr
 
         lp.sec("ac1: remove ac2 contact from chat")
         chat.remove_contact(contact)
-        ev = in_list.get(timeout=10)
-        assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
-        assert ev.action == "removed"
-        assert ev.message.get_sender_contact().addr == ac1_addr
+        ev1 = in_list.get()
+        assert ev1[2] == ac2.get_self_contact()
 
     def test_set_get_group_image(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1541,6 +1519,7 @@ class TestGroupStressTests:
         to_remove = contacts[-1]
 
         msg.chat.remove_contact(to_remove)
+        ac2._evtracker.get_matching("DC_EVENT_MEMBER_REMOVED")
 
         lp.sec("ac1: receiving system message about contact removal")
         sysmsg = ac1._evtracker.wait_next_incoming_message()

src/chat.rs

@@ -385,7 +385,7 @@ impl ChatId {
         let sql = &context.sql;
         let query = format!(
             "SELECT {} \
-             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) AND NOT hidden \
+             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) \
              ORDER BY timestamp DESC, id DESC \
              LIMIT 1;",
             fields
@@ -574,9 +574,13 @@ impl Chat {
         self.param.exists(Param::Devicetalk)
     }
 
+    pub fn is_mailing_list(&self) -> bool {
+        self.param.exists(Param::MailingList)
+    }
+
     /// Returns true if user can send messages to this chat.
     pub fn can_send(&self) -> bool {
-        !self.id.is_special() && !self.is_device_talk()
+        !self.id.is_special() && !self.is_device_talk() && !self.is_mailing_list()
     }
 
     pub fn update_param(&mut self, context: &Context) -> Result<(), Error> {
@@ -604,6 +608,42 @@ impl Chat {
         &self.name
     }
 
+    pub fn get_subtitle(&self, context: &Context) -> String {
+        // returns either the address or the number of chat members
+
+        if self.typ == Chattype::Single && self.param.exists(Param::Selftalk) {
+            return context.stock_str(StockMessage::SelfTalkSubTitle).into();
+        }
+
+        if self.typ == Chattype::Single {
+            return context
+                .sql
+                .query_get_value(
+                    context,
+                    "SELECT c.addr
+                       FROM chats_contacts cc
+                       LEFT JOIN contacts c ON c.id=cc.contact_id
+                      WHERE cc.chat_id=?;",
+                    params![self.id],
+                )
+                .unwrap_or_else(|| "Err".into());
+        }
+
+        if self.typ == Chattype::Group && self.is_mailing_list() {
+            return "".to_string();
+        }
+
+        if self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup {
+            if self.id.is_deaddrop() {
+                return context.stock_str(StockMessage::DeadDrop).into();
+            }
+            let cnt = get_chat_contact_cnt(context, self.id);
+            return context.stock_string_repl_int(StockMessage::Member, cnt as i32);
+        }
+
+        "Err".to_string()
+    }
+
     pub fn get_profile_image(&self, context: &Context) -> Option<PathBuf> {
         if let Some(image_rel) = self.param.get(Param::ProfileImage) {
             if !image_rel.is_empty() {
@@ -661,6 +701,7 @@ impl Chat {
             is_sending_locations: self.is_sending_locations,
             color: self.get_color(context),
             profile_image: self.get_profile_image(context).unwrap_or_else(PathBuf::new),
+            subtitle: self.get_subtitle(context),
             draft,
             is_muted: self.is_muted(),
         })
@@ -1003,6 +1044,9 @@ pub struct ChatInfo {
     /// currently.
     pub profile_image: PathBuf,
 
+    /// Subtitle for the chat.
+    pub subtitle: String,
+
     /// The draft message text.
     ///
     /// If the chat has not draft this is an empty string.
@@ -1061,7 +1105,11 @@ pub fn create_by_msg_id(context: &Context, msg_id: MsgId) -> Result<ChatId, Erro
             msg_id: MsgId::new(0),
         });
     }
-    Contact::scaleup_origin_by_id(context, msg.from_id, Origin::CreateChat);
+
+    // If the message is from a mailing list, the contacts are not counted as "known"
+    if !chat.is_mailing_list() {
+        Contact::scaleup_origin_by_id(context, msg.from_id, Origin::CreateChat);
+    }
     Ok(chat.id)
 }
 
@@ -1441,7 +1489,7 @@ pub fn get_chat_msgs(
     flags: u32,
     marker1before: Option<MsgId>,
 ) -> Vec<MsgId> {
-    match delete_device_expired_messages(context) {
+    match hide_device_expired_messages(context) {
         Err(err) => warn!(context, "Failed to delete expired messages: {}", err),
         Ok(messages_deleted) => {
             if messages_deleted {
@@ -1587,11 +1635,11 @@ pub fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-/// Deletes messages which are expired according to "delete_device_after" setting.
+/// Hides messages which are expired according to "delete_device_after" setting.
 ///
-/// Returns true if any message is deleted, so event can be emitted. If nothing
-/// has been deleted, returns false.
-pub fn delete_device_expired_messages(context: &Context) -> Result<bool, Error> {
+/// Returns true if any message is hidden, so event can be emitted. If nothing
+/// has been hidden, returns false.
+pub fn hide_device_expired_messages(context: &Context) -> Result<bool, Error> {
     if let Some(delete_device_after) = context.get_config_delete_device_after() {
         let threshold_timestamp = time() - delete_device_after;
 
@@ -1602,19 +1650,19 @@ pub fn delete_device_expired_messages(context: &Context) -> Result<bool, Error>
             .unwrap_or_default()
             .0;
 
-        // Delete expired messages
+        // Hide expired messages
         //
         // Only update the rows that have to be updated, to avoid emitting
         // unnecessary "chat modified" events.
         let rows_modified = context.sql.execute(
             "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
+             SET txt = 'DELETED', hidden = 1 \
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
-             AND chat_id != ?",
+             AND chat_id != ? \
+             AND NOT hidden",
             params![
-                DC_CHAT_ID_TRASH,
                 threshold_timestamp,
                 DC_CHAT_ID_LAST_SPECIAL,
                 self_chat_id,
@@ -1795,6 +1843,7 @@ pub fn create_group_chat(
 }
 
 /// add a contact to the chats_contact table
+/// on success emit MemberAdded event and return true
 pub(crate) fn add_to_chat_contacts_table(
     context: &Context,
     chat_id: ChatId,
@@ -1806,7 +1855,14 @@ pub(crate) fn add_to_chat_contacts_table(
         "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)",
         params![chat_id, contact_id as i32],
     ) {
-        Ok(()) => true,
+        Ok(()) => {
+            context.call_cb(Event::MemberAdded {
+                chat_id,
+                contact_id,
+            });
+
+            true
+        }
         Err(err) => {
             error!(
                 context,
@@ -1819,6 +1875,7 @@ pub(crate) fn add_to_chat_contacts_table(
 }
 
 /// remove a contact from the chats_contact table
+/// on success emit MemberRemoved event and return true
 pub(crate) fn remove_from_chat_contacts_table(
     context: &Context,
     chat_id: ChatId,
@@ -1830,7 +1887,14 @@ pub(crate) fn remove_from_chat_contacts_table(
         "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?",
         params![chat_id, contact_id as i32],
     ) {
-        Ok(()) => true,
+        Ok(()) => {
+            context.call_cb(Event::MemberRemoved {
+                chat_id,
+                contact_id,
+            });
+
+            true
+        }
         Err(_) => {
             warn!(
                 context,
@@ -2071,17 +2135,18 @@ impl rusqlite::types::FromSql for MuteDuration {
 
 pub fn set_muted(context: &Context, chat_id: ChatId, duration: MuteDuration) -> Result<(), Error> {
     ensure!(!chat_id.is_special(), "Invalid chat ID");
-    if sql::execute(
-        context,
-        &context.sql,
-        "UPDATE chats SET muted_until=? WHERE id=?;",
-        params![duration, chat_id],
-    )
-    .is_ok()
+    if real_group_exists(context, chat_id)
+        && sql::execute(
+            context,
+            &context.sql,
+            "UPDATE chats SET muted_until=? WHERE id=?;",
+            params![duration, chat_id],
+        )
+        .is_ok()
     {
         context.call_cb(Event::ChatModified(chat_id));
     } else {
-        bail!("Failed to set mute duration, chat might not exist -");
+        bail!("Failed to set name");
     }
     Ok(())
 }
@@ -2138,6 +2203,10 @@ pub fn remove_contact_from_chat(
                         msg.param.set_cmd(SystemMessage::MemberRemovedFromGroup);
                         msg.param.set(Param::Arg, contact.get_addr());
                         msg.id = send_msg(context, chat_id, &mut msg)?;
+                        context.call_cb(Event::MsgsChanged {
+                            chat_id,
+                            msg_id: msg.id,
+                        });
                     }
                 }
                 // we remove the member from the chat after constructing the
@@ -2445,6 +2514,22 @@ pub(crate) fn get_chat_id_by_grpid(
     )
 }
 
+pub(crate) fn get_chat_id_by_mailinglistid(
+    context: &Context,
+    listid: impl AsRef<str>,
+) -> Result<(ChatId, Blocked), sql::Error> {
+    context.sql.query_row(
+        "SELECT id, blocked, type FROM chats WHERE grpid=?;",
+        params![listid.as_ref()],
+        |row| {
+            let chat_id = row.get::<_, ChatId>(0)?;
+
+            let b = row.get::<_, Option<Blocked>>(1)?.unwrap_or_default();
+            Ok((chat_id, b))
+        },
+    )
+}
+
 /// Adds a message to device chat.
 ///
 /// Optional `label` can be provided to ensure that message is added only once.
@@ -2596,6 +2681,7 @@ mod tests {
                 "is_sending_locations": false,
                 "color": 15895624,
                 "profile_image": "",
+                "subtitle": "bob@example.com",
                 "draft": "",
                 "is_muted": false
             }

src/chatlist.rs

@@ -92,14 +92,9 @@ impl Chatlist {
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
-        let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
-        let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
-        let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
-        let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
-
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
-        // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context) {
+        // messages get hidden to avoid reloading the same chatlist.
+        if let Err(err) = hide_device_expired_messages(context) {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
@@ -116,7 +111,7 @@ impl Chatlist {
                 .map_err(Into::into)
         };
 
-        let skip_id = if flag_for_forwarding {
+        let skip_id = if 0 != listflags & DC_GCL_FOR_FORWARDING {
             chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
                 .unwrap_or_default()
                 .0
@@ -160,7 +155,7 @@ impl Chatlist {
                 process_row,
                 process_rows,
             )?
-        } else if flag_archived_only {
+        } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
             // (this includes the archived device-chat; we could skip it,
             // however, then the number of archived chats do not match, which might be even more irritating.
@@ -216,7 +211,7 @@ impl Chatlist {
             )?
         } else {
             //  show normal chatlist
-            let sort_id_up = if flag_for_forwarding {
+            let sort_id_up = if 0 != listflags & DC_GCL_FOR_FORWARDING {
                 chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
                     .unwrap_or_default()
                     .0
@@ -242,9 +237,9 @@ impl Chatlist {
                 process_row,
                 process_rows,
             )?;
-            if !flag_no_specials {
+            if 0 == listflags & DC_GCL_NO_SPECIALS {
                 if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
-                    if !flag_for_forwarding {
+                    if 0 == listflags & DC_GCL_FOR_FORWARDING {
                         ids.insert(
                             0,
                             (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
@@ -257,7 +252,7 @@ impl Chatlist {
         };
 
         if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
-            if ids.is_empty() && flag_add_alldone_hint {
+            if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
                 ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
             ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));

src/config.rs

@@ -65,9 +65,6 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
-    #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
-    MediaQuality,
-
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -251,11 +248,9 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
-    use crate::constants;
     use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
     use image::GenericImageView;
-    use num_traits::FromPrimitive;
     use std::fs::File;
     use std::io::Write;
 
@@ -328,44 +323,4 @@ mod tests {
         assert_eq!(img.width(), AVATAR_SIZE);
         assert_eq!(img.height(), AVATAR_SIZE);
     }
-
-    #[test]
-    fn test_selfavatar_copy_without_recode() {
-        let t = dummy_context();
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists());
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .unwrap();
-        assert!(avatar_blob.exists());
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-    }
-
-    #[test]
-    fn test_media_quality_config_option() {
-        let t = dummy_context();
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality);
-        assert_eq!(media_quality, 0);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Balanced);
-
-        t.ctx.set_config(Config::MediaQuality, Some("1")).unwrap();
-
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality);
-        assert_eq!(media_quality, 1);
-        assert_eq!(constants::MediaQuality::Worse as i32, 1);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Worse);
-    }
 }

src/configure/mod.rs

@@ -69,7 +69,6 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
     let mut smtp_connected_here = false;
 
     let mut param_autoconfig: Option<LoginParam> = None;
-    let was_configured_before = context.is_configured();
 
     context
         .inbox_thread
@@ -442,15 +441,6 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
     }
 
     if let Some(provider) = provider::get_provider_info(&param.addr) {
-        if !was_configured_before {
-            if let Some(config_defaults) = &provider.config_defaults {
-                for def in config_defaults.iter() {
-                    info!(context, "apply config_defaults {}={}", def.key, def.value);
-                    context.set_config(def.key, Some(def.value));
-                }
-            }
-        }
-
         if !provider.after_login_hint.is_empty() {
             let mut msg = Message::new(Viewtype::Text);
             msg.text = Some(provider.after_login_hint.to_string());

src/constants.rs

@@ -57,19 +57,6 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
-#[repr(u8)]
-pub enum MediaQuality {
-    Balanced = 0,
-    Worse = 1,
-}
-
-impl Default for MediaQuality {
-    fn default() -> Self {
-        MediaQuality::Balanced // also change Config.MediaQuality props(default) on changes
-    }
-}
-
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(u8)]
 pub enum KeyGenType {
@@ -327,6 +314,8 @@ const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 const DC_STR_NOMESSAGES: usize = 1;
 const DC_STR_SELF: usize = 2;
 const DC_STR_DRAFT: usize = 3;
+const DC_STR_MEMBER: usize = 4;
+const DC_STR_CONTACT: usize = 6;
 const DC_STR_VOICEMESSAGE: usize = 7;
 const DC_STR_DEADDROP: usize = 8;
 const DC_STR_IMAGE: usize = 9;
@@ -358,6 +347,7 @@ const DC_STR_ARCHIVEDCHATS: usize = 40;
 const DC_STR_STARREDMSGS: usize = 41;
 const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
 const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
+const DC_STR_SELFTALK_SUBTITLE: usize = 50;
 const DC_STR_CANNOT_LOGIN: usize = 60;
 const DC_STR_SERVER_RESPONSE: usize = 61;
 const DC_STR_MSGACTIONBYUSER: usize = 62;

src/contact.rs

@@ -11,9 +11,10 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::e2ee;
 use crate::error::{bail, ensure, format_err, Result};
 use crate::events::Event;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::*;
 use crate::login_param::LoginParam;
 use crate::message::{MessageState, MsgId};
 use crate::mimeparser::AvatarAction;
@@ -342,20 +343,6 @@ impl Contact {
             return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
-        if !may_be_valid_addr(&addr) {
-            warn!(
-                context,
-                "Bad address \"{}\" for contact \"{}\".",
-                addr,
-                if !name.as_ref().is_empty() {
-                    name.as_ref()
-                } else {
-                    "<unset>"
-                },
-            );
-            bail!("Bad address supplied: {:?}", addr);
-        }
-
         let mut update_addr = false;
         let mut update_name = false;
         let mut update_authname = false;
@@ -646,6 +633,8 @@ impl Contact {
             let peerstate = Peerstate::from_addr(context, &context.sql, &contact.addr);
             let loginparam = LoginParam::from_database(context, "configured_");
 
+            let mut self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
+
             if peerstate.is_some()
                 && peerstate
                     .as_ref()
@@ -660,11 +649,16 @@ impl Contact {
                         StockMessage::E2eAvailable
                     });
                 ret += &p;
-                let self_key = Key::from(SignedPublicKey::load_self(context)?);
+                if self_key.is_none() {
+                    e2ee::ensure_secret_key_exists(context)?;
+                    self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
+                }
                 let p = context.stock_str(StockMessage::FingerPrints);
                 ret += &format!(" {}:", p);
 
-                let fingerprint_self = self_key.formatted_fingerprint();
+                let fingerprint_self = self_key
+                    .map(|k| k.formatted_fingerprint())
+                    .unwrap_or_default();
                 let fingerprint_other_verified = peerstate
                     .peek_key(PeerstateVerifiedStatus::BidirectVerified)
                     .map(|k| k.formatted_fingerprint())

src/context.rs

@@ -1,6 +1,6 @@
 //! Context module
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::ffi::OsString;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Condvar, Mutex, RwLock};
@@ -9,20 +9,18 @@ use crate::chat::*;
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
-use crate::dc_tools::duration_to_str;
 use crate::error::*;
 use crate::events::Event;
 use crate::imap::*;
 use crate::job::*;
 use crate::job_thread::JobThread;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::Key;
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
 use crate::message::{self, Message, MessengerMessage, MsgId};
 use crate::param::Params;
 use crate::smtp::Smtp;
 use crate::sql::Sql;
-use std::time::SystemTime;
 
 /// Callback function type for [Context]
 ///
@@ -59,7 +57,6 @@ pub struct Context {
     /// Mutex to avoid generating the key for the user more than once.
     pub generating_key_mutex: Mutex<()>,
     pub translated_stockstrings: RwLock<HashMap<usize, String>>,
-    creation_time: SystemTime,
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -74,8 +71,8 @@ pub struct RunningState {
 /// actual keys and their values which will be present are not
 /// guaranteed.  Calling [Context::get_info] also includes information
 /// about the context on top of the information here.
-pub fn get_info() -> BTreeMap<&'static str, String> {
-    let mut res = BTreeMap::new();
+pub fn get_info() -> HashMap<&'static str, String> {
+    let mut res = HashMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
     res.insert("sqlite_version", rusqlite::version().to_string());
     res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
@@ -141,7 +138,6 @@ impl Context {
             perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
-            creation_time: std::time::SystemTime::now(),
         };
 
         ensure!(
@@ -226,7 +222,7 @@ impl Context {
      * UI chat/message related API
      ******************************************************************************/
 
-    pub fn get_info(&self) -> BTreeMap<&'static str, String> {
+    pub fn get_info(&self) -> HashMap<&'static str, String> {
         let unset = "0";
         let l = LoginParam::from_database(self, "");
         let l2 = LoginParam::from_database(self, "configured_");
@@ -240,10 +236,7 @@ impl Context {
             .sql
             .get_raw_config_int(self, "dbversion")
             .unwrap_or_default();
-        let journal_mode = self
-            .sql
-            .query_get_value(self, "PRAGMA journal_mode;", rusqlite::NO_PARAMS)
-            .unwrap_or_else(|| "unknown".to_string());
+
         let e2ee_enabled = self.get_config_int(Config::E2eeEnabled);
         let mdns_enabled = self.get_config_int(Config::MdnsEnabled);
         let bcc_self = self.get_config_int(Config::BccSelf);
@@ -258,9 +251,10 @@ impl Context {
             rusqlite::NO_PARAMS,
         );
 
-        let fingerprint_str = match SignedPublicKey::load_self(self) {
-            Ok(key) => Key::from(key).fingerprint(),
-            Err(err) => format!("<key failure: {}>", err),
+        let fingerprint_str = if let Some(key) = Key::from_self_public(self, &l2.addr, &self.sql) {
+            key.fingerprint()
+        } else {
+            "<Not yet calculated>".into()
         };
 
         let inbox_watch = self.get_config_int(Config::InboxWatch);
@@ -288,7 +282,6 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
-        res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
         res.insert(
@@ -319,9 +312,6 @@ impl Context {
         );
         res.insert("fingerprint", fingerprint_str);
 
-        let elapsed = self.creation_time.elapsed();
-        res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
-
         res
     }
 

src/dc_receive_imf.rs

@@ -2,8 +2,8 @@ use itertools::join;
 use sha2::{Digest, Sha256};
 
 use num_traits::FromPrimitive;
-
-use mailparse::SingleInfo;
+use regex::Regex;
+use std::borrow::Cow;
 
 use crate::chat::{self, Chat, ChatId};
 use crate::config::Config;
@@ -34,10 +34,6 @@ enum CreateEvent {
 }
 
 /// Receive a message and add it to the database.
-///
-/// Returns an error on recoverable errors, e.g. database errors. In this case,
-/// message parsing should be retried later. If message itself is wrong, logs
-/// the error and returns success.
 pub fn dc_receive_imf(
     context: &Context,
     imf_raw: &[u8],
@@ -61,19 +57,10 @@ pub fn dc_receive_imf(
         println!("{}", String::from_utf8_lossy(imf_raw));
     }
 
-    let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw) {
-        Err(err) => {
-            warn!(context, "dc_receive_imf: can't parse MIME: {}", err);
-            return Ok(());
-        }
-        Ok(mime_parser) => mime_parser,
-    };
+    let mut mime_parser = MimeMessage::from_bytes(context, imf_raw)?;
 
     // we can not add even an empty record if we have no info whatsoever
-    if !mime_parser.has_headers() {
-        warn!(context, "dc_receive_imf: no headers found");
-        return Ok(());
-    }
+    ensure!(mime_parser.has_headers(), "No Headers Found");
 
     // the function returns the number of created messages in the database
     let mut chat_id = ChatId::new(0);
@@ -86,6 +73,8 @@ pub fn dc_receive_imf(
     let mut created_db_entries = Vec::new();
     let mut create_event_to_send = Some(CreateEvent::MsgsChanged);
 
+    let list_id_header: Option<&String> = mime_parser.get(HeaderDef::ListId);
+
     // helper method to handle early exit and memory cleanup
     let cleanup = |context: &Context,
                    create_event_to_send: &Option<CreateEvent>,
@@ -112,23 +101,30 @@ pub fn dc_receive_imf(
     // we do not check Return-Path any more as this is unreliable, see
     // https://github.com/deltachat/deltachat-core/issues/150)
     let (from_id, from_id_blocked, incoming_origin) =
-        from_field_to_contact_id(context, &mime_parser.from)?;
-
+        if let Some(field_from) = mime_parser.get(HeaderDef::From_) {
+            from_field_to_contact_id(context, field_from, list_id_header)?
+        } else {
+            (0, false, Origin::Unknown)
+        };
     let incoming = from_id != DC_CONTACT_ID_SELF;
 
     let mut to_ids = ContactIds::new();
-
-    to_ids.extend(&dc_add_or_lookup_contacts_by_address_list(
-        context,
-        &mime_parser.recipients,
-        if !incoming {
-            Origin::OutgoingTo
-        } else if incoming_origin.is_known() {
-            Origin::IncomingTo
-        } else {
-            Origin::IncomingUnknownTo
-        },
-    )?);
+    for header_def in &[HeaderDef::To, HeaderDef::Cc] {
+        if let Some(field) = mime_parser.get(header_def.clone()) {
+            to_ids.extend(&dc_add_or_lookup_contacts_by_address_list(
+                context,
+                &field,
+                if !incoming {
+                    Origin::OutgoingTo
+                } else if incoming_origin.is_known() {
+                    Origin::IncomingTo
+                } else {
+                    Origin::IncomingUnknownTo
+                },
+                list_id_header,
+            )?);
+        }
+    }
 
     // Add parts
 
@@ -238,12 +234,14 @@ pub fn dc_receive_imf(
 /// Also returns whether it is blocked or not and its origin.
 pub fn from_field_to_contact_id(
     context: &Context,
-    from_address_list: &[SingleInfo],
+    field_from: &str,
+    list_id_header: Option<&String>,
 ) -> Result<(u32, bool, Origin)> {
     let from_ids = dc_add_or_lookup_contacts_by_address_list(
         context,
-        from_address_list,
+        &field_from,
         Origin::IncomingUnknownFrom,
+        list_id_header,
     )?;
 
     if from_ids.contains(&DC_CONTACT_ID_SELF) {
@@ -252,7 +250,7 @@ pub fn from_field_to_contact_id(
         if from_ids.len() > 1 {
             warn!(
                 context,
-                "mail has more than one From address, only using first: {:?}", from_address_list
+                "mail has more than one From address, only using first: {:?}", field_from
             );
         }
         let from_id = from_ids.get_index(0).cloned().unwrap_or_default();
@@ -265,10 +263,7 @@ pub fn from_field_to_contact_id(
         }
         Ok((from_id, from_id_blocked, incoming_origin))
     } else {
-        warn!(
-            context,
-            "mail has an empty From header: {:?}", from_address_list
-        );
+        warn!(context, "mail has an empty From header: {:?}", field_from);
         // if there is no from given, from_id stays 0 which is just fine. These messages
         // are very rare, however, we have to add them to the database (they go to the
         // "deaddrop" chat) to avoid a re-download from the server. See also [**]
@@ -310,15 +305,14 @@ fn add_parts(
     // check, if the mail is already in our database - if so, just update the folder/uid
     // (if the mail was moved around) and finish. (we may get a mail twice eg. if it is
     // moved between folders. make sure, this check is done eg. before securejoin-processing) */
-    if let Some((old_server_folder, old_server_uid, _)) =
-        message::rfc724_mid_exists(context, &rfc724_mid)?
+    if let Ok((old_server_folder, old_server_uid, _)) =
+        message::rfc724_mid_exists(context, &rfc724_mid)
     {
         if old_server_folder != server_folder.as_ref() || old_server_uid != server_uid {
             message::update_server_uid(context, &rfc724_mid, server_folder.as_ref(), server_uid);
         }
 
-        warn!(context, "Message already in DB");
-        return Ok(());
+        bail!("Message already in DB");
     }
 
     let mut msgrmsg = if mime_parser.has_chat_version() {
@@ -382,8 +376,7 @@ fn add_parts(
                     *hidden = true;
                     context.bob.write().unwrap().status = 0; // secure-join failed
                     context.stop_ongoing();
-                    warn!(context, "Error in Secure-Join message handling: {}", err);
-                    return Ok(());
+                    error!(context, "Error in Secure-Join message handling: {}", err);
                 }
             }
         }
@@ -429,9 +422,34 @@ fn add_parts(
 
         if chat_id.is_unset() {
             // check if the message belongs to a mailing list
-            if mime_parser.is_mailinglist_message() {
-                *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
-                info!(context, "Message belongs to a mailing list and is ignored.",);
+            if let Some(list_id_header) = mime_parser.get(HeaderDef::ListId) {
+                let create_blocked = if !test_normal_chat_id.is_unset()
+                    && test_normal_chat_id_blocked == Blocked::Not
+                {
+                    Blocked::Not
+                } else {
+                    Blocked::Deaddrop
+                };
+
+                let (new_chat_id, new_chat_id_blocked) = create_or_lookup_mailinglist(
+                    context,
+                    if test_normal_chat_id.is_unset() {
+                        allow_creation
+                    } else {
+                        true
+                    },
+                    create_blocked,
+                    list_id_header,
+                );
+                *chat_id = new_chat_id;
+                chat_id_blocked = new_chat_id_blocked;
+                if !chat_id.is_unset()
+                    && chat_id_blocked != Blocked::Not
+                    && create_blocked == Blocked::Not
+                {
+                    new_chat_id.unblock(context);
+                    chat_id_blocked = Blocked::Not;
+                }
             }
         }
 
@@ -510,8 +528,7 @@ fn add_parts(
                 }
                 Err(err) => {
                     *hidden = true;
-                    warn!(context, "Error in Secure-Join watching: {}", err);
-                    return Ok(());
+                    error!(context, "Error in Secure-Join watching: {}", err);
                 }
             }
         }
@@ -689,7 +706,7 @@ fn add_parts(
     );
 
     // check event to send
-    if chat_id.is_trash() || *hidden {
+    if chat_id.is_trash() {
         *create_event_to_send = None;
     } else if incoming && state == MessageState::InFresh {
         if from_id_blocked {
@@ -1111,6 +1128,66 @@ fn create_or_lookup_group(
     Ok((chat_id, chat_id_blocked))
 }
 
+fn create_or_lookup_mailinglist(
+    context: &Context,
+    allow_creation: bool,
+    create_blocked: Blocked,
+    list_id_header: &str,
+) -> (ChatId, Blocked) {
+    let re = Regex::new(r"^(.*.)<(.*.)>$").unwrap();
+    let (name, listid) = match re.captures(list_id_header) {
+        Some(cap) => (cap[1].trim().to_string(), cap[2].trim().to_string()),
+        None => (
+            list_id_header.trim().to_string(),
+            list_id_header.trim().to_string(),
+        ),
+    };
+
+    chat::get_chat_id_by_mailinglistid(context, &listid).unwrap_or_else(|_e| {
+        if allow_creation {
+            // list does not exist but should be created
+            match create_mailinglist_record(context, &listid, &name, create_blocked) {
+                Ok(chat_id) => {
+                    chat::add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF);
+
+                    // Add the mailing list as "unknown" contact
+                    match add_or_lookup_contact_by_addr(
+                        context,
+                        &Some(name),
+                        &listid,
+                        Origin::IncomingUnknownFrom,
+                        None,
+                    ) {
+                        Ok(list_id_contact) => {
+                            chat::add_to_chat_contacts_table(context, chat_id, list_id_contact);
+                        }
+                        Err(e) => warn!(
+                            context,
+                            "Failed to lookup mailing list contact: {}",
+                            e.to_string()
+                        ),
+                    };
+
+                    (chat_id, create_blocked)
+                }
+                Err(e) => {
+                    warn!(
+                        context,
+                        "Failed to create mailinglist '{}' for grpid={}: {}",
+                        &name,
+                        &listid,
+                        e.to_string()
+                    );
+                    (ChatId::new(0), create_blocked)
+                }
+            }
+        } else {
+            info!(context, "creating list forbidden by caller");
+            (ChatId::new(0), Blocked::Not)
+        }
+    })
+}
+
 /// try extract a grpid from a message-id list header value
 fn extract_grpid(mime_parser: &MimeMessage, headerdef: HeaderDef) -> Option<&str> {
     let header = mime_parser.get(headerdef)?;
@@ -1207,9 +1284,9 @@ fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
     // use subject as initial chat name
-    let grpname = mime_parser
-        .get_subject()
-        .unwrap_or_else(|| "Unnamed group".to_string());
+    let grpname = mime_parser.get_subject().unwrap_or_else(|| {
+        context.stock_string_repl_int(StockMessage::Member, member_ids.len() as i32)
+    });
 
     // create group record
     let new_chat_id: ChatId = create_group_record(
@@ -1228,6 +1305,7 @@ fn create_or_lookup_adhoc_group(
     Ok((new_chat_id, create_blocked))
 }
 
+// Insert a group record into the database. Note that this function is also used by create_mailinglist_record() below.
 fn create_group_record(
     context: &Context,
     grpid: impl AsRef<str>,
@@ -1255,7 +1333,7 @@ fn create_group_record(
     {
         warn!(
             context,
-            "Failed to create group '{}' for grpid={}",
+            "Failed to create group or mailinglist '{}' for grpid={}",
             grpname.as_ref(),
             grpid.as_ref()
         );
@@ -1265,7 +1343,7 @@ fn create_group_record(
     let chat_id = ChatId::new(row_id);
     info!(
         context,
-        "Created group '{}' grpid={} as {}",
+        "Created group or mailinglist '{}' grpid={} as {}",
         grpname.as_ref(),
         grpid.as_ref(),
         chat_id
@@ -1273,6 +1351,27 @@ fn create_group_record(
     chat_id
 }
 
+fn create_mailinglist_record(
+    context: &Context,
+    listid: impl AsRef<str>,
+    name: impl AsRef<str>,
+    create_blocked: Blocked,
+) -> Result<ChatId> {
+    let chat_id = create_group_record(
+        context,
+        &listid,
+        &name,
+        create_blocked,
+        VerifiedStatus::Unverified,
+    );
+    let mut chat = Chat::load_from_db(context, chat_id)?;
+
+    chat.param.set(Param::MailingList, "true");
+    chat.update_param(context)?;
+
+    Ok(chat_id)
+}
+
 fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> String {
     /* algorithm:
     - sort normalized, lowercased, e-mail addresses alphabetically
@@ -1520,7 +1619,7 @@ fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
         return false;
     }
 
-    if let Ok(ids) = parse_message_ids(mid_list) {
+    if let Ok(ids) = mailparse::msgidparse(mid_list) {
         for id in ids.iter() {
             if is_known_rfc724_mid(context, id) {
                 return true;
@@ -1568,7 +1667,7 @@ fn is_reply_to_messenger_message(context: &Context, mime_parser: &MimeMessage) -
 }
 
 pub(crate) fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
-    if let Ok(ids) = parse_message_ids(mid_list) {
+    if let Ok(ids) = mailparse::msgidparse(mid_list) {
         for id in ids.iter() {
             if is_msgrmsg_rfc724_mid(context, id) {
                 return true;
@@ -1592,17 +1691,41 @@ fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
 
 fn dc_add_or_lookup_contacts_by_address_list(
     context: &Context,
-    address_list: &[SingleInfo],
+    addr_list_raw: &str,
     origin: Origin,
+    list_id_header: Option<&String>,
 ) -> Result<ContactIds> {
+    let addrs = match mailparse::addrparse(addr_list_raw) {
+        Ok(addrs) => addrs,
+        Err(err) => {
+            bail!("could not parse {:?}: {:?}", addr_list_raw, err);
+        }
+    };
+
     let mut contact_ids = ContactIds::new();
-    for info in address_list.iter() {
-        contact_ids.insert(add_or_lookup_contact_by_addr(
-            context,
-            &info.display_name,
-            &info.addr,
-            origin,
-        )?);
+    for addr in addrs.iter() {
+        match addr {
+            mailparse::MailAddr::Single(info) => {
+                contact_ids.insert(add_or_lookup_contact_by_addr(
+                    context,
+                    &info.display_name,
+                    &info.addr,
+                    origin,
+                    list_id_header,
+                )?);
+            }
+            mailparse::MailAddr::Group(infos) => {
+                for info in &infos.addrs {
+                    contact_ids.insert(add_or_lookup_contact_by_addr(
+                        context,
+                        &info.display_name,
+                        &info.addr,
+                        origin,
+                        list_id_header,
+                    )?);
+                }
+            }
+        }
     }
 
     Ok(contact_ids)
@@ -1614,6 +1737,7 @@ fn add_or_lookup_contact_by_addr(
     display_name: &Option<String>,
     addr: &str,
     origin: Origin,
+    list_id_header: Option<&String>,
 ) -> Result<u32> {
     if context.is_self_addr(addr)? {
         return Ok(DC_CONTACT_ID_SELF);
@@ -1623,8 +1747,26 @@ fn add_or_lookup_contact_by_addr(
         .map(normalize_name)
         .unwrap_or_default();
 
+    let mut addr = Cow::from(addr);
+    if let Some(list_id) = list_id_header {
+        let list_id = list_id.trim().trim_end_matches('>');
+        let addr_email = EmailAddress::new(&addr)?;
+        let mut addr_domain_parts = addr_email.domain.split('.');
+        let mut list_id_parts = list_id.split('.');
+        if list_id_parts.next_back() == addr_domain_parts.next_back()
+            && list_id_parts.next_back() == addr_domain_parts.next_back()
+        {
+            // list_id was something like "name <name.github.com" (after trimming the last '>')
+            // and addr was something like "notifications@github.com".
+            // addr is not the address of the actual sender but the one of the mailing list.
+            // Add the display name to the addr to make it distinguishable from other people
+            // who sent to the same mailing list.
+            *addr.to_mut() = format!("{} – {}", display_name_normalized, addr);
+        }
+    }
+
     let (row_id, _modified) =
-        Contact::add_or_lookup(context, display_name_normalized, addr, origin)?;
+        Contact::add_or_lookup(context, display_name_normalized, &addr, origin)?;
     ensure!(row_id > 0, "could not add contact: {:?}", addr);
 
     Ok(row_id)
@@ -1654,6 +1796,97 @@ mod tests {
     use crate::message::Message;
     use crate::test_utils::{dummy_context, TestContext};
 
+    static MAILINGLIST: &[u8] = b"From: Max Mustermann <notifications@github.com>\n\
+    To: deltachat/deltachat-core-rust <deltachat-core-rust@noreply.github.com>\n\
+    Subject: [deltachat/deltachat-core-rust] PR run failed\n\
+    Message-ID: <3333@example.org>\n\
+    List-ID: deltachat/deltachat-core-rust <deltachat-core-rust.deltachat.github.com>\n\
+    Precedence: list\n\
+    Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+    \n\
+    hello\n";
+
+    static MAILINGLIST2: &[u8] = b"From: Github <notifications@github.com>\n\
+    To: deltachat/deltachat-core-rust <deltachat-core-rust@noreply.github.com>\n\
+    Subject: [deltachat/deltachat-core-rust] PR run failed\n\
+    Message-ID: <3334@example.org>\n\
+    List-ID: deltachat/deltachat-core-rust <deltachat-core-rust.deltachat.github.com>\n\
+    Precedence: list\n\
+    Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+    \n\
+    hello back\n";
+
+    static MAILINGLIST3: &[u8] = b"From: Alice <alice@posteo.org>\n\
+    To: delta-dev@codespeak.net\n\
+    Subject: Re: [delta-dev] What's up?\n\
+    Message-ID: <38942@posteo.org>\n\
+    List-ID: \"discussions about and around https://delta.chat developments\" <delta.codespeak.net>\n\
+    Precedence: list\n\
+    Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+    \n\
+    body\n";
+
+    #[test]
+    fn test_mailing_list() {
+        let t = configured_offline_context();
+        t.ctx.set_config(Config::ShowEmails, Some("2")).unwrap();
+
+        dc_receive_imf(&t.ctx, MAILINGLIST, "INBOX", 1, false).unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert_eq!(chats.len(), 1);
+
+        let chat_id = chat::create_by_msg_id(&t.ctx, chats.get_msg_id(0).unwrap()).unwrap();
+        let chat = chat::Chat::load_from_db(&t.ctx, chat_id).unwrap();
+
+        assert!(chat.is_mailing_list());
+        assert_eq!(chat.can_send(), false);
+        assert_eq!(chat.name, "deltachat/deltachat-core-rust");
+        println!("{:?}", Contact::load_from_db(&t.ctx, 1));
+        println!("{:?}", chat::get_chat_contacts(&t.ctx, chat_id));
+        assert_eq!(chat::get_chat_contacts(&t.ctx, chat_id).len(), 2);
+
+        dc_receive_imf(&t.ctx, MAILINGLIST2, "INBOX", 1, false).unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert_eq!(chats.len(), 1);
+        let contacts = Contact::get_all(&t.ctx, 0, None as Option<String>).unwrap();
+        assert_eq!(contacts.len(), 0); // mailing list recipients and senders do not count as "known contacts"
+        let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None);
+        let contact1 = Contact::load_from_db(
+            &t.ctx,
+            Message::load_from_db(&t.ctx, msgs[0]).unwrap().from_id,
+        );
+        assert_eq!(
+            contact1.unwrap().get_addr(),
+            "Max Mustermann – notifications@github.com"
+        );
+        let contact2 = Contact::load_from_db(
+            &t.ctx,
+            Message::load_from_db(&t.ctx, msgs[1]).unwrap().from_id,
+        );
+        assert_eq!(
+            contact2.unwrap().get_addr(),
+            "Github – notifications@github.com"
+        );
+    }
+
+    #[test]
+    fn test_mailing_list2() {
+        let t = configured_offline_context();
+        t.ctx.set_config(Config::ShowEmails, Some("2")).unwrap();
+        dc_receive_imf(&t.ctx, MAILINGLIST3, "INBOX", 1, false).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        let chat_id = chat::create_by_msg_id(&t.ctx, chats.get_msg_id(0).unwrap()).unwrap();
+
+        let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None);
+        let contact1 = Contact::load_from_db(
+            &t.ctx,
+            Message::load_from_db(&t.ctx, msgs[0]).unwrap().from_id,
+        );
+        assert_eq!(contact1.unwrap().get_addr(), "alice@posteo.org");
+    }
+
     #[test]
     fn test_hex_hash() {
         let data = "hello world";
@@ -1995,151 +2228,4 @@ mod tests {
         let one2one = Chat::load_from_db(&t.ctx, one2one_id).unwrap();
         assert!(one2one.get_visibility() == ChatVisibility::Archived);
     }
-
-    #[test]
-    fn test_no_from() {
-        // if there is no from given, from_id stays 0 which is just fine. These messages
-        // are very rare, however, we have to add them to the database (they go to the
-        // "deaddrop" chat) to avoid a re-download from the server. See also [**]
-
-        let t = configured_offline_context();
-        let context = &t.ctx;
-
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        assert!(chats.get_msg_id(0).is_err());
-
-        dc_receive_imf(
-            context,
-            b"To: bob@example.org\n\
-                 Subject: foo\n\
-                 Message-ID: <3924@example.org>\n\
-                 Chat-Version: 1.0\n\
-                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
-                 \n\
-                 hello\n",
-            "INBOX",
-            1,
-            false,
-        )
-        .unwrap();
-
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        // Check that the message was added to the database:
-        assert!(chats.get_msg_id(0).is_ok());
-    }
-
-    #[test]
-    fn test_escaped_from() {
-        let t = configured_offline_context();
-        let contact_id = Contact::create(&t.ctx, "foobar", "foobar@example.com").unwrap();
-        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id).unwrap();
-        dc_receive_imf(
-            &t.ctx,
-            b"From: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
-                 To: alice@example.org\n\
-                 Subject: foo\n\
-                 Message-ID: <asdklfjjaweofi@example.org>\n\
-                 Chat-Version: 1.0\n\
-                 Chat-Disposition-Notification-To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
-                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
-                 \n\
-                 hello\n",
-            "INBOX",
-            1,
-            false,
-        ).unwrap();
-        assert_eq!(
-            Contact::load_from_db(&t.ctx, contact_id)
-                .unwrap()
-                .get_authname(),
-            "Фамилия Имя", // The name was "Имя, Фамилия" and ("lastname, firstname") and should be swapped to "firstname, lastname"
-        );
-        let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None);
-        assert_eq!(msgs.len(), 1);
-        let msg_id = msgs.first().unwrap();
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone()).unwrap();
-        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
-        assert_eq!(msg.text.unwrap(), "hello");
-        assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1);
-    }
-
-    #[test]
-    fn test_escaped_recipients() {
-        let t = configured_offline_context();
-        Contact::create(&t.ctx, "foobar", "foobar@example.com").unwrap();
-
-        let carl_contact_id =
-            Contact::add_or_lookup(&t.ctx, "Carl", "carl@host.tld", Origin::IncomingUnknownFrom)
-                .unwrap()
-                .0;
-
-        dc_receive_imf(
-            &t.ctx,
-            b"From: Foobar <foobar@example.com>\n\
-                 To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= alice@example.org\n\
-                 Cc: =?utf-8?q?=3Ch2=3E?= <carl@host.tld>\n\
-                 Subject: foo\n\
-                 Message-ID: <asdklfjjaweofi@example.org>\n\
-                 Chat-Version: 1.0\n\
-                 Chat-Disposition-Notification-To: <foobar@example.com>\n\
-                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
-                 \n\
-                 hello\n",
-            "INBOX",
-            1,
-            false,
-        )
-        .unwrap();
-        assert_eq!(
-            Contact::load_from_db(&t.ctx, carl_contact_id)
-                .unwrap()
-                .get_name(),
-            "h2"
-        );
-
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        let msg = Message::load_from_db(&t.ctx, chats.get_msg_id(0).unwrap()).unwrap();
-        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
-        assert_eq!(msg.text.unwrap(), "hello");
-        assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1);
-    }
-
-    #[test]
-    fn test_cc_to_contact() {
-        let t = configured_offline_context();
-        Contact::create(&t.ctx, "foobar", "foobar@example.com").unwrap();
-
-        let carl_contact_id = Contact::add_or_lookup(
-            &t.ctx,
-            "garabage",
-            "carl@host.tld",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap()
-        .0;
-
-        dc_receive_imf(
-            &t.ctx,
-            b"From: Foobar <foobar@example.com>\n\
-                 To: alice@example.org\n\
-                 Cc: Carl <carl@host.tld>\n\
-                 Subject: foo\n\
-                 Message-ID: <asdklfjjaweofi@example.org>\n\
-                 Chat-Version: 1.0\n\
-                 Chat-Disposition-Notification-To: <foobar@example.com>\n\
-                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
-                 \n\
-                 hello\n",
-            "INBOX",
-            1,
-            false,
-        )
-        .unwrap();
-        assert_eq!(
-            Contact::load_from_db(&t.ctx, carl_contact_id)
-                .unwrap()
-                .get_name(),
-            "Carl"
-        );
-    }
 }

src/dc_tools.rs

@@ -5,14 +5,14 @@ use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::time::{Duration, SystemTime};
+use std::time::SystemTime;
 use std::{fmt, fs};
 
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::context::Context;
-use crate::error::{bail, Error};
+use crate::error::{bail, ensure, Error};
 use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
@@ -75,14 +75,6 @@ pub fn dc_timestamp_to_str(wanted: i64) -> String {
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
-pub fn duration_to_str(duration: Duration) -> String {
-    let secs = duration.as_secs();
-    let h = secs / 3600;
-    let m = (secs % 3600) / 60;
-    let s = (secs % 3600) % 60;
-    format!("{}h {}m {}s", h, m, s)
-}
-
 pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
@@ -460,23 +452,6 @@ pub(crate) fn time() -> i64 {
         .as_secs() as i64
 }
 
-/// An invalid email address was encountered
-#[derive(Debug, thiserror::Error)]
-#[error("Invalid email address: {message} ({addr})")]
-pub struct InvalidEmailError {
-    message: String,
-    addr: String,
-}
-
-impl InvalidEmailError {
-    fn new(msg: impl Into<String>, addr: impl Into<String>) -> InvalidEmailError {
-        InvalidEmailError {
-            message: msg.into(),
-            addr: addr.into(),
-        }
-    }
-}
-
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -500,7 +475,7 @@ pub struct EmailAddress {
 }
 
 impl EmailAddress {
-    pub fn new(input: &str) -> Result<Self, InvalidEmailError> {
+    pub fn new(input: &str) -> Result<Self, Error> {
         input.parse::<EmailAddress>()
     }
 }
@@ -512,58 +487,35 @@ impl fmt::Display for EmailAddress {
 }
 
 impl FromStr for EmailAddress {
-    type Err = InvalidEmailError;
+    type Err = Error;
 
     /// Performs a dead-simple parse of an email address.
-    fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
-        if input.is_empty() {
-            return Err(InvalidEmailError::new("empty string is not valid", input));
-        }
+    fn from_str(input: &str) -> Result<EmailAddress, Error> {
+        ensure!(!input.is_empty(), "empty string is not valid");
         let parts: Vec<&str> = input.rsplitn(2, '@').collect();
 
-        let err = |msg: &str| {
-            Err(InvalidEmailError {
-                message: msg.to_string(),
-                addr: input.to_string(),
-            })
-        };
         match &parts[..] {
             [domain, local] => {
-                if local.is_empty() {
-                    return err("empty string is not valid for local part");
-                }
-                if domain.len() <= 3 {
-                    return err("domain is too short");
-                }
+                ensure!(
+                    !local.is_empty(),
+                    "empty string is not valid for local part"
+                );
+                ensure!(domain.len() > 3, "domain is too short");
+
                 let dot = domain.find('.');
-                match dot {
-                    None => {
-                        return err("invalid domain");
-                    }
-                    Some(dot_idx) => {
-                        if dot_idx >= domain.len() - 2 {
-                            return err("invalid domain");
-                        }
-                    }
-                }
+                ensure!(dot.is_some(), "invalid domain");
+                ensure!(dot.unwrap() < domain.len() - 2, "invalid domain");
+
                 Ok(EmailAddress {
                     local: (*local).to_string(),
                     domain: (*domain).to_string(),
                 })
             }
-            _ => err("missing '@' character"),
+            _ => bail!("missing '@' character"),
         }
     }
 }
 
-impl rusqlite::types::ToSql for EmailAddress {
-    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-        let val = rusqlite::types::Value::Text(self.to_string());
-        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        Ok(out)
-    }
-}
-
 /// Utility to check if a in the binary represantion of listflags
 /// the bit at position bitindex is 1.
 pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
@@ -862,37 +814,4 @@ mod tests {
         let next = dc_smeared_time(&t.ctx);
         assert!((start + count - 1) < next);
     }
-
-    #[test]
-    fn test_duration_to_str() {
-        assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s");
-        assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s");
-        assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s");
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 59)),
-            "0h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 60)),
-            "1h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)),
-            "2h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)),
-            "3h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)),
-            "3h 0m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)),
-            "3h 1m 0s"
-        );
-    }
 }

src/e2ee.rs

@@ -1,17 +1,19 @@
 //! End-to-end encryption support.
 
 use std::collections::HashSet;
+use std::convert::TryFrom;
 
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::config::Config;
+use crate::constants::KeyGenType;
 use crate::context::Context;
+use crate::dc_tools::EmailAddress;
 use crate::error::*;
-use crate::headerdef::HeaderDef;
-use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::key::{self, Key, KeyPairUse, SignedPublicKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -36,7 +38,7 @@ impl EncryptHelper {
             Some(addr) => addr,
         };
 
-        let public_key = SignedPublicKey::load_self(context)?;
+        let public_key = load_or_generate_self_public_key(context, &addr)?;
 
         Ok(EncryptHelper {
             prefer_encrypt,
@@ -106,7 +108,8 @@ impl EncryptHelper {
         }
         let public_key = Key::from(self.public_key.clone());
         keyring.add_ref(&public_key);
-        let sign_key = Key::from(SignedSecretKey::load_self(context)?);
+        let sign_key = Key::from_self_private(context, self.addr.clone(), &context.sql)
+            .ok_or_else(|| format_err!("missing own private key"))?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
@@ -123,8 +126,8 @@ pub fn try_decrypt(
 ) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
     let from = mail
         .headers
-        .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
+        .get_header_value(HeaderDef::From_)
+        .and_then(|from_addr| mailparse::addrparse(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -186,6 +189,41 @@ pub fn try_decrypt(
     Ok((out_mail, signatures))
 }
 
+/// Load public key from database or generate a new one.
+///
+/// This will load a public key from the database, generating and
+/// storing a new one when one doesn't exist yet.  Care is taken to
+/// only generate one key per context even when multiple threads call
+/// this function concurrently.
+fn load_or_generate_self_public_key(
+    context: &Context,
+    self_addr: impl AsRef<str>,
+) -> Result<SignedPublicKey> {
+    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
+        return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key"));
+    }
+    let _guard = context.generating_key_mutex.lock().unwrap();
+
+    // Check again in case the key was generated while we were waiting for the lock.
+    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
+        return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key"));
+    }
+
+    let start = std::time::Instant::now();
+
+    let keygen_type =
+        KeyGenType::from_i32(context.get_config_int(Config::KeyGenType)).unwrap_or_default();
+    info!(context, "Generating keypair with type {}", keygen_type);
+    let keypair = pgp::create_keypair(EmailAddress::new(self_addr.as_ref())?, keygen_type)?;
+    key::store_self_keypair(context, &keypair, KeyPairUse::Default)?;
+    info!(
+        context,
+        "Keypair generated in {:.3}s.",
+        start.elapsed().as_secs()
+    );
+    Ok(keypair.public)
+}
+
 /// Returns a reference to the encrypted payload and validates the autocrypt structure.
 fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
     ensure!(
@@ -307,7 +345,6 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 ///
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
-// TODO, remove this once deltachat::key::Key no longer exists.
 pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
     let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| {
         format_err!(concat!(
@@ -315,7 +352,7 @@ pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
             "cannot ensure secret key if not configured."
         ))
     })?;
-    SignedPublicKey::load_self(context)?;
+    load_or_generate_self_public_key(context, &self_addr)?;
     Ok(self_addr)
 }
 
@@ -366,6 +403,47 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         );
     }
 
+    mod load_or_generate_self_public_key {
+        use super::*;
+
+        #[test]
+        fn test_existing() {
+            let t = dummy_context();
+            let addr = configure_alice_keypair(&t.ctx);
+            let key = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key.is_ok());
+        }
+
+        #[test]
+        fn test_generate() {
+            let t = dummy_context();
+            let addr = "alice@example.org";
+            let key0 = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key0.is_ok());
+            let key1 = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key1.is_ok());
+            assert_eq!(key0.unwrap(), key1.unwrap());
+        }
+
+        #[test]
+        fn test_generate_concurrent() {
+            use std::sync::Arc;
+            use std::thread;
+
+            let t = dummy_context();
+            let ctx = Arc::new(t.ctx);
+            let ctx0 = Arc::clone(&ctx);
+            let thr0 =
+                thread::spawn(move || load_or_generate_self_public_key(&ctx0, "alice@example.org"));
+            let ctx1 = Arc::clone(&ctx);
+            let thr1 =
+                thread::spawn(move || load_or_generate_self_public_key(&ctx1, "alice@example.org"));
+            let res0 = thr0.join().unwrap();
+            let res1 = thr1.join().unwrap();
+            assert_eq!(res0.unwrap(), res1.unwrap());
+        }
+    }
+
     #[test]
     fn test_has_decrypted_pgp_armor() {
         let data = b" -----BEGIN PGP MESSAGE-----";

src/events.rs

@@ -201,4 +201,22 @@ pub enum Event {
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
+
+    /// This event is sent out to the inviter when a joiner successfully joined a group.
+    /// @param data1 (int) chat_id
+    /// @param data2 (int) contact_id
+    #[strum(props(id = "2062"))]
+    SecurejoinMemberAdded { chat_id: ChatId, contact_id: u32 },
+
+    /// This event is sent for each contact added to a chat.
+    /// @param data1 (int) chat_id
+    /// @param data2 (int) contact_id
+    #[strum(props(id = "2063"))]
+    MemberAdded { chat_id: ChatId, contact_id: u32 },
+
+    /// This event is sent for each contact removed from a chat.
+    /// @param data1 (int) chat_id
+    /// @param data2 (int) contact_id
+    #[strum(props(id = "2064"))]
+    MemberRemoved { chat_id: ChatId, contact_id: u32 },
 }

src/headerdef.rs

@@ -53,16 +53,12 @@ impl HeaderDef {
 
 pub trait HeaderDefMap {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String>;
-    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader>;
 }
 
 impl HeaderDefMap for [MailHeader<'_>] {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String> {
         self.get_first_value(headerdef.get_headername())
     }
-    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader> {
-        self.get_first_header(headerdef.get_headername())
-    }
 }
 
 #[cfg(test)]

src/imap/mod.rs

@@ -25,7 +25,6 @@ use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{job_add, Action};
 use crate::login_param::{CertificateChecks, LoginParam};
 use crate::message::{self, update_server_uid};
-use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::stock::StockMessage;
@@ -135,7 +134,7 @@ impl async_imap::Authenticator for OAuth2 {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 enum FolderMeaning {
     Unknown,
     SentObjects,
@@ -269,7 +268,7 @@ impl Imap {
                     let imap_port = config.imap_port;
                     context.stock_string_repl_str2(
                         StockMessage::ServerResponse,
-                        format!("IMAP {}:{}", imap_server, imap_port),
+                        format!("{}:{}", imap_server, imap_port),
                         err.to_string(),
                     )
                 };
@@ -605,12 +604,7 @@ impl Imap {
 
             let headers = get_fetch_headers(fetch)?;
             let message_id = prefetch_get_message_id(&headers).unwrap_or_default();
-            if let Ok(true) =
-                precheck_imf(context, &message_id, folder.as_ref(), cur_uid).map_err(|err| {
-                    warn!(context, "precheck_imf error: {}", err);
-                    err
-                })
-            {
+            if precheck_imf(context, &message_id, folder.as_ref(), cur_uid) {
                 // we know the message-id already or don't want the message otherwise.
                 info!(
                     context,
@@ -619,9 +613,6 @@ impl Imap {
                     folder.as_ref(),
                 );
             } else {
-                // we do not know the message-id
-                // or the message-id is missing (in this case, we create one in the further process)
-                // or some other error happened
                 let show = prefetch_should_download(context, &headers, show_emails)
                     .map_err(|err| {
                         warn!(context, "prefetch_should_download error: {}", err);
@@ -742,7 +733,13 @@ impl Imap {
                 if let Err(err) =
                     dc_receive_imf(context, &body, folder.as_ref(), server_uid, is_seen)
                 {
-                    return Err(Error::Other(format!("dc_receive_imf error: {}", err)));
+                    warn!(
+                        context,
+                        "dc_receive_imf failed for imap-message {}/{}: {:?}",
+                        folder.as_ref(),
+                        server_uid,
+                        err
+                    );
                 }
             }
         } else {
@@ -1063,15 +1060,13 @@ impl Imap {
                     }
                 };
 
-                let sentbox_folder = folders
-                    .iter()
-                    .find(|folder| get_folder_meaning(folder) == FolderMeaning::SentObjects)
-                    .or_else(|| {
-                        info!(context, "can't find sentbox by attributes, checking names");
-                        folders.iter().find(|folder| {
-                            get_folder_meaning_by_name(folder) == FolderMeaning::SentObjects
-                        })
-                    });
+                let sentbox_folder =
+                    folders
+                        .iter()
+                        .find(|folder| match get_folder_meaning(folder) {
+                            FolderMeaning::SentObjects => true,
+                            _ => false,
+                        });
                 info!(context, "sentbox folder is {:?}", sentbox_folder);
 
                 let mut delimiter = ".";
@@ -1237,7 +1232,7 @@ impl Imap {
 // CAVE: if possible, take care not to add a name here that is "sent" in one language
 // but sth. different in others - a hard job.
 fn get_folder_meaning_by_name(folder_name: &Name) -> FolderMeaning {
-    let sent_names = vec!["sent", "sentmail", "sent objects", "gesendet"];
+    let sent_names = vec!["sent", "sent objects", "gesendet"];
     let lower = folder_name.name().to_lowercase();
 
     if sent_names.into_iter().any(|s| s == lower) {
@@ -1248,28 +1243,32 @@ fn get_folder_meaning_by_name(folder_name: &Name) -> FolderMeaning {
 }
 
 fn get_folder_meaning(folder_name: &Name) -> FolderMeaning {
+    if folder_name.attributes().is_empty() {
+        return FolderMeaning::Unknown;
+    }
+
+    let mut res = FolderMeaning::Unknown;
     let special_names = vec!["\\Spam", "\\Trash", "\\Drafts", "\\Junk"];
 
     for attr in folder_name.attributes() {
         if let NameAttribute::Custom(ref label) = attr {
             if special_names.iter().any(|s| *s == label) {
-                return FolderMeaning::Other;
+                res = FolderMeaning::Other;
             } else if label == "\\Sent" {
-                return FolderMeaning::SentObjects;
+                res = FolderMeaning::SentObjects
             }
         }
     }
-    FolderMeaning::Unknown
+
+    match res {
+        FolderMeaning::Unknown => get_folder_meaning_by_name(folder_name),
+        _ => res,
+    }
 }
 
-fn precheck_imf(
-    context: &Context,
-    rfc724_mid: &str,
-    server_folder: &str,
-    server_uid: u32,
-) -> Result<bool> {
-    if let Some((old_server_folder, old_server_uid, msg_id)) =
-        message::rfc724_mid_exists(context, &rfc724_mid)?
+fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server_uid: u32) -> bool {
+    if let Ok((old_server_folder, old_server_uid, msg_id)) =
+        message::rfc724_mid_exists(context, &rfc724_mid)
     {
         if old_server_folder.is_empty() && old_server_uid == 0 {
             info!(
@@ -1323,9 +1322,9 @@ fn precheck_imf(
         if old_server_folder != server_folder || old_server_uid != server_uid {
             update_server_uid(context, &rfc724_mid, server_folder, server_uid);
         }
-        Ok(true)
+        true
     } else {
-        Ok(false)
+        false
     }
 }
 
@@ -1375,8 +1374,15 @@ fn prefetch_should_download(
         .get_header_value(HeaderDef::AutocryptSetupMessage)
         .is_some();
 
-    let (_contact_id, blocked_contact, origin) =
-        from_field_to_contact_id(context, &mimeparser::get_from(headers))?;
+    let from_field = headers
+        .get_header_value(HeaderDef::From_)
+        .unwrap_or_default();
+
+    let (_contact_id, blocked_contact, origin) = from_field_to_contact_id(
+        context,
+        &from_field,
+        headers.get_header_value(HeaderDef::ListId).as_ref(),
+    )?;
     let accepted_contact = origin.is_known();
 
     let show = is_autocrypt_setup_message

src/imap/select_folder.rs

@@ -23,34 +23,6 @@ pub enum Error {
 }
 
 impl Imap {
-    /// Issues a CLOSE command to expunge selected folder.
-    ///
-    /// CLOSE is considerably faster than an EXPUNGE, see
-    /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    async fn close_folder(&self, context: &Context) -> Result<()> {
-        if let Some(ref folder) = self.config.read().await.selected_folder {
-            info!(context, "Expunge messages in \"{}\".", folder);
-
-            if let Some(ref mut session) = &mut *self.session.lock().await {
-                match session.close().await {
-                    Ok(_) => {
-                        info!(context, "close/expunge succeeded");
-                    }
-                    Err(err) => {
-                        self.trigger_reconnect();
-                        return Err(Error::CloseExpungeFailed(err));
-                    }
-                }
-            } else {
-                return Err(Error::NoSession);
-            }
-        }
-        let mut cfg = self.config.write().await;
-        cfg.selected_folder = None;
-        cfg.selected_folder_needs_expunge = false;
-        Ok(())
-    }
-
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
@@ -66,14 +38,39 @@ impl Imap {
             return Err(Error::NoSession);
         }
 
-        let needs_expunge = self.config.read().await.selected_folder_needs_expunge;
-        if needs_expunge {
-            self.close_folder(context).await?;
+        // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
+        // if there is _no_ new folder, we continue as we might want to expunge below.
+        if let Some(ref folder) = folder {
+            if let Some(ref selected_folder) = self.config.read().await.selected_folder {
+                if folder.as_ref() == selected_folder {
+                    return Ok(());
+                }
+            }
         }
 
-        let folder_str: Option<&str> = folder.as_ref().map(|x| x.as_ref());
-        if self.config.read().await.selected_folder.as_deref() == folder_str {
-            return Ok(());
+        // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
+        let needs_expunge = { self.config.read().await.selected_folder_needs_expunge };
+        if needs_expunge {
+            if let Some(ref folder) = self.config.read().await.selected_folder {
+                info!(context, "Expunge messages in \"{}\".", folder);
+
+                // A CLOSE-SELECT is considerably faster than an EXPUNGE-SELECT, see
+                // https://tools.ietf.org/html/rfc3501#section-6.4.2
+                if let Some(ref mut session) = &mut *self.session.lock().await {
+                    match session.close().await {
+                        Ok(_) => {
+                            info!(context, "close/expunge succeeded");
+                        }
+                        Err(err) => {
+                            self.trigger_reconnect();
+                            return Err(Error::CloseExpungeFailed(err));
+                        }
+                    }
+                } else {
+                    return Err(Error::NoSession);
+                }
+            }
+            self.config.write().await.selected_folder_needs_expunge = false;
         }
 
         // select new folder

src/imex.rs

@@ -17,7 +17,7 @@ use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
 use crate::job::*;
-use crate::key::{self, DcKey, Key, SignedSecretKey};
+use crate::key::{self, Key};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
@@ -175,7 +175,9 @@ pub fn render_setup_file(context: &Context, passphrase: &str) -> Result<String>
         passphrase.len() >= 2,
         "Passphrase must be at least 2 chars long."
     );
-    let private_key = Key::from(SignedSecretKey::load_self(context)?);
+    let self_addr = e2ee::ensure_secret_key_exists(context)?;
+    let private_key = Key::from_self_private(context, self_addr, &context.sql)
+        .ok_or_else(|| format_err!("Failed to get private key."))?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled) {
         false => None,
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),

src/job.rs

@@ -75,19 +75,7 @@ impl Default for Thread {
     }
 }
 
-#[derive(
-    Debug,
-    Display,
-    Copy,
-    Clone,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-)]
+#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(i32)]
 pub enum Action {
     Unknown = 0,
@@ -162,68 +150,30 @@ impl fmt::Display for Job {
 }
 
 impl Job {
-    fn new(action: Action, foreign_id: u32, param: Params, delay_seconds: i64) -> Self {
-        let timestamp = time();
-
-        Self {
-            job_id: 0,
-            action,
-            foreign_id,
-            desired_timestamp: timestamp + delay_seconds,
-            added_timestamp: timestamp,
-            tries: 0,
-            param,
-            pending_error: None,
-        }
-    }
-
     /// Deletes the job from the database.
     fn delete(&self, context: &Context) -> bool {
-        if self.job_id != 0 {
-            context
-                .sql
-                .execute("DELETE FROM jobs WHERE id=?;", params![self.job_id as i32])
-                .is_ok()
-        } else {
-            // Already deleted.
-            true
-        }
+        context
+            .sql
+            .execute("DELETE FROM jobs WHERE id=?;", params![self.job_id as i32])
+            .is_ok()
     }
 
-    /// Saves the job to the database, creating a new entry if necessary.
+    /// Updates the job already stored in the database.
     ///
-    /// The Job is consumed by this method.
-    fn save(self, context: &Context) -> bool {
-        let thread: Thread = self.action.into();
-
-        if self.job_id != 0 {
-            sql::execute(
-                context,
-                &context.sql,
-                "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
-                params![
-                    self.desired_timestamp,
-                    self.tries as i64,
-                    self.param.to_string(),
-                    self.job_id as i32,
-                ],
-            )
-            .is_ok()
-        } else {
-            sql::execute(
-                context,
-                &context.sql,
-                "INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);",
-                params![
-                    self.added_timestamp,
-                    thread,
-                    self.action,
-                    self.foreign_id,
-                    self.param.to_string(),
-                    self.desired_timestamp
-                ]
-            ).is_ok()
-        }
+    /// To add a new job, use [job_add].
+    fn update(&self, context: &Context) -> bool {
+        sql::execute(
+            context,
+            &context.sql,
+            "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
+            params![
+                self.desired_timestamp,
+                self.tries as i64,
+                self.param.to_string(),
+                self.job_id as i32,
+            ],
+        )
+        .is_ok()
     }
 
     fn smtp_send<F>(
@@ -993,34 +943,39 @@ pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<()> {
     Ok(())
 }
 
-fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+fn add_imap_deletion_jobs(context: &Context) -> sql::Result<()> {
     if let Some(delete_server_after) = context.get_config_delete_server_after() {
         let threshold_timestamp = time() - delete_server_after;
 
-        context.sql.query_row_optional(
+        // Select all expired messages which don't have a
+        // corresponding message deletion job yet.
+        let msg_ids = context.sql.query_map(
             "SELECT id FROM msgs \
              WHERE timestamp < ? \
-             AND server_uid != 0",
-            params![threshold_timestamp],
+             AND server_uid != 0 \
+             AND NOT EXISTS (SELECT 1 FROM jobs WHERE foreign_id = msgs.id \
+             AND action = ?)",
+            params![threshold_timestamp, Action::DeleteMsgOnImap],
             |row| row.get::<_, MsgId>(0),
-        )
-    } else {
-        Ok(None)
-    }
-}
+            |ids| {
+                ids.collect::<std::result::Result<Vec<_>, _>>()
+                    .map_err(Into::into)
+            },
+        )?;
 
-fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
-    let res = if let Some(msg_id) = load_imap_deletion_msgid(context)? {
-        Some(Job::new(
-            Action::DeleteMsgOnImap,
-            msg_id.to_u32(),
-            Params::new(),
-            0,
-        ))
-    } else {
-        None
-    };
-    Ok(res)
+        // Schedule IMAP deletion for expired messages.
+        for msg_id in msg_ids {
+            job_add(
+                context,
+                Action::DeleteMsgOnImap,
+                msg_id.to_u32() as i32,
+                Params::new(),
+                0,
+            )
+        }
+    }
+
+    Ok(())
 }
 
 pub fn perform_inbox_jobs(context: &Context) {
@@ -1030,6 +985,9 @@ pub fn perform_inbox_jobs(context: &Context) {
     *context.probe_imap_network.write().unwrap() = false;
     *context.perform_inbox_jobs_needed.write().unwrap() = false;
 
+    if let Err(err) = add_imap_deletion_jobs(context) {
+        warn!(context, "Can't add IMAP message deletion jobs: {}", err);
+    }
     job_perform(context, Thread::Imap, probe_imap_network);
     info!(context, "dc_perform_inbox_jobs ended.",);
 }
@@ -1043,17 +1001,7 @@ pub fn perform_sentbox_jobs(context: &Context) {
 }
 
 fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
-    let mut jobs_loaded = 0;
-
     while let Some(mut job) = load_next_job(context, thread, probe_network) {
-        jobs_loaded += 1;
-        if thread == Thread::Imap && jobs_loaded > 20 {
-            // Let the fetch run, but return back to the job afterwards.
-            info!(context, "postponing {}-job {} to run fetch...", thread, job);
-            *context.perform_inbox_jobs_needed.write().unwrap() = true;
-            break;
-        }
-
         info!(context, "{}-job {} started...", thread, job);
 
         // some configuration jobs are "exclusive":
@@ -1111,6 +1059,7 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
                     job.tries = tries;
                     let time_offset = get_backoff_time_offset(tries);
                     job.desired_timestamp = time() + time_offset;
+                    job.update(context);
                     info!(
                         context,
                         "{}-job #{} not succeeded on try #{}, retry in {} seconds.",
@@ -1119,7 +1068,6 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
                         tries,
                         time_offset
                     );
-                    job.save(context);
                     if thread == Thread::Smtp && tries < JOB_RETRIES - 1 {
                         context
                             .smtp_state
@@ -1190,7 +1138,7 @@ fn perform_job_action(context: &Context, mut job: &mut Job, thread: Thread, trie
         Action::ImexImap => match JobImexImap(context, &job) {
             Ok(()) => Status::Finished(Ok(())),
             Err(err) => {
-                error!(context, "Import/export failed: {}", err);
+                error!(context, "{}", err);
                 Status::Finished(Err(err))
             }
         },
@@ -1277,16 +1225,27 @@ pub fn job_add(
         return;
     }
 
-    let job = Job::new(action, foreign_id as u32, param, delay_seconds);
-    job.save(context);
+    let timestamp = time();
+    let thread: Thread = action.into();
 
-    if delay_seconds == 0 {
-        let thread: Thread = action.into();
-        match thread {
-            Thread::Imap => interrupt_inbox_idle(context),
-            Thread::Smtp => interrupt_smtp_idle(context),
-            Thread::Unknown => {}
-        }
+    sql::execute(
+        context,
+        &context.sql,
+        "INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);",
+        params![
+            timestamp,
+            thread,
+            action,
+            foreign_id,
+            param.to_string(),
+            (timestamp + delay_seconds as i64)
+        ]
+    ).ok();
+
+    match thread {
+        Thread::Imap => interrupt_inbox_idle(context),
+        Thread::Smtp => interrupt_smtp_idle(context),
+        Thread::Unknown => {}
     }
 }
 
@@ -1330,7 +1289,7 @@ fn load_next_job(context: &Context, thread: Thread, probe_network: bool) -> Opti
         params_probe
     };
 
-    let job = context
+    context
         .sql
         .query_map(
             query,
@@ -1359,23 +1318,7 @@ fn load_next_job(context: &Context, thread: Thread, probe_network: bool) -> Opti
                 Ok(None)
             },
         )
-        .unwrap_or_default();
-
-    if thread == Thread::Imap {
-        if let Some(job) = job {
-            if job.action < Action::DeleteMsgOnImap {
-                load_imap_deletion_job(context)
-                    .unwrap_or_default()
-                    .or(Some(job))
-            } else {
-                Some(job)
-            }
-        } else {
-            load_imap_deletion_job(context).unwrap_or_default()
-        }
-    } else {
-        job
-    }
+        .unwrap_or_default()
 }
 
 #[cfg(test)]

src/key.rs

@@ -4,16 +4,14 @@ use std::collections::BTreeMap;
 use std::io::Cursor;
 use std::path::Path;
 
-use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
 
-use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::{dc_write_file, time, EmailAddress, InvalidEmailError};
-use crate::sql;
+use crate::dc_tools::*;
+use crate::sql::Sql;
 
 // Re-export key types
 pub use crate::pgp::KeyPair;
@@ -21,22 +19,11 @@ pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
 
 /// Error type for deltachat key handling.
 #[derive(Debug, thiserror::Error)]
-#[non_exhaustive]
 pub enum Error {
     #[error("Could not decode base64")]
     Base64Decode(#[from] base64::DecodeError),
-    #[error("rPGP error: {}", _0)]
-    Pgp(#[from] pgp::errors::Error),
-    #[error("Failed to generate PGP key: {}", _0)]
-    Keygen(#[from] crate::pgp::PgpKeygenError),
-    #[error("Failed to load key: {}", _0)]
-    LoadKey(#[from] sql::Error),
-    #[error("Failed to save generated key: {}", _0)]
-    StoreKey(#[from] SaveKeyError),
-    #[error("No address configured")]
-    NoConfiguredAddr,
-    #[error("Configured address is invalid: {}", _0)]
-    InvalidConfiguredAddr(#[from] InvalidEmailError),
+    #[error("rPGP error: {0}")]
+    PgpError(#[from] pgp::errors::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -64,9 +51,6 @@ pub trait DcKey: Serialize + Deserializable {
         Self::from_slice(&bytes)
     }
 
-    /// Load the users' default key from the database.
-    fn load_self(context: &Context) -> Result<Self::KeyType>;
-
     /// Serialise the key to a base64 string.
     fn to_base64(&self) -> String {
         // Not using Serialize::to_bytes() to make clear *why* it is
@@ -81,91 +65,10 @@ pub trait DcKey: Serialize + Deserializable {
 
 impl DcKey for SignedPublicKey {
     type KeyType = SignedPublicKey;
-
-    fn load_self(context: &Context) -> Result<Self::KeyType> {
-        match context.sql.query_row(
-            r#"
-            SELECT public_key
-              FROM keypairs
-             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1;
-            "#,
-            params![],
-            |row| row.get::<_, Vec<u8>>(0),
-        ) {
-            Ok(bytes) => Self::from_slice(&bytes),
-            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-                let keypair = generate_keypair(context)?;
-                Ok(keypair.public)
-            }
-            Err(err) => Err(err.into()),
-        }
-    }
 }
 
 impl DcKey for SignedSecretKey {
     type KeyType = SignedSecretKey;
-
-    fn load_self(context: &Context) -> Result<Self::KeyType> {
-        match context.sql.query_row(
-            r#"
-            SELECT private_key
-              FROM keypairs
-             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1;
-            "#,
-            params![],
-            |row| row.get::<_, Vec<u8>>(0),
-        ) {
-            Ok(bytes) => Self::from_slice(&bytes),
-            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-                let keypair = generate_keypair(context)?;
-                Ok(keypair.secret)
-            }
-            Err(err) => Err(err.into()),
-        }
-    }
-}
-
-fn generate_keypair(context: &Context) -> Result<KeyPair> {
-    let addr = context
-        .get_config(Config::ConfiguredAddr)
-        .ok_or_else(|| Error::NoConfiguredAddr)?;
-    let addr = EmailAddress::new(&addr)?;
-    let _guard = context.generating_key_mutex.lock().unwrap();
-
-    // Check if the key appeared while we were waiting on the lock.
-    match context.sql.query_row(
-        r#"
-        SELECT public_key, private_key
-          FROM keypairs
-         WHERE addr=?1
-           AND is_default=1;
-        "#,
-        params![addr],
-        |row| Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, Vec<u8>>(1)?)),
-    ) {
-        Ok((pub_bytes, sec_bytes)) => Ok(KeyPair {
-            addr,
-            public: SignedPublicKey::from_slice(&pub_bytes)?,
-            secret: SignedSecretKey::from_slice(&sec_bytes)?,
-        }),
-        Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-            let start = std::time::Instant::now();
-            let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType))
-                .unwrap_or_default();
-            info!(context, "Generating keypair with type {}", keytype);
-            let keypair = crate::pgp::create_keypair(addr, keytype)?;
-            store_self_keypair(context, &keypair, KeyPairUse::Default)?;
-            info!(
-                context,
-                "Keypair generated in {:.3}s.",
-                start.elapsed().as_secs()
-            );
-            Ok(keypair)
-        }
-        Err(err) => Err(err.into()),
-    }
 }
 
 /// Cryptographic key
@@ -282,6 +185,34 @@ impl Key {
         }
     }
 
+    pub fn from_self_public(
+        context: &Context,
+        self_addr: impl AsRef<str>,
+        sql: &Sql,
+    ) -> Option<Self> {
+        let addr = self_addr.as_ref();
+
+        sql.query_get_value(
+            context,
+            "SELECT public_key FROM keypairs WHERE addr=? AND is_default=1;",
+            &[addr],
+        )
+        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Public))
+    }
+
+    pub fn from_self_private(
+        context: &Context,
+        self_addr: impl AsRef<str>,
+        sql: &Sql,
+    ) -> Option<Self> {
+        sql.query_get_value(
+            context,
+            "SELECT private_key FROM keypairs WHERE addr=? AND is_default=1;",
+            &[self_addr.as_ref()],
+        )
+        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Private))
+    }
+
     pub fn to_bytes(&self) -> Vec<u8> {
         match self {
             Key::Public(k) => k.to_bytes().unwrap_or_default(),
@@ -608,59 +539,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         }
     }
 
-    #[test]
-    fn test_load_self_existing() {
-        let alice = alice_keypair();
-        let t = dummy_context();
-        configure_alice_keypair(&t.ctx);
-        let pubkey = SignedPublicKey::load_self(&t.ctx).unwrap();
-        assert_eq!(alice.public, pubkey);
-        let seckey = SignedSecretKey::load_self(&t.ctx).unwrap();
-        assert_eq!(alice.secret, seckey);
-    }
-
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_public() {
-        let t = dummy_context();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
-            .unwrap();
-        let key = SignedPublicKey::load_self(&t.ctx);
-        assert!(key.is_ok());
-    }
-
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_secret() {
-        let t = dummy_context();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
-            .unwrap();
-        let key = SignedSecretKey::load_self(&t.ctx);
-        assert!(key.is_ok());
-    }
-
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_concurrent() {
-        use std::sync::Arc;
-        use std::thread;
-
-        let t = dummy_context();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
-            .unwrap();
-        let ctx = Arc::new(t.ctx);
-        let ctx0 = Arc::clone(&ctx);
-        let thr0 = thread::spawn(move || SignedPublicKey::load_self(&ctx0));
-        let ctx1 = Arc::clone(&ctx);
-        let thr1 = thread::spawn(move || SignedPublicKey::load_self(&ctx1));
-        let res0 = thr0.join().unwrap();
-        let res1 = thr1.join().unwrap();
-        assert_eq!(res0.unwrap(), res1.unwrap());
-    }
-
     #[test]
     fn test_ascii_roundtrip() {
         let public_key = Key::from(KEYPAIR.public.clone());

src/message.rs

@@ -1215,7 +1215,7 @@ pub fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef
         }
         if let Some(error) = error {
             msg.param.set(Param::Error, error.as_ref());
-            warn!(context, "Message failed: {}", error.as_ref());
+            error!(context, "{}", error.as_ref());
         }
 
         if sql::execute(
@@ -1431,15 +1431,12 @@ pub fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> i32 {
 pub(crate) fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
-) -> Result<Option<(String, u32, MsgId)>, Error> {
-    if rfc724_mid.is_empty() {
-        warn!(context, "Empty rfc724_mid passed to rfc724_mid_exists");
-        return Ok(None);
-    }
+) -> Result<(String, u32, MsgId), Error> {
+    ensure!(!rfc724_mid.is_empty(), "empty rfc724_mid");
 
     context
         .sql
-        .query_row_optional(
+        .query_row(
             "SELECT server_folder, server_uid, id FROM msgs WHERE rfc724_mid=?",
             &[rfc724_mid],
             |row| {

src/mimefactory.rs

@@ -15,7 +15,6 @@ use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
-use crate::simplify::escape_message_footer_marks;
 use crate::stock::StockMessage;
 
 // attachments of 25 mb brutto should work on the majority of providers
@@ -808,7 +807,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let message_text = format!(
             "{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
-            escape_message_footer_marks(final_text),
+            &final_text,
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {

src/mimeparser.rs

@@ -1,8 +1,9 @@
 use std::collections::{HashMap, HashSet};
 
+use anyhow::Context as _;
 use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
-use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
+use mailparse::{DispositionType, MailAddr, MailHeaderMap};
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
@@ -36,11 +37,6 @@ use crate::stock::StockMessage;
 pub struct MimeMessage {
     pub parts: Vec<Part>,
     header: HashMap<String, String>,
-
-    /// Addresses are normalized and lowercased:
-    pub recipients: Vec<SingleInfo>,
-    pub from: Vec<SingleInfo>,
-    pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
     pub signatures: HashSet<String>,
     pub gossipped_addr: HashSet<String>,
@@ -92,19 +88,9 @@ impl MimeMessage {
             .unwrap_or_default();
 
         let mut headers = Default::default();
-        let mut recipients = Default::default();
-        let mut from = Default::default();
-        let mut chat_disposition_notification_to = None;
 
         // init known headers with what mailparse provided us
-        MimeMessage::merge_headers(
-            context,
-            &mut headers,
-            &mut recipients,
-            &mut from,
-            &mut chat_disposition_notification_to,
-            &mail.headers,
-        );
+        MimeMessage::merge_headers(&mut headers, &mail.headers);
 
         // remove headers that are allowed _only_ in the encrypted part
         headers.remove("secure-join-fingerprint");
@@ -132,14 +118,7 @@ impl MimeMessage {
 
                     // let known protected headers from the decrypted
                     // part override the unencrypted top-level
-                    MimeMessage::merge_headers(
-                        context,
-                        &mut headers,
-                        &mut recipients,
-                        &mut from,
-                        &mut chat_disposition_notification_to,
-                        &decrypted_mail.headers,
-                    );
+                    MimeMessage::merge_headers(&mut headers, &decrypted_mail.headers);
 
                     (decrypted_mail, signatures)
                 } else {
@@ -163,9 +142,6 @@ impl MimeMessage {
         let mut parser = MimeMessage {
             parts: Vec::new(),
             header: headers,
-            recipients,
-            from,
-            chat_disposition_notification_to,
             decrypting_failed: false,
 
             // only non-empty if it was a valid autocrypt message
@@ -306,18 +282,16 @@ impl MimeMessage {
                     prepend_subject = false
                 }
             }
+            // For mailing lists, always add the subject because sometimes there are different topics
+            // and otherwise it might be hard to keep track:
+            if self.get(HeaderDef::ListId).is_some() {
+                prepend_subject = true;
+            }
             if prepend_subject {
-                let subj = if let Some(n) = subject.find('[') {
-                    &subject[0..n]
-                } else {
-                    subject
-                }
-                .trim();
-
-                if !subj.is_empty() {
+                if !subject.is_empty() {
                     for part in self.parts.iter_mut() {
                         if part.typ == Viewtype::Text {
-                            part.msg = format!("{} – {}", subj, part.msg);
+                            part.msg = format!("{} – {}", subject, part.msg);
                             break;
                         }
                     }
@@ -334,9 +308,11 @@ impl MimeMessage {
 
         // See if an MDN is requested from the other side
         if !self.decrypting_failed && !self.parts.is_empty() {
-            if let Some(ref dn_to) = self.chat_disposition_notification_to {
-                if let Some(ref from) = self.from.get(0) {
-                    if from.addr == dn_to.addr {
+            if let Some(ref dn_to_addr) =
+                self.parse_first_addr(context, HeaderDef::ChatDispositionNotificationTo)
+            {
+                if let Some(ref from_addr) = self.parse_first_addr(context, HeaderDef::From_) {
+                    if compare_addrs(from_addr, dn_to_addr) {
                         if let Some(part) = self.parts.last_mut() {
                             part.param.set_int(Param::WantsMdn, 1);
                         }
@@ -410,6 +386,20 @@ impl MimeMessage {
         self.header.get(headerdef.get_headername())
     }
 
+    fn parse_first_addr(&self, context: &Context, headerdef: HeaderDef) -> Option<MailAddr> {
+        if let Some(value) = self.get(headerdef.clone()) {
+            match mailparse::addrparse(&value) {
+                Ok(ref addrs) => {
+                    return addrs.first().cloned();
+                }
+                Err(err) => {
+                    warn!(context, "header {} parse error: {:?}", headerdef, err);
+                }
+            }
+        }
+        None
+    }
+
     fn parse_mime_recursive(
         &mut self,
         context: &Context,
@@ -753,41 +743,17 @@ impl MimeMessage {
             .and_then(|msgid| parse_message_id(msgid).ok())
     }
 
-    fn merge_headers(
-        context: &Context,
-        headers: &mut HashMap<String, String>,
-        recipients: &mut Vec<SingleInfo>,
-        from: &mut Vec<SingleInfo>,
-        chat_disposition_notification_to: &mut Option<SingleInfo>,
-        fields: &[mailparse::MailHeader<'_>],
-    ) {
+    fn merge_headers(headers: &mut HashMap<String, String>, fields: &[mailparse::MailHeader<'_>]) {
         for field in fields {
             // lowercasing all headers is technically not correct, but makes things work better
             let key = field.get_key().to_lowercase();
             if !headers.contains_key(&key) || // key already exists, only overwrite known types (protected headers)
                     is_known(&key) || key.starts_with("chat-")
             {
-                if key == HeaderDef::ChatDispositionNotificationTo.get_headername() {
-                    match addrparse_header(field) {
-                        Ok(addrlist) => {
-                            *chat_disposition_notification_to = addrlist.extract_single_info();
-                        }
-                        Err(e) => warn!(context, "Could not read {} address: {}", key, e),
-                    }
-                } else {
-                    let value = field.get_value();
-                    headers.insert(key.to_string(), value);
-                }
+                let value = field.get_value();
+                headers.insert(key.to_string(), value);
             }
         }
-        let recipients_new = get_recipients(fields);
-        if !recipients_new.is_empty() {
-            *recipients = recipients_new;
-        }
-        let from_new = get_from(fields);
-        if !from_new.is_empty() {
-            *from = from_new;
-        }
     }
 
     fn process_report(
@@ -855,15 +821,23 @@ fn update_gossip_peerstates(
     gossip_headers: Vec<String>,
 ) -> Result<HashSet<String>> {
     // XXX split the parsing from the modification part
+    let mut recipients: Option<HashSet<String>> = None;
     let mut gossipped_addr: HashSet<String> = Default::default();
 
     for value in &gossip_headers {
         let gossip_header = value.parse::<Aheader>();
 
         if let Ok(ref header) = gossip_header {
-            if get_recipients(&mail.headers)
-                .iter()
-                .any(|info| info.addr == header.addr.to_lowercase())
+            if recipients.is_none() {
+                recipients = Some(get_recipients(
+                    mail.headers.iter().map(|v| (v.get_key(), v.get_value())),
+                ));
+            }
+
+            if recipients
+                .as_ref()
+                .unwrap()
+                .contains(&header.addr.to_lowercase())
             {
                 let mut peerstate = Peerstate::from_addr(context, &context.sql, &header.addr);
                 if let Some(ref mut peerstate) = peerstate {
@@ -901,29 +875,13 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
-pub(crate) fn parse_message_ids(ids: &str) -> Result<Vec<String>> {
-    // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>`
-    let mut msgids = Vec::new();
-    for id in ids.split_whitespace() {
-        let mut id = id.to_string();
-        if id.starts_with('<') {
-            id = id[1..].to_string();
-        }
-        if id.ends_with('>') {
-            id = id[..id.len() - 1].to_string();
-        }
-        if !id.is_empty() {
-            msgids.push(id);
-        }
-    }
-    Ok(msgids)
-}
+pub(crate) fn parse_message_id(value: &str) -> crate::error::Result<String> {
+    let ids = mailparse::msgidparse(value).context("failed to parse message id")?;
 
-pub(crate) fn parse_message_id(ids: &str) -> Result<String> {
-    if let Some(id) = parse_message_ids(ids)?.first() {
+    if let Some(id) = ids.first() {
         Ok(id.to_string())
     } else {
-        bail!("could not parse message_id: {}", ids);
+        bail!("could not parse message_id: {}", value);
     }
 }
 
@@ -1024,11 +982,6 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String
     let desired_filename =
         desired_filename.or_else(|| ct.params.get("name").map(|s| s.to_string()));
 
-    // MS Outlook is known to specify filename in the "name" attribute of
-    // Content-Type and omit Content-Disposition.
-    let desired_filename =
-        desired_filename.or_else(|| mail.ctype.params.get("name").map(|s| s.to_string()));
-
     // If there is no filename, but part is an attachment, guess filename
     if ct.disposition == DispositionType::Attachment && desired_filename.is_none() {
         if let Some(subtype) = mail.ctype.mimetype.split('/').nth(1) {
@@ -1044,50 +997,50 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String
     }
 }
 
-/// Returned addresses are normalized and lowercased.
-fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
-    get_all_addresses_from_header(headers, |header_key| {
-        header_key == "to" || header_key == "cc"
-    })
-}
+// returned addresses are normalized and lowercased.
+fn get_recipients<S: AsRef<str>, T: Iterator<Item = (S, S)>>(headers: T) -> HashSet<String> {
+    let mut recipients: HashSet<String> = Default::default();
 
-/// Returned addresses are normalized and lowercased.
-pub(crate) fn get_from(headers: &[MailHeader]) -> Vec<SingleInfo> {
-    get_all_addresses_from_header(headers, |header_key| header_key == "from")
-}
-
-fn get_all_addresses_from_header<F>(headers: &[MailHeader], pred: F) -> Vec<SingleInfo>
-where
-    F: Fn(String) -> bool,
-{
-    let mut result: Vec<SingleInfo> = Default::default();
-
-    headers
-        .iter()
-        .filter(|header| pred(header.get_key().to_lowercase()))
-        .filter_map(|header| mailparse::addrparse_header(header).ok())
-        .for_each(|addrs| {
-            for addr in addrs.iter() {
-                match addr {
-                    mailparse::MailAddr::Single(ref info) => {
-                        result.push(SingleInfo {
-                            addr: addr_normalize(&info.addr).to_lowercase(),
-                            display_name: info.display_name.clone(),
-                        });
-                    }
-                    mailparse::MailAddr::Group(ref infos) => {
-                        for info in &infos.addrs {
-                            result.push(SingleInfo {
-                                addr: addr_normalize(&info.addr).to_lowercase(),
-                                display_name: info.display_name.clone(),
-                            });
+    for (hkey, hvalue) in headers {
+        let hkey = hkey.as_ref().to_lowercase();
+        let hvalue = hvalue.as_ref();
+        if hkey == "to" || hkey == "cc" {
+            if let Ok(addrs) = mailparse::addrparse(hvalue) {
+                for addr in addrs.iter() {
+                    match addr {
+                        mailparse::MailAddr::Single(ref info) => {
+                            recipients.insert(addr_normalize(&info.addr).to_lowercase());
+                        }
+                        mailparse::MailAddr::Group(ref infos) => {
+                            for info in &infos.addrs {
+                                recipients.insert(addr_normalize(&info.addr).to_lowercase());
+                            }
                         }
                     }
                 }
             }
-        });
+        }
+    }
 
-    result
+    recipients
+}
+
+/// Check if the only addrs match, ignoring names.
+fn compare_addrs(a: &mailparse::MailAddr, b: &mailparse::MailAddr) -> bool {
+    match a {
+        mailparse::MailAddr::Group(group_a) => match b {
+            mailparse::MailAddr::Group(group_b) => group_a
+                .addrs
+                .iter()
+                .zip(group_b.addrs.iter())
+                .all(|(a, b)| a.addr == b.addr),
+            _ => false,
+        },
+        mailparse::MailAddr::Single(single_a) => match b {
+            mailparse::MailAddr::Single(single_b) => single_a.addr == single_b.addr,
+            _ => false,
+        },
+    }
 }
 
 #[cfg(test)]
@@ -1136,11 +1089,12 @@ mod tests {
 
     #[test]
     fn test_get_recipients() {
+        let context = dummy_context();
         let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
-        let mail = mailparse::parse_mail(&raw[..]).unwrap();
-        let recipients = get_recipients(&mail.headers);
-        assert!(recipients.iter().any(|info| info.addr == "abc@bcd.com"));
-        assert!(recipients.iter().any(|info| info.addr == "def@def.de"));
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+        let recipients = get_recipients(mimeparser.header.iter());
+        assert!(recipients.contains("abc@bcd.com"));
+        assert!(recipients.contains("def@def.de"));
         assert_eq!(recipients.len(), 2);
     }
 
@@ -1195,10 +1149,14 @@ mod tests {
 
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
 
-        let of = &mimeparser.from[0];
-        assert_eq!(of.addr, "hello@one.org");
+        let of = mimeparser
+            .parse_first_addr(&context.ctx, HeaderDef::From_)
+            .unwrap();
+        assert_eq!(of, mailparse::addrparse("hello@one.org").unwrap()[0]);
 
-        assert!(mimeparser.chat_disposition_notification_to.is_none());
+        let of =
+            mimeparser.parse_first_addr(&context.ctx, HeaderDef::ChatDispositionNotificationTo);
+        assert!(of.is_none());
     }
 
     #[test]
@@ -1656,142 +1614,4 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
         assert_eq!(message.parts[0].typ, Viewtype::Image);
         assert_eq!(message.parts[0].msg, "Test");
     }
-
-    // Outlook specifies filename in the "name" attribute of Content-Type
-    #[test]
-    fn parse_outlook_html_embedded_image() {
-        let context = dummy_context();
-        let raw = br##"From: Anonymous <anonymous@example.org>
-To: Anonymous <anonymous@example.org>
-Subject: Delta Chat is great stuff!
-Date: Tue, 5 May 2020 01:23:45 +0000
-MIME-Version: 1.0
-Content-Type: multipart/related;
-	boundary="----=_NextPart_000_0003_01D622B3.CA753E60"
-X-Mailer: Microsoft Outlook 15.0
-
-This is a multipart message in MIME format.
-
-------=_NextPart_000_0003_01D622B3.CA753E60
-Content-Type: multipart/alternative;
-	boundary="----=_NextPart_001_0004_01D622B3.CA753E60"
-
-
-------=_NextPart_001_0004_01D622B3.CA753E60
-Content-Type: text/plain;
-	charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-
-
-
-
-------=_NextPart_001_0004_01D622B3.CA753E60
-Content-Type: text/html;
-	charset="us-ascii"
-Content-Transfer-Encoding: quoted-printable
-
-<html>
-<body>
-<p>
-Test<img src="cid:image001.jpg@01D622B3.C9D8D750">
-</p>
-</body>
-</html>
-------=_NextPart_001_0004_01D622B3.CA753E60--
-
-------=_NextPart_000_0003_01D622B3.CA753E60
-Content-Type: image/jpeg;
-	name="image001.jpg"
-Content-Transfer-Encoding: base64
-Content-ID: <image001.jpg@01D622B3.C9D8D750>
-
-ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG
-acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT
-/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal
-n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C
-xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2
-rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU
-8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI
-K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4
-CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
-
-------=_NextPart_000_0003_01D622B3.CA753E60--
-"##;
-
-        let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
-        assert_eq!(
-            message.get_subject(),
-            Some("Delta Chat is great stuff!".to_string())
-        );
-
-        assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.parts[0].typ, Viewtype::Image);
-        assert_eq!(message.parts[0].msg, "Test");
-    }
-
-    #[test]
-    fn test_parse_message_id() {
-        let test = parse_message_id("<foobar>");
-        assert!(test.is_ok());
-        assert_eq!(test.unwrap(), "foobar");
-
-        let test = parse_message_id("<foo> <bar>");
-        assert!(test.is_ok());
-        assert_eq!(test.unwrap(), "foo");
-
-        let test = parse_message_id("  < foo > <bar>");
-        assert!(test.is_ok());
-        assert_eq!(test.unwrap(), "foo");
-
-        let test = parse_message_id("foo");
-        assert!(test.is_ok());
-        assert_eq!(test.unwrap(), "foo");
-
-        let test = parse_message_id(" foo ");
-        assert!(test.is_ok());
-        assert_eq!(test.unwrap(), "foo");
-
-        let test = parse_message_id("foo bar");
-        assert!(test.is_ok());
-        assert_eq!(test.unwrap(), "foo");
-
-        let test = parse_message_id("  foo  bar ");
-        assert!(test.is_ok());
-        assert_eq!(test.unwrap(), "foo");
-
-        let test = parse_message_id("");
-        assert!(test.is_err());
-
-        let test = parse_message_id(" ");
-        assert!(test.is_err());
-
-        let test = parse_message_id("<>");
-        assert!(test.is_err());
-
-        let test = parse_message_id("<> bar");
-        assert!(test.is_ok());
-        assert_eq!(test.unwrap(), "bar");
-    }
-
-    #[test]
-    fn test_parse_message_ids() {
-        let test = parse_message_ids("  foo  bar <foobar>").unwrap();
-        assert_eq!(test.len(), 3);
-        assert_eq!(test[0], "foo");
-        assert_eq!(test[1], "bar");
-        assert_eq!(test[2], "foobar");
-
-        let test = parse_message_ids("  < foobar >").unwrap();
-        assert_eq!(test.len(), 1);
-        assert_eq!(test[0], "foobar");
-
-        let test = parse_message_ids("").unwrap();
-        assert!(test.is_empty());
-
-        let test = parse_message_ids(" ").unwrap();
-        assert!(test.is_empty());
-
-        let test = parse_message_ids("  < ").unwrap();
-        assert!(test.is_empty());
-    }
 }

src/param.rs

@@ -117,6 +117,8 @@ pub enum Param {
 
     /// For MDN-sending job
     MsgId = b'I',
+
+    MailingList = b't',
 }
 
 /// Possible values for `Param::ForcePlaintext`.

src/pgp.rs

@@ -119,7 +119,7 @@ pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap<String, Str
 /// since all variability is hardcoded.
 #[derive(Debug, thiserror::Error)]
 #[error("PgpKeygenError: {message}")]
-pub struct PgpKeygenError {
+pub(crate) struct PgpKeygenError {
     message: String,
     #[source]
     cause: anyhow::Error,

src/provider/data.rs

@@ -18,7 +18,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // aol.md: aol.com
@@ -29,7 +28,6 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/aol",
         server: vec![
         ],
-        config_defaults: None,
     };
 
     // autistici.org.md: autistici.org
@@ -42,7 +40,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mail.autistici.org", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -55,7 +52,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imaps.bluewin.ch", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // comcast.md: xfinity.com, comcast.net
@@ -77,7 +73,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.example.com", port: 1337, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // fastmail.md: fastmail.com
@@ -88,7 +83,6 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/fastmail",
         server: vec![
         ],
-        config_defaults: None,
     };
 
     // freenet.de.md: freenet.de
@@ -101,7 +95,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mx.freenet.de", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -114,7 +107,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.gmail.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -128,7 +120,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "mail.gmx.net", port: 465, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // i.ua.md: i.ua
@@ -144,7 +135,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.me.com", port: 993, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // kolst.com.md: kolst.com
@@ -169,15 +159,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.nauta.cu", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.nauta.cu", port: 25, username_pattern: EMAIL },
         ],
-        config_defaults: Some(vec![
-            ConfigDefault { key: Config::DeleteServerAfter, value: "1" },
-            ConfigDefault { key: Config::BccSelf, value: "0" },
-            ConfigDefault { key: Config::SentboxWatch, value: "0" },
-            ConfigDefault { key: Config::MvboxWatch, value: "0" },
-            ConfigDefault { key: Config::MvboxMove, value: "0" },
-            ConfigDefault { key: Config::E2eeEnabled, value: "0" },
-            ConfigDefault { key: Config::MediaQuality, value: "1" },
-        ]),
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -190,7 +171,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap-mail.outlook.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // posteo.md: posteo.de
@@ -203,7 +183,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "posteo.de", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -214,7 +193,6 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/protonmail",
         server: vec![
         ],
-        config_defaults: None,
     };
 
     // riseup.net.md: riseup.net
@@ -223,9 +201,6 @@ lazy_static::lazy_static! {
     // rogers.com.md: rogers.com
     // - skipping provider with status OK and no special things to do
 
-    // systemli.org.md: systemli.org
-    // - skipping provider with status OK and no special things to do
-
     // t-online.md: t-online.de, magenta.de
     static ref P_T_ONLINE: Provider = Provider {
         status: Status::PREPARATION,
@@ -234,7 +209,6 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/t-online",
         server: vec![
         ],
-        config_defaults: None,
     };
 
     // testrun.md: testrun.org
@@ -248,7 +222,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "testrun.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // tiscali.it.md: tiscali.it
@@ -261,7 +234,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.tiscali.it", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // ukr.net.md: ukr.net
@@ -281,7 +253,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.web.de", port: 143, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
-        config_defaults: None,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
@@ -294,19 +265,10 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.yahoo.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // yandex.ru.md: yandex.ru, yandex.com
-    static ref P_YANDEX_RU: Provider = Provider {
-        status: Status::PREPARATION,
-        before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/yandex-ru",
-        server: vec![
-        ],
-        config_defaults: None,
-    };
+    // - skipping provider with status OK and no special things to do
 
     // ziggo.nl.md: ziggo.nl
     static ref P_ZIGGO_NL: Provider = Provider {
@@ -318,7 +280,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.ziggo.nl", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
@@ -399,8 +360,6 @@ lazy_static::lazy_static! {
         ("ymail.com", &*P_YAHOO),
         ("rocketmail.com", &*P_YAHOO),
         ("yahoodns.net", &*P_YAHOO),
-        ("yandex.ru", &*P_YANDEX_RU),
-        ("yandex.com", &*P_YANDEX_RU),
         ("ziggo.nl", &*P_ZIGGO_NL),
     ].iter().copied().collect();
 }

src/provider/mod.rs

@@ -2,7 +2,6 @@
 
 mod data;
 
-use crate::config::Config;
 use crate::dc_tools::EmailAddress;
 use crate::provider::data::PROVIDER_DATA;
 
@@ -58,12 +57,6 @@ impl Server {
     }
 }
 
-#[derive(Debug)]
-pub struct ConfigDefault {
-    pub key: Config,
-    pub value: &'static str,
-}
-
 #[derive(Debug)]
 pub struct Provider {
     pub status: Status,
@@ -71,7 +64,6 @@ pub struct Provider {
     pub after_login_hint: &'static str,
     pub overview_page: &'static str,
     pub server: Vec<Server>,
-    pub config_defaults: Option<Vec<ConfigDefault>>,
 }
 
 impl Provider {

src/provider/update.py

@@ -9,9 +9,6 @@ out_all = ""
 out_domains = ""
 domains_dict = {}
 
-def camel(name):
-    words = name.split("_")
-    return "".join(w.capitalize() for i, w in enumerate(words))
 
 def cleanstr(s):
     s = s.strip()
@@ -34,18 +31,6 @@ def file2url(f):
     return "https://providers.delta.chat/" + f
 
 
-def process_config_defaults(data):
-    if not "config_defaults" in data:
-        return "None"
-    defaults = "Some(vec![\n"
-    config_defaults = data.get("config_defaults", "")
-    for key in config_defaults:
-        value = str(config_defaults[key])
-        defaults += "            ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
-    defaults += "        ])"
-    return defaults
-
-
 def process_data(data, file):
     status = data.get("status", "")
     if status != "OK" and status != "PREPARATION" and status != "BROKEN":
@@ -98,8 +83,6 @@ def process_data(data, file):
             server += ("            Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
             + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern + " },\n")
 
-    config_defaults = process_config_defaults(data)
-
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -110,7 +93,6 @@ def process_data(data, file):
         provider += "        after_login_hint: \"" + after_login_hint + "\",\n"
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
-        provider += "        config_defaults: " + config_defaults + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -121,7 +103,7 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None":
+    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "":
         out_all += "    // - skipping provider with status OK and no special things to do\n\n"
     else:
         out_all += provider

src/qr.rs

@@ -37,10 +37,6 @@ impl Into<Lot> for Error {
     }
 }
 
-fn starts_with_ignore_case(string: &str, pattern: &str) -> bool {
-    string.to_lowercase().starts_with(&pattern.to_lowercase())
-}
-
 /// Check a scanned QR code.
 /// The function should be called after a QR code is scanned.
 /// The function takes the raw text scanned and checks what can be done with it.
@@ -49,9 +45,9 @@ pub fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
 
     info!(context, "Scanned QR code: {}", qr);
 
-    if starts_with_ignore_case(qr, OPENPGP4FPR_SCHEME) {
+    if qr.starts_with(OPENPGP4FPR_SCHEME) {
         decode_openpgp(context, qr)
-    } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
+    } else if qr.starts_with(DCACCOUNT_SCHEME) {
         decode_account(context, qr)
     } else if qr.starts_with(MAILTO_SCHEME) {
         decode_mailto(context, qr)
@@ -521,17 +517,6 @@ mod tests {
         assert_ne!(res.get_id(), 0);
         assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
-        // Test it again with lowercased "openpgp4fpr:" uri scheme
-        let res = check_qr(
-            &ctx.ctx,
-            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
-        );
-
-        println!("{:?}", res);
-        assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
-        assert_ne!(res.get_id(), 0);
-        assert_eq!(res.get_text1().unwrap(), "test ? test !");
-
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
     }
@@ -549,16 +534,6 @@ mod tests {
         assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
         assert_ne!(res.get_id(), 0);
 
-        // Test it again with lowercased "openpgp4fpr:" uri scheme
-        let res = check_qr(
-            &ctx.ctx,
-            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
-        );
-
-        println!("{:?}", res);
-        assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
-        assert_ne!(res.get_id(), 0);
-
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
         assert_eq!(contact.get_name(), "Jörn P. P.");
@@ -579,19 +554,6 @@ mod tests {
         );
         assert_eq!(res.get_id(), 0);
 
-        // Test it again with lowercased "openpgp4fpr:" uri scheme
-
-        let res = check_qr(
-            &ctx.ctx,
-            "openpgp4fpr:1234567890123456789012345678901234567890",
-        );
-        assert_eq!(res.get_state(), LotState::QrFprWithoutAddr);
-        assert_eq!(
-            res.get_text1().unwrap(),
-            "1234 5678 9012 3456 7890\n1234 5678 9012 3456 7890"
-        );
-        assert_eq!(res.get_id(), 0);
-
         let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890");
         assert_eq!(res.get_state(), LotState::QrError);
         assert_eq!(res.get_id(), 0);
@@ -607,14 +569,6 @@ mod tests {
         );
         assert_eq!(res.get_state(), LotState::QrAccount);
         assert_eq!(res.get_text1().unwrap(), "example.org");
-
-        // Test it again with lowercased "dcaccount:" uri scheme
-        let res = check_qr(
-            &ctx.ctx,
-            "dcaccount:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
-        assert_eq!(res.get_state(), LotState::QrAccount);
-        assert_eq!(res.get_text1().unwrap(), "example.org");
     }
 
     #[test]
@@ -627,13 +581,5 @@ mod tests {
         );
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
-
-        // Test it again with lowercased "dcaccount:" uri scheme
-        let res = check_qr(
-            &ctx.ctx,
-            "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
-        assert_eq!(res.get_state(), LotState::QrError);
-        assert!(res.get_text1().is_some());
     }
 }

src/securejoin.rs

@@ -12,7 +12,7 @@ use crate::e2ee::*;
 use crate::error::{bail, Error};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
-use crate::key::{dc_normalize_fingerprint, DcKey, Key, SignedPublicKey};
+use crate::key::{dc_normalize_fingerprint, Key};
 use crate::lot::LotState;
 use crate::message::Message;
 use crate::mimeparser::*;
@@ -135,13 +135,12 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<
 }
 
 fn get_self_fingerprint(context: &Context) -> Option<String> {
-    match SignedPublicKey::load_self(context) {
-        Ok(key) => Some(Key::from(key).fingerprint()),
-        Err(_) => {
-            warn!(context, "get_self_fingerprint(): failed to load key");
-            None
+    if let Some(self_addr) = context.get_config(Config::ConfiguredAddr) {
+        if let Some(key) = Key::from_self_public(context, self_addr, &context.sql) {
+            return Some(key.fingerprint());
         }
     }
+    None
 }
 
 /// Take a scanned QR-code and do the setup-contact/join-group handshake.
@@ -754,12 +753,17 @@ pub(crate) fn handle_securejoin_handshake(
                         .get(HeaderDef::SecureJoinGroup)
                         .map(|s| s.as_str())
                         .unwrap_or_else(|| "");
-                    if let Err(err) = chat::get_chat_id_by_grpid(context, &field_grpid) {
+                    let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid)
+                        .map_err(|err| {
                         warn!(context, "Failed to lookup chat_id from grpid: {}", err);
-                        return Err(HandshakeError::ChatNotFound {
+                        HandshakeError::ChatNotFound {
                             group: field_grpid.to_string(),
-                        });
-                    }
+                        }
+                    })?;
+                    context.call_cb(Event::MemberAdded {
+                        chat_id: group_chat_id,
+                        contact_id,
+                    });
                 }
                 Ok(HandshakeMessage::Ignore) // "Done" deletes the message and breaks multi-device
             } else {

src/simplify.rs

@@ -1,41 +1,13 @@
-// protect lines starting with `--` against being treated as a footer.
-// for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B);
-// this should be invisible on most systems and there is no need to unescape it again
-// (which won't be done by non-deltas anyway)
-//
-// this escapes a bit more than actually needed by delta (eg. also lines as "-- footer"),
-// but for non-delta-compatibility, that seems to be better.
-// (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced)
-pub fn escape_message_footer_marks(text: &str) -> String {
-    if text.starts_with("--") {
-        "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-")
-    } else {
-        text.replace("\n--", "\n-\u{200B}-")
-    }
-}
-
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
 fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
-    let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
+        // quoted-printable may encode `-- ` to `-- =20` which is converted
+        // back to `--  `
         match line {
-            // some providers encode `-- ` to `-- =20` which results in `--  `
             "-- " | "--  " => return &lines[..ix],
-            // some providers encode `-- ` to `=2D-` which results in only `--`;
-            // use that only when no other footer is found
-            // and if the line before is empty and the line after is not empty
-            "--" => {
-                if (ix == 0 || lines[ix - 1] == "") && ix != lines.len() - 1 && lines[ix + 1] != ""
-                {
-                    nearly_standard_footer = Some(ix);
-                }
-            }
             _ => (),
         }
     }
-    if let Some(ix) = nearly_standard_footer {
-        return &lines[..ix];
-    }
     lines
 }
 
@@ -69,27 +41,25 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
     let lines = split_lines(&input);
     let (lines, is_forwarded) = skip_forward_header(&lines);
 
-    let original_lines = &lines;
-
     let lines = remove_message_footer(lines);
-
-    let text = if is_chat_message {
-        render_message(lines, false, false)
+    let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
+    let (lines, has_bottom_quote) = if !is_chat_message {
+        remove_bottom_quote(lines)
     } else {
-        let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
-        let (lines, has_bottom_quote) = remove_bottom_quote(lines);
-        let (lines, has_top_quote) = remove_top_quote(lines);
-
-        if lines.iter().all(|it| it.trim().is_empty()) {
-            render_message(original_lines, false, false)
-        } else {
-            render_message(
-                lines,
-                has_top_quote,
-                has_nonstandard_footer || has_bottom_quote,
-            )
-        }
+        (lines, false)
     };
+    let (lines, has_top_quote) = if !is_chat_message {
+        remove_top_quote(lines)
+    } else {
+        (lines, false)
+    };
+
+    // re-create buffer from the remaining lines
+    let text = render_message(
+        lines,
+        has_top_quote,
+        has_nonstandard_footer || has_bottom_quote,
+    );
     (text, is_forwarded)
 }
 
@@ -184,8 +154,7 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) ->
     if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
         ret += " [...]";
     }
-    // redo escaping done by escape_message_footer_marks()
-    ret.replace("\u{200B}", "")
+    ret
 }
 
 /**
@@ -234,25 +203,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_dont_remove_whole_message() {
-        let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
-        assert_eq!(
-            plain,
-            "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
-        );
-        assert!(!is_forwarded);
-    }
-
-    #[test]
-    fn test_chat_message() {
-        let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
-        let (plain, is_forwarded) = simplify(input, true);
-        assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
-        assert!(!is_forwarded);
-    }
-
     #[test]
     fn test_simplify_trim() {
         let input = "line1\n\r\r\rline2".to_string();
@@ -297,46 +247,4 @@ mod tests {
         assert_eq!(lines, &["not a quote", "> first", "> second"]);
         assert!(!has_top_quote);
     }
-
-    #[test]
-    fn test_escape_message_footer_marks() {
-        let esc = escape_message_footer_marks("--\n--text --in line");
-        assert_eq!(esc, "-\u{200B}-\n-\u{200B}-text --in line");
-
-        let esc = escape_message_footer_marks("--\r\n--text");
-        assert_eq!(esc, "-\u{200B}-\r\n-\u{200B}-text");
-    }
-
-    #[test]
-    fn test_remove_message_footer() {
-        let input = "text\n--\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n--\nno footer");
-
-        let input = "text\n\n--\n\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n\n--\n\nno footer");
-
-        let input = "text\n\n-- no footer\n\n".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n\n-- no footer");
-
-        let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n\n--\nno footer");
-
-        let input = "text\n\n--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
-        assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
-        let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
-        assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
-
-        let input = "--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
-        assert_eq!(plain, ""); // see remove_message_footer() for some explanations
-        let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
-        assert_eq!(plain, "--\ntreated as footer when unescaped");
-    }
 }

src/smtp/mod.rs

@@ -12,7 +12,6 @@ use crate::context::Context;
 use crate::events::Event;
 use crate::login_param::{dc_build_tls, LoginParam};
 use crate::oauth2::*;
-use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
@@ -172,14 +171,7 @@ impl Smtp {
         let mut trans = client.into_transport();
 
         trans.connect().await.map_err(|err| {
-            let message = {
-                context.stock_string_repl_str2(
-                    StockMessage::ServerResponse,
-                    format!("SMTP {}:{}", domain, port),
-                    err.to_string(),
-                )
-            };
-            emit_event!(context, Event::ErrorNetwork(message));
+            emit_event!(context, Event::ErrorNetwork(err.to_string()));
             Error::ConnectionFailure(err)
         })?;
 

src/sql.rs

@@ -242,7 +242,7 @@ impl Sql {
         match self.query_get_value_result(query, params) {
             Ok(res) => res,
             Err(err) => {
-                warn!(context, "sql: Failed query_row: {}", err);
+                error!(context, "sql: Failed query_row: {}", err);
                 None
             }
         }
@@ -400,7 +400,7 @@ fn open(
     }
     let mgr = r2d2_sqlite::SqliteConnectionManager::file(dbfile.as_ref())
         .with_flags(open_flags)
-        .with_init(|c| c.execute_batch("PRAGMA journal_mode=WAL; PRAGMA secure_delete=on;"));
+        .with_init(|c| c.execute_batch("PRAGMA secure_delete=on;"));
     let pool = r2d2::Pool::builder()
         .min_idle(Some(2))
         .max_size(10)
@@ -1173,7 +1173,7 @@ pub fn housekeeping(context: &Context) {
     if let Err(err) = prune_tombstones(context) {
         warn!(
             context,
-            "Housekeeping: Cannot prune message tombstones: {}", err
+            "Houskeeping: Cannot prune message tombstones: {}", err
         );
     }
 

src/stock.rs

@@ -35,6 +35,12 @@ pub enum StockMessage {
     #[strum(props(fallback = "Draft"))]
     Draft = 3,
 
+    #[strum(props(fallback = "%1$s member(s)"))]
+    Member = 4,
+
+    #[strum(props(fallback = "%1$s contact(s)"))]
+    Contact = 6,
+
     #[strum(props(fallback = "Voice message"))]
     VoiceMessage = 7,
 
@@ -130,6 +136,9 @@ pub enum StockMessage {
     ))]
     AcSetupMsgBody = 43,
 
+    #[strum(props(fallback = "Messages I sent to myself"))]
+    SelfTalkSubTitle = 50,
+
     #[strum(props(fallback = "Cannot login as %1$s."))]
     CannotLogin = 60,
 
@@ -421,9 +430,8 @@ mod tests {
         let t = dummy_context();
         // uses %1$s substitution
         assert_eq!(
-            t.ctx
-                .stock_string_repl_str(StockMessage::MsgAddMember, "Foo"),
-            "Member Foo added."
+            t.ctx.stock_string_repl_str(StockMessage::Member, "42"),
+            "42 member(s)"
         );
         // We have no string using %1$d to test...
     }
@@ -432,8 +440,8 @@ mod tests {
     fn test_stock_string_repl_int() {
         let t = dummy_context();
         assert_eq!(
-            t.ctx.stock_string_repl_int(StockMessage::MsgAddMember, 42),
-            "Member 42 added."
+            t.ctx.stock_string_repl_int(StockMessage::Member, 42),
+            "42 member(s)"
         );
     }
 

src/test_utils.rs

@@ -42,14 +42,14 @@ pub(crate) fn test_context(callback: Option<Box<ContextCallback>>) -> TestContex
 /// specified in [test_context] but there is no callback hooked up,
 /// i.e. [Context::call_cb] will always return `0`.
 pub(crate) fn dummy_context() -> TestContext {
-    test_context(Some(Box::new(logging_cb)))
+    test_context(None)
 }
 
 pub(crate) fn logging_cb(_ctx: &Context, evt: Event) {
     match evt {
         Event::Info(msg) => println!("I: {}", msg),
-        Event::Warning(msg) => eprintln!("=== WARNING ===\n{}\n===============", msg),
-        Event::Error(msg) => eprintln!("\n===================== ERROR =====================\n{}\n=================================================\n", msg),
+        Event::Warning(msg) => println!("W: {}", msg),
+        Event::Error(msg) => println!("E: {}", msg),
         _ => (),
     }
 }