prepare 1.64 (#2802)

* update changelog for 1.64

* bump version to 1.64.0

.github/workflows/ci.yml

@@ -76,14 +76,13 @@ jobs:
             rust: 1.54.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.48.0
-          # This is the Debian "bullseye" release version of Rust.
+          # Minimum Supported Rust Version = 1.51.0
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.48.0
+            rust: 1.51.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:

.gitignore

@@ -28,3 +28,4 @@ deltachat-ffi/xml
 .rsynclist
 
 coverage/
+.DS_Store

CHANGELOG.md

@@ -1,4 +1,103 @@
-# Changelog 
+# Changelog
+
+## 1.64.0
+
+### Fixes
+- add 'waiting for being added to the group' only for group-joins,
+  not for setup-contact #2797
+- prioritize In-Reply-To: and References: headers over group IDs when assigning
+  messages to chats to fix incorrect assignment of Delta Chat replies to
+  classic email threads #2795
+
+
+## 1.63.0
+
+### API changes
+- `dc_get_last_error()` added #2788
+
+### Changes
+- Optimize Autocrypt gossip #2743
+
+### Fixes
+- fix permanently hiding of one-to-one chats after secure-join #2791
+
+
+## 1.62.0
+
+### API Changes
+- `dc_join_securejoin()` now always returns immediately;
+  the returned chat may not allow sending (`dc_chat_can_send()` returns false)
+  which may change as usual on `DC_EVENT_CHAT_MODIFIED` #2508 #2767
+- introduce multi-device-sync-messages;
+  as older cores display them as files in self-chat,
+  they are currently only sent if config option `send_sync_msgs` is set #2669
+- add `DC_EVENT_SELFAVATAR_CHANGED` #2742
+
+### Changes
+- use system DNS instead of google for MX queries #2780
+- improve error logging #2758
+- improve tests #2764 #2781
+- improve ci #2770
+- refactorings #2677 #2728 #2740 #2729 #2766 #2778
+
+### Fixes
+- add Let's Encrypt certificate to core as it may be missing older devices #2752
+- prioritize certificate setting from user over the one from provider-db #2749
+- fix "QR process failed" error #2725
+- do not update quota in endless loop #2726
+
+
+## 1.61.0
+
+### API Changes
+- download-on-demand added: `dc_msg_get_download_status()`, `dc_download_full_msg()`
+  and `download_limit` config option #2631 #2696
+- `dc_create_broadcast_list()` and chat type `DC_CHAT_TYPE_BROADCAST` added #2707 #2722
+- allow ui-specific configs: `dc_set_ui_config()` and `dc_get_ui_config()` #2672
+- new strings from `DC_STR_PARTIAL_DOWNLOAD_MSG_BODY`
+  to `DC_STR_PART_OF_TOTAL_USED` #2631 #2694 #2707 #2723
+- emit warnings and errors from account manager with account-id 0 #2712
+
+### Changes
+- notify about incoming contact requests #2690
+- messages are marked as read on first read receipt #2699
+- quota warning reappears after import, rewarning at 95% #2702
+- lock strict TLS if certificate checks are automatic #2711
+- always check certificates strictly when connecting over SOCKS5 in Automatic mode #2657
+- `Accounts` is not cloneable anymore #2654 #2658
+- update chat/contact data only when there was no newer update #2642
+- better detection of mailing list names #2665 #2685
+- log all decisions when applying ephemeral timer to chats #2679
+- connectivity view now translatable #2694 #2723
+- improve Doxygen documentation #2647 #2668 #2684 #2688 #2705
+- refactorings #2656 #2659 #2677 #2673 #2678 #2675 #2663 #2692 #2706
+- update provider database #2618
+
+### Fixes
+- ephemeral timer rollback protection #2693 #2709
+- recreate configured folders if they are deleted #2691
+- ignore MDNs sent to self #2674
+- recognize NDNs that put headers into "message/global-headers" part #2598
+- avoid `dc_get_contacts()` returning duplicate contact ids #2591
+- do not leak group names on forwarding messages #2719
+- in case of smtp-errors, iterate over all addresses to fix ipv6/v4 problems #2720
+- fix pkg-config file #2660
+- fix "QR process failed" error #2725
+
+
+## 1.60.0
+
+### Added
+- add device message to warn about QUOTA #2621
+- add SOCKS5 support #2474 #2620
+
+### Changes
+- don't emit multiple events with the same import/export progress number #2639
+- reduce message length limit to 5000 chars #2615
+
+### Fixes
+- keep event emitter from closing when there are no accounts #2636
+
 
 ## 1.59.0
 
@@ -24,7 +123,6 @@
 ## 1.58.0
 
 ### Fixes
-
 - move WAL file together with database
   and avoid using data if the database was not closed correctly before #2583
 

CMakeLists.txt

@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.16)
 project(deltachat LANGUAGES C)
+include(GNUInstallDirs)
 
 find_program(CARGO cargo)
 
@@ -35,7 +36,6 @@ add_custom_target(
 	"target/release/pkgconfig/deltachat.pc"
 )
 
-include(GNUInstallDirs)
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})

Cargo.toml

@@ -1,9 +1,10 @@
 [package]
 name = "deltachat"
-version = "1.59.0"
+version = "1.64.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
+resolver = "2"
 
 [profile.dev]
 debug = 0
@@ -15,76 +16,76 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1.0.42"
+anyhow = "1"
 async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3.3" }
+async-native-tls = { version = "0.3" }
 async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
-async-std-resolver = "0.20.3"
-async-std = { version = "~1.9.0", features = ["unstable"] }
-async-tar = "0.3.0"
-async-trait = "0.1.50"
-backtrace = "0.3.59"
+async-std-resolver = "0.20"
+async-std = { version = "1", features = ["unstable"] }
+async-tar = "0.4"
+async-trait = "0.1"
+backtrace = "0.3"
 base64 = "0.13"
-bitflags = "1.3.1"
-byteorder = "1.3.1"
-chrono = "0.4.6"
-dirs = { version = "3.0.2", optional=true }
+bitflags = "1.3"
+byteorder = "1.3"
+chrono = "0.4"
+dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1.1"
-futures = "0.3.16"
+escaper = "0.1"
+futures = "0.3"
 hex = "0.4.0"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-indexmap = "1.7.0"
-itertools = "0.10.1"
+indexmap = "1.7"
+itertools = "0.10"
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2.98"
+libc = "0.2"
 log = {version = "0.4.8", optional = true }
-mailparse = "0.13.5"
-native-tls = "0.2.3"
-num_cpus = "1.13.0"
-num-derive = "0.3.0"
-num-traits = "0.2.6"
+mailparse = "0.13"
+native-tls = "0.2"
+num_cpus = "1.13"
+num-derive = "0.3"
+num-traits = "0.2"
 once_cell = "1.8.0"
 percent-encoding = "2.0"
-pgp = { version = "0.7.0", default-features = false }
-pretty_env_logger = { version = "0.4.0", optional = true }
-quick-xml = "0.22.0"
-r2d2 = "0.8.9"
-r2d2_sqlite = "0.18.0"
-rand = "0.7.0"
-regex = "1.4.6"
+pgp = { version = "0.7", default-features = false }
+pretty_env_logger = { version = "0.4", optional = true }
+quick-xml = "0.22"
+r2d2 = "0.8"
+r2d2_sqlite = "0.18"
+rand = "0.7"
+regex = "1.5"
 rusqlite = "0.25"
-rust-hsluv = "0.1.4"
-rustyline = { version = "8.2.0", optional = true }
-sanitize-filename = "0.3.0"
+rust-hsluv = "0.1"
+rustyline = { version = "9.0", optional = true }
+sanitize-filename = "0.3"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9.7"
-sha2 = "0.9.5"
-smallvec = "1.0.0"
-stop-token = "0.2.0"
-strum = "0.21.0"
-strum_macros = "0.21.1"
-surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
-thiserror = "1.0.26"
-toml = "0.5.6"
-url = "2.2.2"
+sha-1 = "0.9"
+sha2 = "0.9"
+smallvec = "1"
+stop-token = "0.6"
+strum = "0.22"
+strum_macros = "0.22"
+surf = { version = "2.3", default-features = false, features = ["h1-client"] }
+thiserror = "1"
+toml = "0.5"
+url = "2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
-fast-socks5 = "0.4.2"
-humansize = "1.1.1"
+fast-socks5 = "0.4"
+humansize = "1"
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1.9.0", features = ["unstable", "attributes"] }
+async-std = { version = "1", features = ["unstable", "attributes"] }
 criterion = "0.3"
-futures-lite = "1.12.0"
-log = "0.4.11"
-pretty_assertions = "0.7.2"
-pretty_env_logger = "0.4.0"
-proptest = "1.0"
-tempfile = "3.0"
+futures-lite = "1.12"
+log = "0.4"
+pretty_assertions = "1.0"
+pretty_env_logger = "0.4"
+proptest = "1"
+tempfile = "3"
 
 [workspace]
 members = [

assets/icon-broadcast.svg

@@ -0,0 +1,149 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   enable-background="new 0 0 128 128"
+   viewBox="0 0 60 60"
+   version="1.1"
+   id="svg878"
+   sodipodi:docname="icon-broadcast.svg"
+   width="60"
+   height="60"
+   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
+   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-broadcast.png"
+   inkscape:export-xdpi="409.60001"
+   inkscape:export-ydpi="409.60001">
+  <metadata
+     id="metadata884">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs882" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1329"
+     inkscape:window-height="847"
+     id="namedview880"
+     showgrid="false"
+     inkscape:zoom="5.21875"
+     inkscape:cx="36.598802"
+     inkscape:cy="32.191617"
+     inkscape:window-x="111"
+     inkscape:window-y="205"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg878"
+     inkscape:document-rotation="0" />
+  <radialGradient
+     id="c"
+     cx="65.25"
+     cy="89"
+     r="26.440001"
+     gradientTransform="matrix(0.77611266,0.11996647,-0.18999676,1.2286617,-11.305867,-60.065999)"
+     gradientUnits="userSpaceOnUse">
+    <stop
+       stop-color="#FFC107"
+       offset="0"
+       id="stop833" />
+    <stop
+       stop-color="#FFBD06"
+       offset=".3502"
+       id="stop835" />
+    <stop
+       stop-color="#FFB104"
+       offset=".6938"
+       id="stop837" />
+    <stop
+       stop-color="#FFA000"
+       offset="1"
+       id="stop839" />
+  </radialGradient>
+  <radialGradient
+     id="b"
+     cx="52.5"
+     cy="19.75"
+     r="92.975998"
+     gradientUnits="userSpaceOnUse"
+     gradientTransform="rotate(45.323856,68.997115,75.979538)">
+    <stop
+       stop-color="#EF5350"
+       offset="0"
+       id="stop848" />
+    <stop
+       stop-color="#EB4F4C"
+       offset=".246"
+       id="stop850" />
+    <stop
+       stop-color="#E04341"
+       offset=".4878"
+       id="stop852" />
+    <stop
+       stop-color="#CD302F"
+       offset=".7272"
+       id="stop854" />
+    <stop
+       stop-color="#C62828"
+       offset=".8004"
+       id="stop856" />
+    <stop
+       stop-color="#C62828"
+       offset="1"
+       id="stop858" />
+  </radialGradient>
+  <radialGradient
+     id="a"
+     cx="16.979"
+     cy="92"
+     r="24.165001"
+     gradientUnits="userSpaceOnUse"
+     gradientTransform="rotate(45.323856,68.997115,75.979538)"
+     xlink:href="#b">
+    <stop
+       stop-color="#E0E0E0"
+       offset="0"
+       id="stop863" />
+    <stop
+       stop-color="#CFCFCF"
+       offset=".3112"
+       id="stop865" />
+    <stop
+       stop-color="#A4A4A4"
+       offset=".9228"
+       id="stop867" />
+    <stop
+       stop-color="#9E9E9E"
+       offset="1"
+       id="stop869" />
+  </radialGradient>
+  <rect
+     y="0"
+     x="0"
+     height="60"
+     width="60"
+     id="rect1420"
+     style="fill:#7cc0bc;fill-opacity:1;stroke:none;stroke-width:1.29077" />
+  <path
+     id="path872"
+     style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.336872;stroke-opacity:1"
+     d="m 8.6780027,35.573064 0.032831,-11.910176 c 0.00138,-0.476406 0.4881282,-0.794259 0.9235226,-0.604877 l 4.1144877,2.345752 -0.02386,8.656315 -4.1268029,2.122946 C 9.1617452,36.370003 8.6766889,36.049472 8.6780027,35.573064 Z m 5.0469633,-1.508222 0.02386,-8.656314 31.145424,-9.537653 c 0.841472,-0.219211 1.65915,0.41667 1.656755,1.283728 l -0.06929,25.139995 c -0.0024,0.867062 -0.825942,1.500799 -1.663803,1.274581 z m 3.8042,6.892234 C 16.681121,40.104348 16.315444,38.819414 16.69043,37.591308 l 2.252234,-7.347193 c 0.2644,-0.861571 0.845185,-1.567441 1.641953,-1.989251 0.796769,-0.421808 1.706956,-0.509819 2.568531,-0.245419 l 7.263888,2.225804 c 1.775518,0.543235 2.780299,2.432591 2.232297,4.208094 L 30.3971,41.790532 c -0.545627,1.777887 -2.432591,2.780297 -4.208095,2.232298 l -7.263891,-2.225804 c -0.545033,-0.165864 -1.01825,-0.460162 -1.395948,-0.83995 z m 12.377693,-7.976728 c -0.07601,-0.07642 -0.17114,-0.133864 -0.280621,-0.167516 l -7.263891,-2.225803 c -0.233244,-0.07209 -0.421626,0.0013 -0.512275,0.04861 -0.09064,0.0474 -0.25772,0.166033 -0.327435,0.396899 l -2.252234,7.347191 c -0.108166,0.354628 0.09088,0.731541 0.447888,0.842099 l 7.263891,2.225802 c 0.354626,0.108174 0.731539,-0.09088 0.842099,-0.447888 l 2.249845,-7.344814 c 0.07453,-0.245145 0.0014,-0.504991 -0.167267,-0.67458 z" />
+</svg>

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.59.0"
+version = "1.64.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -17,13 +17,13 @@ crate-type = ["cdylib", "staticlib"]
 [dependencies]
 deltachat = { path = "../", default-features = false }
 libc = "0.2"
-human-panic = "1.0.1"
-num-traits = "0.2.6"
+human-panic = "1"
+num-traits = "0.2"
 serde_json = "1.0"
-async-std = "1.9.0"
-anyhow = "1.0.42"
-thiserror = "1.0.26"
-rand = "0.7.3"
+async-std = "1"
+anyhow = "1"
+thiserror = "1"
+rand = "0.7"
 
 [features]
 default = ["vendored"]

deltachat-ffi/Doxyfile

@@ -583,7 +583,7 @@ SORT_MEMBERS_CTORS_1ST = NO
 # appear in their defined order.
 # The default value is: NO.
 
-SORT_GROUP_NAMES       = NO
+SORT_GROUP_NAMES       = YES
 
 # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
 # fully-qualified names, including namespaces. If set to NO, the class list will

deltachat-ffi/Doxyfile.css

@@ -4,4 +4,16 @@ div.fragment {
 	background-color: #e0e0e0;
 	border: 0;
 	padding: 1em;
+	border-radius: 6px;
 }
+
+code {
+	background-color: #e0e0e0;
+	padding-left: .5em;
+	padding-right: .5em;
+	border-radius: 6px;
+}
+
+li {
+	margin-bottom: .5em;
+}

deltachat-ffi/deltachat.h

@@ -351,6 +351,19 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
  *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
+ * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
+ *                    For larger messages, only the header is downloaded and a placeholder is shown.
+ *                    These messages can be downloaded fully using dc_download_full_msg() later.
+ *                    The limit is compared against raw message sizes, including headers.
+ *                    The actually used limit may be corrected
+ *                    to not mess up with non-delivery-reports or read-receipts.
+ *                    0=no limit (default).
+ *                    Changes affect future messages only.
+ * - `ui.*`         = All keys prefixed by `ui.` can be used by the user-interfaces for system-specific purposes.
+ *                    The prefix should be followed by the system and maybe subsystem,
+ *                    eg. `ui.desktop.foo`, `ui.desktop.linux.bar`, `ui.android.foo`, `ui.dc40.bar`, `ui.bot.simplebot.baz`.
+ *                    These keys go to backups and allow easy per-account settings when using @ref dc_accounts_t,
+ *                    however, are not handled by the core otherwise.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -377,6 +390,9 @@ int             dc_set_config                (dc_context_t* context, const char*
  *                    an error (no warning as it should be shown to the user) is logged but the attachment is sent anyway.
  * - `sys.config_keys` = get a space-separated list of all config-keys available.
  *                    The config-keys are the keys that can be passed to the parameter `key` of this function.
+ * - `quota_exceeding` = 0: quota is unknown or in normal range;
+ *                    >=80: quota is about to exceed, the value is the concrete percentage,
+ *                    a device message is added when that happens, however, that value may still be interesting for bots.
  *
  * @memberof dc_context_t
  * @param context The context object. For querying system values, this can be NULL.
@@ -583,6 +599,14 @@ void            dc_configure                 (dc_context_t* context);
  * Typically, for unconfigured accounts, the user is prompted
  * to enter some settings and dc_configure() is called in a thread then.
  *
+ * A once successfully configured context cannot become unconfigured again;
+ * if a subsequent call to dc_configure() fails,
+ * the prior configuration is used.
+ *
+ * However, of course, also a configuration may stop working,
+ * as eg. the password was changed on the server.
+ * To check that use eg. dc_get_connectivity().
+ *
  * @memberof dc_context_t
  * @param context The context object.
  * @return 1=context is configured and can be used;
@@ -904,7 +928,7 @@ uint32_t        dc_send_text_msg             (dc_context_t* context, uint32_t ch
  * dc_send_videochat_invitation() is blocking and may take a while,
  * so the UIs will typically call the function from within a thread.
  * Moreover, UIs will typically enter the room directly without an additional click on the message,
- * for this purpose, the function returns the message-id directly.
+ * for this purpose, the function returns the message id directly.
  *
  * As for other messages sent, this function
  * sends the event #DC_EVENT_MSGS_CHANGED on success, the message has a delivery state, and so on.
@@ -1275,6 +1299,8 @@ void            dc_accept_chat               (dc_context_t* context, uint32_t ch
  *   explicitly as it may happen that oneself gets removed from a still existing
  *   group
  *
+ * - for broadcasts, all recipients are returned, DC_CONTACT_ID_SELF is not included
+ *
  * - for mailing lists, the behavior is not documented currently, we will decide on that later.
  *   for now, the UI should not show the list for mailing lists.
  *   (we do not know all members and there is not always a global mailing list address,
@@ -1384,6 +1410,36 @@ dc_chat_t*      dc_get_chat                  (dc_context_t* context, uint32_t ch
 uint32_t        dc_create_group_chat         (dc_context_t* context, int protect, const char* name);
 
 
+/**
+ * Create a new broadcast list.
+ *
+ * Broadcast lists are similar to groups on the sending device,
+ * however, recipients get the messages in normal one-to-one chats
+ * and will not be aware of other members.
+ *
+ * Replies to broadcasts go only to the sender
+ * and not to all broadcast recipients.
+ * Moreover, replies will not appear in the broadcast list
+ * but in the one-to-one chat with the person answering.
+ *
+ * The name and the image of the broadcast list is set automatically
+ * and is visible to the sender only.
+ * Not asking for these data allows more focused creation
+ * and we bypass the question who will get which data.
+ * Also, many users will have at most one broadcast list
+ * so, a generic name and image is sufficient at the first place.
+ *
+ * Later on, however, the name can be changed using dc_set_chat_name().
+ * The image cannot be changed to have a unique, recognizable icon in the chat lists.
+ * All in all, this is also what other messengers are doing here.
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @return The chat ID of the new broadcast list, 0 on errors.
+ */
+uint32_t        dc_create_broadcast_list     (dc_context_t* context);
+
+
 /**
  * Check if a given contact ID is a member of a group chat.
  *
@@ -1581,6 +1637,28 @@ char*           dc_get_msg_info              (dc_context_t* context, uint32_t ms
 char*           dc_get_msg_html              (dc_context_t* context, uint32_t msg_id);
 
 
+/**
+  * Asks the core to start downloading a message fully.
+  * This function is typically called when the user hits the "Download" button
+  * that is shown by the UI in case dc_msg_get_download_state()
+  * returns @ref DC_DOWNLOAD_AVAILABLE or @ref DC_DOWNLOAD_FAILURE.
+  *
+  * On success, the @ref DC_MSG "view type of the message" may change
+  * or the message may be replaced completely by one or more messages with other message ids.
+  * That may happen eg. in cases where the message was encrypted
+  * and the type could not be determined without fully downloading.
+  * Downloaded content can be accessed as usual after download,
+  * eg. using dc_msg_get_file().
+  *
+  * To reflect these changes a @ref DC_EVENT_MSGS_CHANGED event will be emitted.
+  *
+  * @memberof dc_context_t
+  * @param context The context object.
+  * @param msg_id Message ID to download the content for.
+  */
+void dc_download_full_msg (dc_context_t* context, int msg_id);
+
+
 /**
  * Get the raw mime-headers of the given message.
  * Raw headers are saved for incoming messages
@@ -1631,7 +1709,7 @@ void            dc_forward_msgs              (dc_context_t* context, const uint3
  *
  * - For normal chats, the IMAP state is updated, MDN is sent
  *   (if dc_set_config()-options `mdns_enabled` is set)
- *   and the internal state is changed to DC_STATE_IN_SEEN to reflect these actions.
+ *   and the internal state is changed to @ref DC_STATE_IN_SEEN to reflect these actions.
  *
  * - For contact requests, no IMAP or MDNs is done
  *   and the internal state is not changed therefore.
@@ -2052,7 +2130,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_FPR_MISMATCH           220 // id=contact
 #define         DC_QR_FPR_WITHOUT_ADDR       230 // test1=formatted fingerprint
 #define         DC_QR_ACCOUNT                250 // text1=domain
-#define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain
+#define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain, text2=instance pattern
 #define         DC_QR_ADDR                   320 // id=contact
 #define         DC_QR_TEXT                   330 // text1=text
 #define         DC_QR_URL                    332 // text1=URL
@@ -2171,21 +2249,12 @@ char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t ch
  * This function is typically called when dc_check_qr() returns
  * lot.state=DC_QR_ASK_VERIFYCONTACT or lot.state=DC_QR_ASK_VERIFYGROUP.
  *
- * Depending on the given QR code,
- * this function may takes some time and sends and receives several messages.
- * Therefore, you should call it always in a separate thread;
- * if you want to abort it, you should call dc_stop_ongoing_process().
+ * The function returns immediately and the handshake runs in background,
+ * sending and receiving several messages.
+ * During the handshake, info messages are added to the chat,
+ * showing progress, success or errors.
  *
- * - If the given QR code starts the Setup-Contact protocol,
- *   the function typically returns immediately
- *   and the handshake runs in background.
- *   Subsequent calls of dc_join_securejoin() will abort unfinished tasks.
- *   The returned chat is the one-to-one opportunistic chat.
- *   When the protocol has finished, an info-message is added to that chat.
- * - If the given QR code starts the Verified-Group-Invite protocol,
- *   the function waits until the protocol has finished.
- *   This is because the protected group is not opportunistic
- *   and can be created only when the contacts have verified each other.
+ * Subsequent calls of dc_join_securejoin() will abort previous, unfinished handshakes.
  *
  * See https://countermitm.readthedocs.io/en/latest/new.html
  * for details about both protocols.
@@ -2195,10 +2264,8 @@ char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t ch
  * @param qr The text of the scanned QR code. Typically, the same string as given
  *     to dc_check_qr().
  * @return Chat-id of the joined chat, the UI may redirect to the this chat.
- *     If the out-of-band verification failed or was aborted, 0 is returned.
+ *     On errors, 0 is returned, however, most errors will happen during handshake later on.
  *     A returned chat-id does not guarantee that the chat is protected or the belonging contact is verified.
- *     If needed, this be checked with dc_chat_is_protected() and dc_contact_is_verified(),
- *     however, in practise, the UI will just listen to #DC_EVENT_CONTACTS_CHANGED unconditionally.
  */
 uint32_t        dc_join_securejoin           (dc_context_t* context, const char* qr);
 
@@ -2347,6 +2414,22 @@ dc_array_t* dc_get_locations                (dc_context_t* context, uint32_t cha
 void        dc_delete_all_locations         (dc_context_t* context);
 
 
+/**
+ * Get last error string.
+ *
+ * This is the same error string as logged via #DC_EVENT_ERROR,
+ * however, using this function avoids race conditions
+ * if the failing function is called in another thread than dc_get_next_event().
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @return Last error or an empty string if there is no last error.
+ *     NULL is never returned.
+ *     The returned value must be released using dc_str_unref() after usage.
+ */
+char* dc_get_last_error (dc_context_t* context);
+
+
 /**
  * Release a string returned by another deltachat-core function.
  * - Strings returned by any deltachat-core-function
@@ -2728,13 +2811,13 @@ uint32_t         dc_array_get_contact_id     (const dc_array_t* array, size_t in
 
 
 /**
- * Return the message-id of the item at the given index.
+ * Return the message id of the item at the given index.
  *
  * @memberof dc_array_t
  * @param array The array object.
  * @param index Index of the item. Must be between 0 and dc_array_get_cnt()-1.
- * @return Message-id of the item at the given index.
- *     0 if there is no message-id bound to the given item,
+ * @return Message id of the item at the given index.
+ *     0 if there is no message id bound to the given item,
  */
 uint32_t         dc_array_get_msg_id         (const dc_array_t* array, size_t index);
 
@@ -2880,7 +2963,7 @@ uint32_t         dc_chatlist_get_msg_id      (const dc_chatlist_t* chatlist, siz
  *
  * - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
  *
- * - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).  0 if not applicable.
+ * - dc_lot_t::state: The state of the message as one of the @ref DC_STATE constants.  0 if not applicable.
  *
  * @memberof dc_chatlist_t
  * @param chatlist The chatlist to query as returned e.g. from dc_get_chatlist().
@@ -2896,7 +2979,7 @@ dc_lot_t*        dc_chatlist_get_summary     (const dc_chatlist_t* chatlist, siz
  * Create a chatlist summary item when the chatlist object is already unref()'d.
  *
  * This function is similar to dc_chatlist_get_summary(), however,
- * takes the chat-id and message-id as returned by dc_chatlist_get_chat_id() and dc_chatlist_get_msg_id()
+ * takes the chat-id and message id as returned by dc_chatlist_get_chat_id() and dc_chatlist_get_msg_id()
  * as arguments. The chatlist object itself is not needed directly.
  *
  * This maybe useful if you convert the complete object into a different represenation
@@ -2934,7 +3017,7 @@ dc_context_t*    dc_chatlist_get_context     (dc_chatlist_t* chatlist);
  * color: color of this chat
  * last-message-from: who sent the last message
  * last-message-text: message (truncated)
- * last-message-state: DC_STATE* constant
+ * last-message-state: @ref DC_STATE constant
  * last-message-date:
  * avatar-path: path-to-blobfile
  * is_verified: yes/no
@@ -2958,12 +3041,6 @@ char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat
 #define         DC_CHAT_ID_LAST_SPECIAL      9 // larger chat IDs are "real" chats, their messages are "real" messages.
 
 
-#define         DC_CHAT_TYPE_UNDEFINED       0
-#define         DC_CHAT_TYPE_SINGLE          100
-#define         DC_CHAT_TYPE_GROUP           120
-#define         DC_CHAT_TYPE_MAILINGLIST     140
-
-
 /**
  * Free a chat object.
  *
@@ -2990,21 +3067,30 @@ uint32_t        dc_chat_get_id               (const dc_chat_t* chat);
 
 
 /**
- * Get chat type.
+ * Get chat type as one of the @ref DC_CHAT_TYPE constants:
  *
- * Currently, there are two chat types:
- *
- * - DC_CHAT_TYPE_SINGLE (100) - a normal chat is a chat with a single contact,
+ * - @ref DC_CHAT_TYPE_SINGLE - a normal chat is a chat with a single contact,
  *   chats_contacts contains one record for the user.  DC_CONTACT_ID_SELF
  *   (see dc_contact_t::id) is added _only_ for a self talk.
+ *   These chats are created by dc_create_chat_by_contact_id().
  *
- * - DC_CHAT_TYPE_GROUP  (120) - a group chat, chats_contacts contain all group
- *   members, incl. DC_CONTACT_ID_SELF
+ * - @ref DC_CHAT_TYPE_GROUP - a group chat, chats_contacts contain all group
+ *   members, incl. DC_CONTACT_ID_SELF.
+ *   Groups are created by dc_create_group_chat().
  *
- * - DC_CHAT_TYPE_MAILINGLIST (140) - a mailing list, this is similar to groups,
+ * - @ref DC_CHAT_TYPE_MAILINGLIST - a mailing list, this is similar to groups,
  *   however, the member list cannot be retrieved completely
  *   and cannot be changed using this api.
- *   moreover, for now, mailist lists are read-only.
+ *   Mailing lists are created as needed by incoming messages
+ *   and usually require some special server;
+ *   they cannot be created by a function call as the other chat types.
+ *   Moreover, for now, mailing lists are read-only.
+ *
+ * - @ref DC_CHAT_TYPE_BROADCAST - a broadcast list,
+ *   the recipients will get messages in a one-to-one chats and
+ *   the sender will get answers in a one-to-one as well.
+ *   chats_contacts contain all recipients but DC_CONTACT_ID_SELF.
+ *   Broadcasts are created by dc_create_broadcast_list().
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -3213,18 +3299,6 @@ int64_t          dc_chat_get_remaining_mute_duration (const dc_chat_t* chat);
 #define         DC_MSG_ID_LAST_SPECIAL       9
 
 
-#define         DC_STATE_UNDEFINED           0
-#define         DC_STATE_IN_FRESH            10
-#define         DC_STATE_IN_NOTICED          13
-#define         DC_STATE_IN_SEEN             16
-#define         DC_STATE_OUT_PREPARING       18
-#define         DC_STATE_OUT_DRAFT           19
-#define         DC_STATE_OUT_PENDING         20
-#define         DC_STATE_OUT_FAILED          24
-#define         DC_STATE_OUT_DELIVERED       26 // to check if a mail was sent, use dc_msg_is_sent()
-#define         DC_STATE_OUT_MDN_RCVD        28
-
-
 /**
  * Create new message object. Message objects are needed e.g. for sending messages using
  * dc_send_msg().  Moreover, they are returned e.g. from dc_get_msg(),
@@ -3304,28 +3378,37 @@ int             dc_msg_get_viewtype           (const dc_msg_t* msg);
  * Get the state of a message.
  *
  * Incoming message states:
- * - DC_STATE_IN_FRESH (10) - Incoming _fresh_ message. Fresh messages are neither noticed nor seen and are typically shown in notifications. Use dc_get_fresh_msgs() to get all fresh messages.
- * - DC_STATE_IN_NOTICED (13) - Incoming _noticed_ message. E.g. chat opened but message not yet read - noticed messages are not counted as unread but were not marked as read nor resulted in MDNs. Use dc_marknoticed_chat() to mark messages as being noticed.
- * - DC_STATE_IN_SEEN (16) - Incoming message, really _seen_ by the user. Marked as read on IMAP and MDN may be sent. Use dc_markseen_msgs() to mark messages as being seen.
+ * - @ref DC_STATE_IN_FRESH - Incoming _fresh_ message.
+ *   Fresh messages are neither noticed nor seen and are typically shown in notifications.
+ *   Use dc_get_fresh_msgs() to get all fresh messages.
+ * - @ref DC_STATE_IN_NOTICED - Incoming _noticed_ message.
+ *   E.g. chat opened but message not yet read.
+ *   Noticed messages are not counted as unread but were not marked as read nor resulted in MDNs.
+ *   Use dc_marknoticed_chat() to mark messages as being noticed.
+ * - @ref DC_STATE_IN_SEEN - Incoming message, really _seen_ by the user.
+ *   Marked as read on IMAP and MDN may be sent. Use dc_markseen_msgs() to mark messages as being seen.
  *
  * Outgoing message states:
- * - DC_STATE_OUT_PREPARING (18) - For files which need time to be prepared before they can be sent,
- *   the message enters this state before DC_STATE_OUT_PENDING.
- * - DC_STATE_OUT_DRAFT (19) - Message saved as draft using dc_set_draft()
- * - DC_STATE_OUT_PENDING (20) - The user has pressed the "send" button but the
+ * - @ref DC_STATE_OUT_PREPARING - For files which need time to be prepared before they can be sent,
+ *   the message enters this state before @ref DC_STATE_OUT_PENDING.
+ * - @ref DC_STATE_OUT_DRAFT - Message saved as draft using dc_set_draft()
+ * - @ref DC_STATE_OUT_PENDING - The user has pressed the "send" button but the
  *   message is not yet sent and is pending in some way. Maybe we're offline (no checkmark).
- * - DC_STATE_OUT_FAILED (24) - _Unrecoverable_ error (_recoverable_ errors result in pending messages), you'll receive the event #DC_EVENT_MSG_FAILED.
- * - DC_STATE_OUT_DELIVERED (26) - Outgoing message successfully delivered to server (one checkmark). Note, that already delivered messages may get into the state DC_STATE_OUT_FAILED if we get such a hint from the server.
+ * - @ref DC_STATE_OUT_FAILED - _Unrecoverable_ error (_recoverable_ errors result in pending messages),
+ *   you'll receive the event #DC_EVENT_MSG_FAILED.
+ * - @ref DC_STATE_OUT_DELIVERED - Outgoing message successfully delivered to server (one checkmark).
+ *   Note, that already delivered messages may get into the state @ref DC_STATE_OUT_FAILED if we get such a hint from the server.
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_DELIVERED.
- * - DC_STATE_OUT_MDN_RCVD (28) - Outgoing message read by the recipient (two checkmarks; this requires goodwill on the receiver's side)
+ * - @ref DC_STATE_OUT_MDN_RCVD - Outgoing message read by the recipient
+ *   (two checkmarks; this requires goodwill on the receiver's side)
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_READ.
  *   Also messages already read by some recipients
- *   may get into the state DC_STATE_OUT_FAILED at a later point,
+ *   may get into the state @ref DC_STATE_OUT_FAILED at a later point,
  *   e.g. when in a group, delivery fails for some recipients.
  *
  * If you just want to check if a message is sent or not, please use dc_msg_is_sent() which regards all states accordingly.
  *
- * The state of just created message objects is DC_STATE_UNDEFINED (0).
+ * The state of just created message objects is @ref DC_STATE_UNDEFINED.
  * The state is always set by the core-library, users of the library cannot set the state directly, but it is changed implicitly e.g.
  * when calling  dc_marknoticed_chat() or dc_markseen_msgs().
  *
@@ -3589,7 +3672,7 @@ int64_t          dc_msg_get_ephemeral_timestamp (const dc_msg_t* msg);
  *   Typically used to show dc_lot_t::text1 with different colors. 0 if not applicable.
  * - dc_lot_t::text2: contains an excerpt of the message text.
  * - dc_lot_t::timestamp: the timestamp of the message.
- * - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
+ * - dc_lot_t::state: The state of the message as one of the @ref DC_STATE constants.
  *
  * Typically used to display a search result. See also dc_chatlist_get_summary() to display a list of chats.
  *
@@ -3903,6 +3986,31 @@ int dc_msg_get_videochat_type (const dc_msg_t* msg);
 int dc_msg_has_html (dc_msg_t* msg);
 
 
+/**
+  * Check if the message is completely downloaded
+  * or if some further action is needed.
+  *
+  * Messages may be not fully downloaded
+  * if they are larger than the limit set by the dc_set_config()-option `download_limit`.
+  *
+  * The function returns one of:
+  * - @ref DC_DOWNLOAD_DONE        - The message does not need any further download action
+  *                                  and should be rendered as usual.
+  * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
+  *                                  In addition to the usual message rendering,
+  *                                  the UI shall show a download button that calls dc_download_full_msg()
+  * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_download_full_msg() and is still in progress.
+  *                                  If the download fails or succeeds,
+  *                                  the event @ref DC_EVENT_MSGS_CHANGED is emitted.
+  * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_download_full_msg() again.
+  *
+  * @memberof dc_msg_t
+  * @param msg The message object.
+  * @return One of the @ref DC_DOWNLOAD values
+  */
+int dc_msg_get_download_state (const dc_msg_t* msg);
+
+
 /**
  * Set the text of a message object.
  * This does not alter any information in the database; this may be done by dc_send_msg() later.
@@ -4585,6 +4693,115 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 
+/**
+  * @defgroup DC_STATE DC_STATE
+  *
+  * These constants describe the state of a message.
+  * The state can be retrieved using dc_msg_get_state()
+  * and may change by various actions reported by various events
+  *
+  * @addtogroup DC_STATE
+  * @{
+  */
+
+/**
+ * Message just created. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_UNDEFINED           0
+
+/**
+ * Incoming fresh message. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_IN_FRESH            10
+
+/**
+ * Incoming noticed message. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_IN_NOTICED          13
+
+/**
+ * Incoming seen message. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_IN_SEEN             16
+
+/**
+ * Outgoing message being prepared. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_PREPARING       18
+
+/**
+ * Outgoing message drafted. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_DRAFT           19
+
+/**
+ * Outgoing message waiting to be sent. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_PENDING         20
+
+/**
+ * Outgoing message failed sending. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_FAILED          24
+
+/**
+ * Outgoing message sent. To check if a mail was actually sent, use dc_msg_is_sent().
+ * See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_DELIVERED       26
+
+/**
+ * Outgoing message sent and seen by recipients(s). See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_MDN_RCVD        28
+
+/**
+ * @}
+ */
+
+
+/**
+ * @defgroup DC_CHAT_TYPE DC_CHAT_TYPE
+ *
+ * These constants describe the type of a chat.
+ * The chat type can be retrieved using dc_chat_get_type()
+ * and the type does not change during the chat's lifetime.
+ *
+ * @addtogroup DC_CHAT_TYPE
+ * @{
+ */
+
+/**
+ * Undefined chat type.
+ * Normally, this type is not returned.
+ */
+#define         DC_CHAT_TYPE_UNDEFINED       0
+
+/**
+ * A one-to-one chat with a single contact. See dc_chat_get_type() for details.
+ */
+#define         DC_CHAT_TYPE_SINGLE          100
+
+/**
+ * A group chat. See dc_chat_get_type() for details.
+ */
+#define         DC_CHAT_TYPE_GROUP           120
+
+/**
+ * A mailing list. See dc_chat_get_type() for details.
+ */
+#define         DC_CHAT_TYPE_MAILINGLIST     140
+
+/**
+ * A broadcast list. See dc_chat_get_type() for details.
+ */
+#define         DC_CHAT_TYPE_BROADCAST       160
+
+/**
+ * @}
+ */
+
+
 /**
  * @defgroup DC_SOCKET DC_SOCKET
  *
@@ -4829,7 +5046,7 @@ char* dc_event_get_data2_str(dc_event_t* event);
  *
  * @memberof dc_event_t
  * @param event Event object as returned from dc_accounts_get_next_event().
- * @return account-id belonging to the event or 0 for errors.
+ * @return account-id belonging to the event, 0 for account manager errors.
  */
 uint32_t dc_event_get_account_id(dc_event_t* event);
 
@@ -4977,8 +5194,8 @@ void dc_event_unref(dc_event_t* event);
  * - Chats created, deleted or archived
  * - A draft has been set
  *
- * @param data1 (int) chat_id for single added messages
- * @param data2 (int) msg_id for single added messages
+ * @param data1 (int) chat_id if only a single chat is affected by the changes, otherwise 0
+ * @param data2 (int) msg_id if only a single message is affected by the changes, otherwise 0
  */
 #define DC_EVENT_MSGS_CHANGED             2000
 
@@ -5011,8 +5228,8 @@ void dc_event_unref(dc_event_t* event);
 
 
 /**
- * A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
- * DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
+ * A single message is sent successfully. State changed from @ref DC_STATE_OUT_PENDING to
+ * @ref DC_STATE_OUT_DELIVERED.
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
@@ -5022,8 +5239,8 @@ void dc_event_unref(dc_event_t* event);
 
 /**
  * A single message could not be sent.
- * State changed from DC_STATE_OUT_PENDING, DC_STATE_OUT_DELIVERED or DC_STATE_OUT_MDN_RCVD
- * to DC_STATE_OUT_FAILED, see dc_msg_get_state().
+ * State changed from @ref DC_STATE_OUT_PENDING, @ref DC_STATE_OUT_DELIVERED or @ref DC_STATE_OUT_MDN_RCVD
+ * to @ref DC_STATE_OUT_FAILED.
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
@@ -5032,8 +5249,8 @@ void dc_event_unref(dc_event_t* event);
 
 
 /**
- * A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
- * DC_STATE_OUT_MDN_RCVD, see dc_msg_get_state().
+ * A single message is read by the receiver. State changed from @ref DC_STATE_OUT_DELIVERED to
+ * @ref DC_STATE_OUT_MDN_RCVD.
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
@@ -5054,6 +5271,9 @@ void dc_event_unref(dc_event_t* event);
 
 /**
  * Chat ephemeral timer changed.
+ *
+ * @param data1 (int) chat_id
+ * @param data2 (int) Timer value in seconds or 0 for disabled timer
  */
 #define DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED 2021
 
@@ -5153,6 +5373,14 @@ void dc_event_unref(dc_event_t* event);
  */
 #define DC_EVENT_CONNECTIVITY_CHANGED             2100
 
+
+/**
+ * The user's avatar changed.
+ * You can get the new avatar file with `dc_get_config(context, "selfavatar")`.
+ */
+#define DC_EVENT_SELFAVATAR_CHANGED               2110
+
+
 /**
  * @}
  */
@@ -5277,6 +5505,44 @@ void dc_event_unref(dc_event_t* event);
  */
 #define         DC_CHAT_VISIBILITY_PINNED      2
 
+/**
+ * @}
+ */
+
+
+/**
+  * @defgroup DC_DOWNLOAD DC_DOWNLOAD
+  *
+  * These constants describe the download state of a message.
+  * The download state can be retrieved using dc_msg_get_download_state()
+  * and usually changes after calling dc_download_full_msg().
+  *
+  * @addtogroup DC_DOWNLOAD
+  * @{
+  */
+
+/**
+ * Download not needed, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_DONE          0
+
+/**
+ * Download available, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_AVAILABLE    10
+
+/**
+ * Download failed, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_FAILURE      20
+
+/**
+ * Download in progress, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_IN_PROGRESS  1000
+
+
+
 /**
  * @}
  */
@@ -5675,6 +5941,138 @@ void dc_event_unref(dc_event_t* event);
 /// Used in message summary text for notifications and chatlist.
 #define DC_STR_FORWARDED                  97
 
+/// "Quota exceeding, already %1$s%% used."
+///
+/// Used as device message text.
+///
+/// `%1$s` will be replaced by the percentage used
+#define DC_STR_QUOTA_EXCEEDING_MSG_BODY   98
+
+/// "%1$s message"
+///
+/// Used as the message body when a message
+/// was not yet downloaded completely
+/// (dc_msg_get_download_state() is eg. @ref DC_DOWNLOAD_AVAILABLE).
+///
+/// `%1$s` will be replaced by human-readable size (eg. "1.2 MiB").
+#define DC_STR_PARTIAL_DOWNLOAD_MSG_BODY  99
+
+/// "Download maximum available until %1$s"
+///
+/// Appended after some separator to @ref DC_STR_PARTIAL_DOWNLOAD_MSG_BODY.
+///
+/// `%1$s` will be replaced by human-readable date and time.
+#define DC_STR_DOWNLOAD_AVAILABILITY      100
+
+/// "Multi Device Synchronization"
+///
+/// Used in subjects of outgoing sync messages.
+#define DC_STR_SYNC_MSG_SUBJECT           101
+
+/// "This message is used to synchronize data between your devices."
+///
+///
+/// Used as message text of outgoing sync messages.
+/// The text is visible in non-dc-muas or in outdated Delta Chat versions,
+/// the default text therefore adds the following hint:
+/// "If you see this message in Delta Chat,
+/// please update your Delta Chat apps on all devices."
+#define DC_STR_SYNC_MSG_BODY              102
+
+/// "Incoming Messages"
+///
+/// Used as a headline in the connectivity view.
+#define DC_STR_INCOMING_MESSAGES          103
+
+/// "Outgoing Messages"
+///
+/// Used as a headline in the connectivity view.
+#define DC_STR_OUTGOING_MESSAGES          104
+
+/// "Storage on %1$s"
+///
+/// Used as a headline in the connectivity view.
+///
+/// `%1$s` will be replaced by the domain of the configured email-address.
+#define DC_STR_STORAGE_ON_DOMAIN          105
+
+/// "One moment…"
+///
+/// Used in the connectivity view when some information are not yet there.
+#define DC_STR_ONE_MOMENT                 106
+
+/// "Connected"
+///
+/// Used as status in the connectivity view.
+#define DC_STR_CONNECTED                  107
+
+/// "Connecting…"
+///
+/// Used as status in the connectivity view.
+#define DC_STR_CONNTECTING                108
+
+/// "Updating…"
+///
+/// Used as status in the connectivity view.
+#define DC_STR_UPDATING                   109
+
+/// "Sending…"
+///
+/// Used as status in the connectivity view.
+#define DC_STR_SENDING                    110
+
+/// "Your last message was sent successfully."
+///
+/// Used as status in the connectivity view.
+#define DC_STR_LAST_MSG_SENT_SUCCESSFULLY 111
+
+/// "Error: %1$s"
+///
+/// Used as status in the connectivity view.
+///
+/// `%1$s` will be replaced by a possibly more detailed, typically english, error description.
+#define DC_STR_ERROR                      112
+
+/// "Not supported by your provider."
+///
+/// Used in the connectivity view.
+#define DC_STR_NOT_SUPPORTED_BY_PROVIDER  113
+
+/// "Messages"
+///
+/// Used as a subtitle in quota context; can be plural always.
+#define DC_STR_MESSAGES                   114
+
+/// "Broadcast List"
+///
+/// Used as the default name for broadcast lists; a number may be added.
+#define DC_STR_BROADCAST_LIST             115
+
+/// "%1$s of %2$s used"
+///
+/// Used for describing resource usage, resulting string will be eg. "1.2 GiB of 3 GiB used".
+#define DC_STR_PART_OF_TOTAL_USED         116
+
+/// "%1$s invited you to join this group. Waiting for the device of %2$s to reply…"
+///
+/// Added as an info-message directly after scanning a QR code for joining a group.
+/// May be followed by the info-messages
+/// #DC_STR_SECURE_JOIN_REPLIES, #DC_STR_CONTACT_VERIFIED and #DC_STR_MSGADDMEMBER.
+///
+/// `%1$s` will be replaced by name and address of the inviter,
+/// `%2$s` will be replaced by the name of the inviter.
+#define DC_STR_SECURE_JOIN_STARTED        117
+
+/// "%1$s replied, waiting for being added to the group…"
+///
+/// Info-message on scanning a QR code for joining a group.
+/// Added after #DC_STR_SECURE_JOIN_STARTED.
+/// If the handshake allows to skip a step and go for #DC_STR_CONTACT_VERIFIED directly,
+/// this info-message is skipped.
+///
+/// `%1$s` will be replaced by the name of the inviter.
+#define DC_STR_SECURE_JOIN_REPLIES        118
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -15,13 +15,15 @@ extern crate num_traits;
 extern crate serde_json;
 
 use std::collections::BTreeMap;
-use std::convert::TryInto;
+use std::convert::TryFrom;
 use std::fmt::Write;
+use std::ops::Deref;
 use std::ptr;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use anyhow::Context as _;
+use async_std::sync::RwLock;
 use async_std::task::{block_on, spawn};
 use num_traits::{FromPrimitive, ToPrimitive};
 
@@ -37,6 +39,7 @@ use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
 
 mod dc_array;
+mod lot;
 
 mod string;
 use self::string::*;
@@ -132,20 +135,30 @@ pub unsafe extern "C" fn dc_set_config(
     }
     let ctx = &*context;
     let key = to_string_lossy(key);
-    match config::Config::from_str(&key) {
-        Ok(key) => block_on(async move {
-            let value = to_opt_string_lossy(value);
-            ctx.set_config(key, value.as_deref())
+    let value = to_opt_string_lossy(value);
+
+    block_on(async move {
+        if key.starts_with("ui.") {
+            ctx.set_ui_config(&key, value.as_deref())
                 .await
                 .with_context(|| format!("Can't set {} to {:?}", key, value))
                 .log_err(ctx, "dc_set_config() failed")
                 .is_ok() as libc::c_int
-        }),
-        Err(_) => {
-            warn!(ctx, "dc_set_config(): invalid key");
-            0
+        } else {
+            match config::Config::from_str(&key) {
+                Ok(key) => ctx
+                    .set_config(key, value.as_deref())
+                    .await
+                    .with_context(|| format!("Can't set {} to {:?}", key, value))
+                    .log_err(ctx, "dc_set_config() failed")
+                    .is_ok() as libc::c_int,
+                Err(_) => {
+                    warn!(ctx, "dc_set_config(): invalid key");
+                    0
+                }
+            }
         }
-    }
+    })
 }
 
 #[no_mangle]
@@ -158,20 +171,33 @@ pub unsafe extern "C" fn dc_get_config(
         return "".strdup();
     }
     let ctx = &*context;
-    match config::Config::from_str(&to_string_lossy(key)) {
-        Ok(key) => block_on(async move {
-            ctx.get_config(key)
+
+    let key = to_string_lossy(key);
+
+    block_on(async move {
+        if key.starts_with("ui.") {
+            ctx.get_ui_config(&key)
                 .await
-                .log_err(ctx, "Can't get config")
+                .log_err(ctx, "Can't get ui-config")
                 .unwrap_or_default()
                 .unwrap_or_default()
                 .strdup()
-        }),
-        Err(_) => {
-            warn!(ctx, "dc_get_config(): invalid key");
-            "".strdup()
+        } else {
+            match config::Config::from_str(&key) {
+                Ok(key) => ctx
+                    .get_config(key)
+                    .await
+                    .log_err(ctx, "Can't get config")
+                    .unwrap_or_default()
+                    .unwrap_or_default()
+                    .strdup(),
+                Err(_) => {
+                    warn!(ctx, "dc_get_config(): invalid key");
+                    "".strdup()
+                }
+            }
         }
-    }
+    })
 }
 
 #[no_mangle]
@@ -412,6 +438,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::Warning(_)
         | EventType::Error(_)
         | EventType::ConnectivityChanged
+        | EventType::SelfavatarChanged
         | EventType::ErrorSelfNotInGroup(_) => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -462,6 +489,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ImexFileWritten(_)
         | EventType::MsgsNoticed(_)
         | EventType::ConnectivityChanged
+        | EventType::SelfavatarChanged
         | EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
@@ -511,6 +539,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SecurejoinInviterProgress { .. }
         | EventType::SecurejoinJoinerProgress { .. }
         | EventType::ConnectivityChanged
+        | EventType::SelfavatarChanged
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -1305,6 +1334,19 @@ pub unsafe extern "C" fn dc_create_group_chat(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_create_broadcast_list(context: *mut dc_context_t) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_create_broadcast_list()");
+        return 0;
+    }
+    let ctx = &*context;
+    block_on(chat::create_broadcast_list(ctx))
+        .log_err(ctx, "Failed to create broadcast list")
+        .map(|id| id.to_u32())
+        .unwrap_or(0)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_is_contact_in_chat(
     context: *mut dc_context_t,
@@ -1317,8 +1359,13 @@ pub unsafe extern "C" fn dc_is_contact_in_chat(
     }
     let ctx = &*context;
 
-    block_on(async move { chat::is_contact_in_chat(ctx, ChatId::new(chat_id), contact_id).await })
-        .into()
+    block_on(chat::is_contact_in_chat(
+        ctx,
+        ChatId::new(chat_id),
+        contact_id,
+    ))
+    .log_err(ctx, "is_contact_in_chat failed")
+    .unwrap_or_default() as libc::c_int
 }
 
 #[no_mangle]
@@ -1333,9 +1380,13 @@ pub unsafe extern "C" fn dc_add_contact_to_chat(
     }
     let ctx = &*context;
 
-    block_on(async move {
-        chat::add_contact_to_chat(ctx, ChatId::new(chat_id), contact_id).await as libc::c_int
-    })
+    block_on(chat::add_contact_to_chat(
+        ctx,
+        ChatId::new(chat_id),
+        contact_id,
+    ))
+    .log_err(ctx, "Failed to add contact")
+    .is_ok() as libc::c_int
 }
 
 #[no_mangle]
@@ -1350,12 +1401,13 @@ pub unsafe extern "C" fn dc_remove_contact_from_chat(
     }
     let ctx = &*context;
 
-    block_on(async move {
-        chat::remove_contact_from_chat(ctx, ChatId::new(chat_id), contact_id)
-            .await
-            .map(|_| 1)
-            .unwrap_or_log_default(ctx, "Failed to remove contact")
-    })
+    block_on(chat::remove_contact_from_chat(
+        ctx,
+        ChatId::new(chat_id),
+        contact_id,
+    ))
+    .log_err(ctx, "Failed to remove contact")
+    .is_ok() as libc::c_int
 }
 
 #[no_mangle]
@@ -1570,6 +1622,8 @@ pub unsafe extern "C" fn dc_delete_msgs(
     let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
 
     block_on(message::delete_msgs(ctx, &msg_ids))
+        .log_err(ctx, "failed dc_delete_msgs() call")
+        .ok();
 }
 
 #[no_mangle]
@@ -1648,6 +1702,18 @@ pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) ->
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_download_full_msg(context: *mut dc_context_t, msg_id: u32) {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_download_full_msg()");
+        return;
+    }
+    let ctx = &*context;
+    block_on(MsgId::new(msg_id).download_full(ctx))
+        .log_err(ctx, "Failed to download message fully.")
+        .ok();
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_may_be_valid_addr(addr: *const libc::c_char) -> libc::c_int {
     if addr.is_null() {
@@ -1981,10 +2047,11 @@ pub unsafe extern "C" fn dc_check_qr(
     }
     let ctx = &*context;
 
-    block_on(async move {
-        let lot = qr::check_qr(ctx, &to_string_lossy(qr)).await;
-        Box::into_raw(Box::new(lot))
-    })
+    let lot = match block_on(qr::check_qr(ctx, &to_string_lossy(qr))) {
+        Ok(qr) => qr.into(),
+        Err(err) => err.into(),
+    };
+    Box::into_raw(Box::new(lot))
 }
 
 #[no_mangle]
@@ -2003,12 +2070,9 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
         Some(ChatId::new(chat_id))
     };
 
-    block_on(async move {
-        securejoin::dc_get_securejoin_qr(ctx, chat_id)
-            .await
-            .unwrap_or_else(|| "".to_string())
-            .strdup()
-    })
+    block_on(securejoin::dc_get_securejoin_qr(ctx, chat_id))
+        .unwrap_or_else(|_| "".to_string())
+        .strdup()
 }
 
 #[no_mangle]
@@ -2047,7 +2111,9 @@ pub unsafe extern "C" fn dc_send_locations_to_chat(
         ctx,
         ChatId::new(chat_id),
         seconds as i64,
-    ));
+    ))
+    .log_err(ctx, "Failed dc_send_locations_to_chat()")
+    .ok();
 }
 
 #[no_mangle]
@@ -2066,7 +2132,8 @@ pub unsafe extern "C" fn dc_is_sending_locations_to_chat(
         Some(ChatId::new(chat_id))
     };
 
-    block_on(location::is_sending_locations_to_chat(ctx, chat_id)) as libc::c_int
+    block_on(location::is_sending_locations_to_chat(ctx, chat_id))
+        .unwrap_or_log_default(ctx, "Failed dc_is_sending_locations_to_chat()") as libc::c_int
 }
 
 #[no_mangle]
@@ -2139,6 +2206,16 @@ pub unsafe extern "C" fn dc_delete_all_locations(context: *mut dc_context_t) {
     });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_last_error(context: *mut dc_context_t) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_last_error()");
+        return "".strdup();
+    }
+    let ctx = &*context;
+    block_on(ctx.get_last_error()).strdup()
+}
+
 // dc_array_t
 
 pub type dc_array_t = dc_array::dc_array_t;
@@ -2397,13 +2474,13 @@ pub unsafe extern "C" fn dc_chatlist_get_summary(
     let ctx = &*ffi_list.context;
 
     block_on(async move {
-        let lot = ffi_list
+        let summary = ffi_list
             .list
             .get_summary(ctx, index as usize, maybe_chat)
             .await
             .log_err(ctx, "get_summary failed")
             .unwrap_or_default();
-        Box::into_raw(Box::new(lot))
+        Box::into_raw(Box::new(summary.into()))
     })
 }
 
@@ -2423,13 +2500,15 @@ pub unsafe extern "C" fn dc_chatlist_get_summary2(
     } else {
         Some(MsgId::new(msg_id))
     };
-    block_on(async move {
-        let lot = Chatlist::get_summary2(ctx, ChatId::new(chat_id), msg_id, None)
-            .await
-            .log_err(ctx, "get_summary2 failed")
-            .unwrap_or_default();
-        Box::into_raw(Box::new(lot))
-    })
+    let summary = block_on(Chatlist::get_summary2(
+        ctx,
+        ChatId::new(chat_id),
+        msg_id,
+        None,
+    ))
+    .log_err(ctx, "get_summary2 failed")
+    .unwrap_or_default();
+    Box::into_raw(Box::new(summary.into()))
 }
 
 #[no_mangle]
@@ -2594,8 +2673,10 @@ pub unsafe extern "C" fn dc_chat_can_send(chat: *mut dc_chat_t) -> libc::c_int {
         return 0;
     }
     let ffi_chat = &*chat;
-    let cxt = &*ffi_chat.context;
-    block_on(ffi_chat.chat.can_send(cxt)) as libc::c_int
+    let ctx = &*ffi_chat.context;
+    block_on(ffi_chat.chat.can_send(ctx))
+        .log_err(ctx, "can_send failed")
+        .unwrap_or_default() as libc::c_int
 }
 
 #[no_mangle]
@@ -2782,6 +2863,16 @@ pub unsafe extern "C" fn dc_msg_get_state(msg: *mut dc_msg_t) -> libc::c_int {
     ffi_msg.message.get_state() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_download_state(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_download_state()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.download_state() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_timestamp(msg: *mut dc_msg_t) -> i64 {
     if msg.is_null() {
@@ -2971,10 +3062,10 @@ pub unsafe extern "C" fn dc_msg_get_summary(
     let ffi_msg = &mut *msg;
     let ctx = &*ffi_msg.context;
 
-    block_on(async move {
-        let lot = ffi_msg.message.get_summary(ctx, maybe_chat).await;
-        Box::into_raw(Box::new(lot))
-    })
+    let summary = block_on(ffi_msg.message.get_summary(ctx, maybe_chat))
+        .log_err(ctx, "dc_msg_get_summary failed")
+        .unwrap_or_default();
+    Box::into_raw(Box::new(summary.into()))
 }
 
 #[no_mangle]
@@ -2989,12 +3080,13 @@ pub unsafe extern "C" fn dc_msg_get_summarytext(
     let ffi_msg = &mut *msg;
     let ctx = &*ffi_msg.context;
 
-    block_on({
-        ffi_msg
-            .message
-            .get_summarytext(ctx, approx_characters.try_into().unwrap_or_default())
-    })
-    .strdup()
+    let summary = block_on(ffi_msg.message.get_summary(ctx, None))
+        .log_err(ctx, "dc_msg_get_summarytext failed")
+        .unwrap_or_default();
+    match usize::try_from(approx_characters) {
+        Ok(chars) => summary.truncated_text(chars).strdup(),
+        Err(_) => summary.text.strdup(),
+    }
 }
 
 #[no_mangle]
@@ -3471,7 +3563,9 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
     let ffi_contact = &*contact;
     let ctx = &*ffi_contact.context;
 
-    block_on(async move { ffi_contact.contact.is_verified(ctx).await as libc::c_int })
+    block_on(ffi_contact.contact.is_verified(ctx))
+        .log_err(ctx, "is_verified failed")
+        .unwrap_or_default() as libc::c_int
 }
 
 // dc_lot_t
@@ -3529,7 +3623,7 @@ pub unsafe extern "C" fn dc_lot_get_state(lot: *mut dc_lot_t) -> libc::c_int {
     }
 
     let lot = &*lot;
-    lot.get_state().to_i64().expect("impossible") as libc::c_int
+    lot.get_state() as libc::c_int
 }
 
 #[no_mangle]
@@ -3683,8 +3777,29 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
 
 // -- Accounts
 
+/// Reader-writer lock wrapper for accounts manager to guarantee thread safety when using
+/// `dc_accounts_t` in multiple threads at once.
+pub struct AccountsWrapper {
+    inner: RwLock<Accounts>,
+}
+
+impl Deref for AccountsWrapper {
+    type Target = RwLock<Accounts>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl AccountsWrapper {
+    fn new(accounts: Accounts) -> Self {
+        let inner = RwLock::new(accounts);
+        Self { inner }
+    }
+}
+
 /// Struct representing a list of deltachat accounts.
-pub type dc_accounts_t = Accounts;
+pub type dc_accounts_t = AccountsWrapper;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_new(
@@ -3707,7 +3822,7 @@ pub unsafe extern "C" fn dc_accounts_new(
     let accs = block_on(Accounts::new(os_name, as_path(dbfile).to_path_buf().into()));
 
     match accs {
-        Ok(accs) => Box::into_raw(Box::new(accs)),
+        Ok(accs) => Box::into_raw(Box::new(AccountsWrapper::new(accs))),
         Err(err) => {
             // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
             eprintln!("failed to create accounts: {:#}", err);
@@ -3739,7 +3854,7 @@ pub unsafe extern "C" fn dc_accounts_get_account(
     }
 
     let accounts = &*accounts;
-    block_on(accounts.get_account(id))
+    block_on(async move { accounts.read().await.get_account(id).await })
         .map(|ctx| Box::into_raw(Box::new(ctx)))
         .unwrap_or_else(std::ptr::null_mut)
 }
@@ -3754,7 +3869,7 @@ pub unsafe extern "C" fn dc_accounts_get_selected_account(
     }
 
     let accounts = &*accounts;
-    block_on(accounts.get_selected_account())
+    block_on(async move { accounts.read().await.get_selected_account().await })
         .map(|ctx| Box::into_raw(Box::new(ctx)))
         .unwrap_or_else(std::ptr::null_mut)
 }
@@ -3770,9 +3885,19 @@ pub unsafe extern "C" fn dc_accounts_select_account(
     }
 
     let accounts = &*accounts;
-    block_on(accounts.select_account(id))
-        .map(|_| 1)
-        .unwrap_or(0)
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        match accounts.select_account(id).await {
+            Ok(()) => 1,
+            Err(err) => {
+                accounts.emit_event(EventType::Error(format!(
+                    "Failed to select account: {:#}",
+                    err
+                )));
+                0
+            }
+        }
+    })
 }
 
 #[no_mangle]
@@ -3782,9 +3907,21 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -
         return 0;
     }
 
-    let accounts = &*accounts;
+    let accounts = &mut *accounts;
 
-    block_on(accounts.add_account()).unwrap_or(0)
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        match accounts.add_account().await {
+            Ok(id) => id,
+            Err(err) => {
+                accounts.emit_event(EventType::Error(format!(
+                    "Failed to add account: {:#}",
+                    err
+                )));
+                0
+            }
+        }
+    })
 }
 
 #[no_mangle]
@@ -3797,11 +3934,21 @@ pub unsafe extern "C" fn dc_accounts_remove_account(
         return 0;
     }
 
-    let accounts = &*accounts;
+    let accounts = &mut *accounts;
 
-    block_on(accounts.remove_account(id))
-        .map(|_| 1)
-        .unwrap_or_else(|_| 0)
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        match accounts.remove_account(id).await {
+            Ok(()) => 1,
+            Err(err) => {
+                accounts.emit_event(EventType::Error(format!(
+                    "Failed to remove account: {:#}",
+                    err
+                )));
+                0
+            }
+        }
+    })
 }
 
 #[no_mangle]
@@ -3814,12 +3961,25 @@ pub unsafe extern "C" fn dc_accounts_migrate_account(
         return 0;
     }
 
-    let accounts = &*accounts;
+    let accounts = &mut *accounts;
     let dbfile = to_string_lossy(dbfile);
 
-    block_on(accounts.migrate_account(async_std::path::PathBuf::from(dbfile)))
-        .map(|_| 1)
-        .unwrap_or_else(|_| 0)
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        match accounts
+            .migrate_account(async_std::path::PathBuf::from(dbfile))
+            .await
+        {
+            Ok(id) => id,
+            Err(err) => {
+                accounts.emit_event(EventType::Error(format!(
+                    "Failed to migrate account: {:#}",
+                    err
+                )));
+                0
+            }
+        }
+    })
 }
 
 #[no_mangle]
@@ -3830,7 +3990,7 @@ pub unsafe extern "C" fn dc_accounts_get_all(accounts: *mut dc_accounts_t) -> *m
     }
 
     let accounts = &*accounts;
-    let list = block_on(accounts.get_all());
+    let list = block_on(async move { accounts.read().await.get_all().await });
     let array: dc_array_t = list.into();
 
     Box::into_raw(Box::new(array))
@@ -3843,7 +4003,7 @@ pub unsafe extern "C" fn dc_accounts_all_work_done(accounts: *mut dc_accounts_t)
         return 0;
     }
     let accounts = &*accounts;
-    block_on(async move { accounts.all_work_done().await as libc::c_int })
+    block_on(async move { accounts.read().await.all_work_done().await as libc::c_int })
 }
 
 #[no_mangle]
@@ -3854,7 +4014,7 @@ pub unsafe extern "C" fn dc_accounts_start_io(accounts: *mut dc_accounts_t) {
     }
 
     let accounts = &*accounts;
-    block_on(accounts.start_io());
+    block_on(async move { accounts.read().await.start_io().await });
 }
 
 #[no_mangle]
@@ -3865,7 +4025,7 @@ pub unsafe extern "C" fn dc_accounts_stop_io(accounts: *mut dc_accounts_t) {
     }
 
     let accounts = &*accounts;
-    block_on(accounts.stop_io());
+    block_on(async move { accounts.read().await.stop_io().await });
 }
 
 #[no_mangle]
@@ -3876,7 +4036,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network(accounts: *mut dc_accounts_t)
     }
 
     let accounts = &*accounts;
-    block_on(accounts.maybe_network());
+    block_on(async move { accounts.read().await.maybe_network().await });
 }
 
 #[no_mangle]
@@ -3887,7 +4047,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     }
 
     let accounts = &*accounts;
-    block_on(accounts.maybe_network_lost());
+    block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
 pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
@@ -3902,7 +4062,7 @@ pub unsafe extern "C" fn dc_accounts_get_event_emitter(
     }
 
     let accounts = &*accounts;
-    let emitter = block_on(accounts.get_event_emitter());
+    let emitter = block_on(async move { accounts.read().await.get_event_emitter().await });
 
     Box::into_raw(Box::new(emitter))
 }

deltachat-ffi/src/lot.rs

@@ -0,0 +1,245 @@
+//! # Legacy generic return values for C API.
+
+use crate::message::MessageState;
+use crate::qr::Qr;
+use crate::summary::{Summary, SummaryPrefix};
+use anyhow::Error;
+use std::borrow::Cow;
+
+/// An object containing a set of values.
+/// The meaning of the values is defined by the function returning the object.
+/// Lot objects are created
+/// eg. by chatlist.get_summary() or dc_msg_get_summary().
+///
+/// *Lot* is used in the meaning *heap* here.
+#[derive(Debug)]
+pub enum Lot {
+    Summary(Summary),
+    Qr(Qr),
+    Error(String),
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Meaning {
+    None = 0,
+    Text1Draft = 1,
+    Text1Username = 2,
+    Text1Self = 3,
+}
+
+impl Default for Meaning {
+    fn default() -> Self {
+        Meaning::None
+    }
+}
+
+impl Lot {
+    pub fn get_text1(&self) -> Option<&str> {
+        match self {
+            Self::Summary(summary) => match &summary.prefix {
+                None => None,
+                Some(SummaryPrefix::Draft(text)) => Some(text),
+                Some(SummaryPrefix::Username(username)) => Some(username),
+                Some(SummaryPrefix::Me(text)) => Some(text),
+            },
+            Self::Qr(qr) => match qr {
+                Qr::AskVerifyContact { .. } => None,
+                Qr::AskVerifyGroup { grpname, .. } => Some(grpname),
+                Qr::FprOk { .. } => None,
+                Qr::FprMismatch { .. } => None,
+                Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
+                Qr::Account { domain } => Some(domain),
+                Qr::WebrtcInstance { domain, .. } => Some(domain),
+                Qr::Addr { .. } => None,
+                Qr::Url { url } => Some(url),
+                Qr::Text { text } => Some(text),
+                Qr::WithdrawVerifyContact { .. } => None,
+                Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
+                Qr::ReviveVerifyContact { .. } => None,
+                Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
+            },
+            Self::Error(err) => Some(err),
+        }
+    }
+
+    pub fn get_text2(&self) -> Option<Cow<str>> {
+        match self {
+            Self::Summary(summary) => Some(summary.truncated_text(160)),
+            Self::Qr(_) => None,
+            Self::Error(_) => None,
+        }
+    }
+
+    pub fn get_text1_meaning(&self) -> Meaning {
+        match self {
+            Self::Summary(summary) => match &summary.prefix {
+                None => Meaning::None,
+                Some(SummaryPrefix::Draft(_text)) => Meaning::Text1Draft,
+                Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
+                Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
+            },
+            Self::Qr(_qr) => Meaning::None,
+            Self::Error(_err) => Meaning::None,
+        }
+    }
+
+    pub fn get_state(&self) -> LotState {
+        match self {
+            Self::Summary(summary) => summary.state.into(),
+            Self::Qr(qr) => match qr {
+                Qr::AskVerifyContact { .. } => LotState::QrAskVerifyContact,
+                Qr::AskVerifyGroup { .. } => LotState::QrAskVerifyGroup,
+                Qr::FprOk { .. } => LotState::QrFprOk,
+                Qr::FprMismatch { .. } => LotState::QrFprMismatch,
+                Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
+                Qr::Account { .. } => LotState::QrAccount,
+                Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
+                Qr::Addr { .. } => LotState::QrAddr,
+                Qr::Url { .. } => LotState::QrUrl,
+                Qr::Text { .. } => LotState::QrText,
+                Qr::WithdrawVerifyContact { .. } => LotState::QrWithdrawVerifyContact,
+                Qr::WithdrawVerifyGroup { .. } => LotState::QrWithdrawVerifyGroup,
+                Qr::ReviveVerifyContact { .. } => LotState::QrReviveVerifyContact,
+                Qr::ReviveVerifyGroup { .. } => LotState::QrReviveVerifyGroup,
+            },
+            Self::Error(_err) => LotState::QrError,
+        }
+    }
+
+    pub fn get_id(&self) -> u32 {
+        match self {
+            Self::Summary(_) => Default::default(),
+            Self::Qr(qr) => match qr {
+                Qr::AskVerifyContact { contact_id, .. } => *contact_id,
+                Qr::AskVerifyGroup { .. } => Default::default(),
+                Qr::FprOk { contact_id } => *contact_id,
+                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default(),
+                Qr::FprWithoutAddr { .. } => Default::default(),
+                Qr::Account { .. } => Default::default(),
+                Qr::WebrtcInstance { .. } => Default::default(),
+                Qr::Addr { contact_id } => *contact_id,
+                Qr::Url { .. } => Default::default(),
+                Qr::Text { .. } => Default::default(),
+                Qr::WithdrawVerifyContact { contact_id, .. } => *contact_id,
+                Qr::WithdrawVerifyGroup { .. } => Default::default(),
+                Qr::ReviveVerifyContact { contact_id, .. } => *contact_id,
+                Qr::ReviveVerifyGroup { .. } => Default::default(),
+            },
+            Self::Error(_) => Default::default(),
+        }
+    }
+
+    pub fn get_timestamp(&self) -> i64 {
+        match self {
+            Self::Summary(summary) => summary.timestamp,
+            Self::Qr(_) => Default::default(),
+            Self::Error(_) => Default::default(),
+        }
+    }
+}
+
+#[repr(u32)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LotState {
+    // Default
+    Undefined = 0,
+
+    // Qr States
+    /// id=contact
+    QrAskVerifyContact = 200,
+
+    /// text1=groupname
+    QrAskVerifyGroup = 202,
+
+    /// id=contact
+    QrFprOk = 210,
+
+    /// id=contact
+    QrFprMismatch = 220,
+
+    /// text1=formatted fingerprint
+    QrFprWithoutAddr = 230,
+
+    /// text1=domain
+    QrAccount = 250,
+
+    /// text1=domain, text2=instance pattern
+    QrWebrtcInstance = 260,
+
+    /// id=contact
+    QrAddr = 320,
+
+    /// text1=text
+    QrText = 330,
+
+    /// text1=URL
+    QrUrl = 332,
+
+    /// text1=error string
+    QrError = 400,
+
+    QrWithdrawVerifyContact = 500,
+
+    /// text1=groupname
+    QrWithdrawVerifyGroup = 502,
+
+    QrReviveVerifyContact = 510,
+
+    /// text1=groupname
+    QrReviveVerifyGroup = 512,
+
+    // Message States
+    MsgInFresh = 10,
+    MsgInNoticed = 13,
+    MsgInSeen = 16,
+    MsgOutPreparing = 18,
+    MsgOutDraft = 19,
+    MsgOutPending = 20,
+    MsgOutFailed = 24,
+    MsgOutDelivered = 26,
+    MsgOutMdnRcvd = 28,
+}
+
+impl Default for LotState {
+    fn default() -> Self {
+        LotState::Undefined
+    }
+}
+
+impl From<MessageState> for LotState {
+    fn from(s: MessageState) -> Self {
+        use MessageState::*;
+        match s {
+            Undefined => LotState::Undefined,
+            InFresh => LotState::MsgInFresh,
+            InNoticed => LotState::MsgInNoticed,
+            InSeen => LotState::MsgInSeen,
+            OutPreparing => LotState::MsgOutPreparing,
+            OutDraft => LotState::MsgOutDraft,
+            OutPending => LotState::MsgOutPending,
+            OutFailed => LotState::MsgOutFailed,
+            OutDelivered => LotState::MsgOutDelivered,
+            OutMdnRcvd => LotState::MsgOutMdnRcvd,
+        }
+    }
+}
+
+impl From<Summary> for Lot {
+    fn from(summary: Summary) -> Self {
+        Lot::Summary(summary)
+    }
+}
+
+impl From<Qr> for Lot {
+    fn from(qr: Qr) -> Self {
+        Lot::Qr(qr)
+    }
+}
+
+// Make it easy to convert errors into the final `Lot`.
+impl From<Error> for Lot {
+    fn from(error: Error) -> Self {
+        Lot::Error(error.to_string())
+    }
+}

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1.0.74"
-quote = "1.0.2"
+syn = "1"
+quote = "1"

examples/repl/cmdline.rs

@@ -2,7 +2,7 @@ extern crate dirs;
 
 use std::str::FromStr;
 
-use anyhow::{bail, ensure, Error};
+use anyhow::{bail, ensure, Result};
 use async_std::path::Path;
 use deltachat::chat::{
     self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
@@ -13,10 +13,10 @@ use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
+use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
-use deltachat::lot::LotState;
 use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
@@ -98,7 +98,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     });
 }
 
-async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
+async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
     let data = dc_read_file(context, filename).await?;
 
     if let Err(err) = dc_receive_imf(context, &data, "import", 0, false).await {
@@ -189,10 +189,18 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         MessageState::OutFailed => " !!",
         _ => "",
     };
+
+    let downloadstate = match msg.download_state() {
+        DownloadState::Done => "",
+        DownloadState::Available => " [⬇ Download available]",
+        DownloadState::InProgress => " [⬇ Download in progress...]️",
+        DownloadState::Failure => " [⬇ Download failed]",
+    };
+
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -226,11 +234,12 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             ""
         },
         statestr,
+        downloadstate,
         &temp2,
     );
 }
 
-async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
+async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
     let mut lines_out = 0;
     for &msg_id in msglist {
         if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
@@ -258,59 +267,59 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error>
     Ok(())
 }
 
-async fn log_contactlist(context: &Context, contacts: &[u32]) {
+async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        if let Ok(contact) = Contact::get_by_id(context, *contact_id).await {
-            let name = contact.get_display_name();
-            let addr = contact.get_addr();
-            let verified_state = contact.is_verified(context).await;
-            let verified_str = if VerifiedStatus::Unverified != verified_state {
-                if verified_state == VerifiedStatus::BidirectVerified {
-                    " √√"
-                } else {
-                    " √"
-                }
+        let contact = Contact::get_by_id(context, *contact_id).await?;
+        let name = contact.get_display_name();
+        let addr = contact.get_addr();
+        let verified_state = contact.is_verified(context).await?;
+        let verified_str = if VerifiedStatus::Unverified != verified_state {
+            if verified_state == VerifiedStatus::BidirectVerified {
+                " √√"
             } else {
-                ""
-            };
-            line = format!(
-                "{}{} <{}>",
-                if !name.is_empty() {
-                    &name
-                } else {
-                    "<name unset>"
-                },
-                verified_str,
-                if !addr.is_empty() {
-                    &addr
-                } else {
-                    "addr unset"
-                }
-            );
-            let peerstate = Peerstate::from_addr(context, &addr)
-                .await
-                .expect("peerstate error");
-            if peerstate.is_some() && *contact_id != 1 {
-                line2 = format!(
-                    ", prefer-encrypt={}",
-                    peerstate.as_ref().unwrap().prefer_encrypt
-                );
+                " √"
             }
-
-            println!("Contact#{}: {}{}", *contact_id, line, line2);
+        } else {
+            ""
+        };
+        line = format!(
+            "{}{} <{}>",
+            if !name.is_empty() {
+                &name
+            } else {
+                "<name unset>"
+            },
+            verified_str,
+            if !addr.is_empty() {
+                &addr
+            } else {
+                "addr unset"
+            }
+        );
+        let peerstate = Peerstate::from_addr(context, &addr)
+            .await
+            .expect("peerstate error");
+        if peerstate.is_some() && *contact_id != 1 {
+            line2 = format!(
+                ", prefer-encrypt={}",
+                peerstate.as_ref().unwrap().prefer_encrypt
+            );
         }
+
+        println!("Contact#{}: {}{}", *contact_id, line, line2);
     }
+    Ok(())
 }
 
 fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<(), Error> {
+pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<()> {
     let mut sel_chat = if !chat_id.is_unset() {
-        Chat::load_from_db(&context, *chat_id).await.ok()
+        Some(Chat::load_from_db(&context, *chat_id).await?)
     } else {
         None
     };
@@ -361,6 +370,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  chat [<chat-id>|0]\n\
                  createchat <contact-id>\n\
                  creategroup <name>\n\
+                 createbroadcast\n\
                  createprotected <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
@@ -376,6 +386,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  sendsticker <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
                  sendhtml <file for html-part> [<text for plain-part>]\n\
+                 sendsyncmsg\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -394,6 +405,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
+                 download <msg-id>\n\
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
@@ -413,6 +425,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  getqr [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
+                 joinqr <qr-content>\n\
                  setqr <qr-content>\n\
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
@@ -450,7 +463,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            continue_key_transfer(&context, MsgId::new(arg1.parse()?), &arg2).await?;
+            continue_key_transfer(&context, MsgId::new(arg1.parse()?), arg2).await?;
         }
         "has-backup" => {
             has_backup(&context, blobdir).await?;
@@ -497,13 +510,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "set" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(&arg1)?;
+            let key = config::Config::from_str(arg1)?;
             let value = if arg2.is_empty() { None } else { Some(arg2) };
             context.set_config(key, value).await?;
         }
         "get" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(&arg1)?;
+            let key = config::Config::from_str(arg1)?;
             let val = context.get_config(key).await;
             println!("{}={:?}", key, val);
         }
@@ -569,26 +582,25 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             ""
                         },
                     );
-                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await?;
+                    let summary = chatlist.get_summary(&context, i, Some(&chat)).await?;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
-                        match lot.get_state() {
-                            LotState::MsgOutPending => " o",
-                            LotState::MsgOutDelivered => " √",
-                            LotState::MsgOutMdnRcvd => " √√",
-                            LotState::MsgOutFailed => " !!",
+                        match summary.state {
+                            MessageState::OutPending => " o",
+                            MessageState::OutDelivered => " √",
+                            MessageState::OutMdnRcvd => " √√",
+                            MessageState::OutFailed => " !!",
                             _ => "",
                         }
                     };
-                    let timestr = dc_timestamp_to_str(lot.get_timestamp());
-                    let text1 = lot.get_text1();
-                    let text2 = lot.get_text2();
+                    let timestr = dc_timestamp_to_str(summary.timestamp);
                     println!(
-                        "{}{}{}{} [{}]{}",
-                        text1.unwrap_or(""),
-                        if text1.is_some() { ": " } else { "" },
-                        text2.unwrap_or(""),
+                        "{}{}{} [{}]{}",
+                        summary
+                            .prefix
+                            .map_or_else(String::new, |prefix| format!("{}: ", prefix)),
+                        summary.text,
                         statestr,
                         &timestr,
                         if chat.is_sending_locations() {
@@ -602,7 +614,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, None).await {
+            if location::is_sending_locations_to_chat(&context, None).await? {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
@@ -705,6 +717,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!("Group#{} created successfully.", chat_id);
         }
+        "createbroadcast" => {
+            let chat_id = chat::create_broadcast_list(&context).await?;
+
+            println!("Broadcast#{} created successfully.", chat_id);
+        }
         "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
@@ -717,17 +734,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
             let contact_id_0: u32 = arg1.parse()?;
-            if chat::add_contact_to_chat(
-                &context,
-                sel_chat.as_ref().unwrap().get_id(),
-                contact_id_0,
-            )
-            .await
-            {
-                println!("Contact added to chat.");
-            } else {
-                bail!("Cannot add contact to chat.");
-            }
+            chat::add_contact_to_chat(&context, sel_chat.as_ref().unwrap().get_id(), contact_id_0)
+                .await?;
+            println!("Contact added to chat.");
         }
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -765,7 +774,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await?;
             println!("Memberlist:");
 
-            log_contactlist(&context, &contacts).await;
+            log_contactlist(&context, &contacts).await?;
             println!(
                 "{} contacts\nLocation streaming: {}",
                 contacts.len(),
@@ -773,7 +782,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     &context,
                     Some(sel_chat.as_ref().unwrap().get_id())
                 )
-                .await,
+                .await?,
             );
         }
         "getlocations" => {
@@ -818,7 +827,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 sel_chat.as_ref().unwrap().get_id(),
                 seconds,
             )
-            .await;
+            .await?;
             println!(
                 "Locations will be sent to Chat#{} for {} seconds. Use 'setlocation <lat> <lng>' to play around.",
                 sel_chat.as_ref().unwrap().get_id(),
@@ -888,6 +897,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }));
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "sendsyncmsg" => match context.send_sync_msg().await? {
+            Some(msg_id) => println!("sync message sent as {}.", msg_id),
+            None => println!("sync message not needed."),
+        },
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -895,12 +908,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
-            let chat = if let Some(ref sel_chat) = sel_chat {
-                Some(sel_chat.get_id())
-            } else {
-                None
-            };
-
+            let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
             let time_start = std::time::SystemTime::now();
             let msglist = context.search_msgs(chat, arg1).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
@@ -1030,6 +1038,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let res = message::get_msg_info(&context, id).await?;
             println!("{}", res);
         }
+        "download" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            println!("Scheduling download for {:?}", id);
+            id.download_full(&context).await?;
+        }
         "html" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
@@ -1067,7 +1081,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
             ids[0] = MsgId::new(arg1.parse()?);
-            message::delete_msgs(&context, &ids).await;
+            message::delete_msgs(&context, &ids).await?;
         }
         "listcontacts" | "contacts" | "listverified" => {
             let contacts = Contact::get_all(
@@ -1080,7 +1094,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 Some(arg1),
             )
             .await?;
-            log_contactlist(&context, &contacts).await;
+            log_contactlist(&context, &contacts).await?;
             println!("{} contacts.", contacts.len());
         }
         "addcontact" => {
@@ -1145,19 +1159,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "listblocked" => {
             let contacts = Contact::get_all_blocked(&context).await?;
-            log_contactlist(&context, &contacts).await;
+            log_contactlist(&context, &contacts).await?;
             println!("{} blocked contacts.", contacts.len());
         }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            let res = check_qr(&context, arg1).await;
-            println!(
-                "state={}, id={}, text1={:?}, text2={:?}",
-                res.get_state(),
-                res.get_id(),
-                res.get_text1(),
-                res.get_text2()
-            );
+            let qr = check_qr(&context, arg1).await?;
+            println!("qr={:?}", qr);
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");

examples/repl/main.rs

@@ -167,13 +167,14 @@ const DB_COMMANDS: [&str; 10] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 33] = [
+const CHAT_COMMANDS: [&str; 35] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
     "creategroup",
-    "createverified",
+    "createbroadcast",
+    "createprotected",
     "addmember",
     "removemember",
     "groupname",
@@ -187,6 +188,7 @@ const CHAT_COMMANDS: [&str; 33] = [
     "sendimage",
     "sendfile",
     "sendhtml",
+    "sendsyncmsg",
     "videochat",
     "draft",
     "listmedia",
@@ -202,13 +204,14 @@ const CHAT_COMMANDS: [&str; 33] = [
     "accept",
     "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 6] = [
+const MESSAGE_COMMANDS: [&str; 7] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
     "delmsg",
+    "download",
 ];
 const CONTACT_COMMANDS: [&str; 9] = [
     "listcontacts",
@@ -221,10 +224,11 @@ const CONTACT_COMMANDS: [&str; 9] = [
     "unblock",
     "listblocked",
 ];
-const MISC_COMMANDS: [&str; 10] = [
+const MISC_COMMANDS: [&str; 11] = [
     "getqr",
     "getbadqr",
     "checkqr",
+    "joinqr",
     "event",
     "fileinfo",
     "clear",
@@ -325,7 +329,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
 
         loop {
             let p = "> ";
-            let readline = rl.readline(&p);
+            let readline = rl.readline(p);
 
             match readline {
                 Ok(line) => {
@@ -409,19 +413,18 @@ async fn handle_cmd(
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
             let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
-                if !qr.is_empty() {
-                    if arg0 == "getbadqr" && qr.len() > 40 {
-                        qr.replace_range(12..22, "0000000000")
-                    }
-                    println!("{}", qr);
-                    let output = Command::new("qrencode")
-                        .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
-                        .output()
-                        .expect("failed to execute process");
-                    io::stdout().write_all(&output.stdout).unwrap();
-                    io::stderr().write_all(&output.stderr).unwrap();
+            let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
+            if !qr.is_empty() {
+                if arg0 == "getbadqr" && qr.len() > 40 {
+                    qr.replace_range(12..22, "0000000000")
                 }
+                println!("{}", qr);
+                let output = Command::new("qrencode")
+                    .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
+                    .output()
+                    .expect("failed to execute process");
+                io::stdout().write_all(&output.stdout).unwrap();
+                io::stderr().write_all(&output.stderr).unwrap();
             }
         }
         "joinqr" => {

proptest-regressions/dc_tools.txt

@@ -7,3 +7,5 @@
 cc c310754465ee0261807b96fa9bcc4861ff9aa286e94667524b5960c69f9b6620 # shrinks to buf = "", approx_chars = 0, do_unwrap = false
 cc 5fd8d730b0a9cdf7308ce58818ca9aefc0255c9ba2a0878944fc48d43a67315b # shrinks to buf = "𑒀ὐ￠🜀\u{1e01b}A a🟠", approx_chars = 0, do_unwrap = false
 cc c6a0029a54137a4b9efc9ef2ea6d9a7dd1d60d1c937bb472b66a174618ba8013 # shrinks to buf = "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ  ", approx_chars = 0, do_unwrap = false
+cc 9796807baeda701227dcdcfc9fdaa93ddd556da2bb1630381bfe2e037bee73f6 # shrinks to buf = " ꫛ®a\u{11300}a", approx_chars = 0
+cc 063a4c42ac1ec9aa37af54521b210ba9cd82dcc9cc3be296ca2fedf8240072d4 # shrinks to buf = "a᪠  0A", approx_chars = 0

python/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
+
+[tool.setuptools_scm]
+root = ".."
+tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
+git_describe_command = "git describe --dirty --tags --long --match py-*.*"

python/setup.py

@@ -8,13 +8,6 @@ def main():
         long_description = f.read()
     setuptools.setup(
         name='deltachat',
-        setup_requires=['setuptools_scm', 'cffi>=1.0.0'],
-        use_scm_version = {
-            "root": "..",
-            "relative_to": __file__,
-            'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
-            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
-        },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',

python/src/deltachat/chat.py

@@ -3,7 +3,7 @@
 import mimetypes
 import calendar
 import json
-from datetime import datetime
+from datetime import datetime, timezone
 import os
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
@@ -512,8 +512,9 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.utcfromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i)
+                timestamp=datetime.fromtimestamp(
+                    lib.dc_array_get_timestamp(dc_array, i),
+                    timezone.utc
                 ),
                 marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )

python/src/deltachat/cutil.py

@@ -1,6 +1,6 @@
 from .capi import lib
 from .capi import ffi
-from datetime import datetime
+from datetime import datetime, timezone
 
 
 def as_dc_charpointer(obj):
@@ -44,4 +44,4 @@ class DCLot:
         ts = lib.dc_lot_get_timestamp(self._dc_lot)
         if ts == 0:
             return None
-        return datetime.utcfromtimestamp(ts)
+        return datetime.fromtimestamp(ts, timezone.utc)

python/src/deltachat/message.py

@@ -6,7 +6,7 @@ from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
 from . import const
-from datetime import datetime
+from datetime import datetime, timezone
 
 
 class Message(object):
@@ -170,7 +170,7 @@ class Message(object):
         :returns: naive datetime.datetime() object.
         """
         ts = lib.dc_msg_get_timestamp(self._dc_msg)
-        return datetime.utcfromtimestamp(ts)
+        return datetime.fromtimestamp(ts, timezone.utc)
 
     @props.with_doc
     def time_received(self):
@@ -180,7 +180,7 @@ class Message(object):
         """
         ts = lib.dc_msg_get_received_timestamp(self._dc_msg)
         if ts:
-            return datetime.utcfromtimestamp(ts)
+            return datetime.fromtimestamp(ts, timezone.utc)
 
     @props.with_doc
     def ephemeral_timer(self):
@@ -200,7 +200,7 @@ class Message(object):
         """
         ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
         if ts:
-            return datetime.utcfromtimestamp(ts)
+            return datetime.fromtimestamp(ts, timezone.utc)
 
     @property
     def quoted_text(self):

python/tests/test_account.py

@@ -10,7 +10,7 @@ from deltachat.tracker import ImexTracker
 from deltachat.hookspec import account_hookimpl
 from deltachat.capi import ffi, lib
 from deltachat.cutil import iter_array
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 
 @pytest.mark.parametrize("msgtext,res", [
@@ -447,7 +447,7 @@ class TestOfflineChat:
         contact1.create_chat().send_text("hello")
 
     def test_chat_message_distinctions(self, ac1, chat1):
-        past1s = datetime.utcnow() - timedelta(seconds=1)
+        past1s = datetime.now(timezone.utc) - timedelta(seconds=1)
         msg = chat1.send_text("msg1")
         ts = msg.time_sent
         assert msg.time_received is None
@@ -973,7 +973,7 @@ class TestOnlineAccount:
         ac1._evtracker.wait_msg_delivered(msg1)
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         assert msg2.text == "message1"
         assert not msg2.is_forwarded()
         assert msg2.get_sender_contact().display_name == ac1.get_config("displayname")
@@ -1125,7 +1125,7 @@ class TestOnlineAccount:
         group1.add_contact(ac2)
         group1.send_text("hello")
 
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         group2 = msg2.create_chat()
         assert group2.get_name() == group1.get_name()
 
@@ -1188,7 +1188,7 @@ class TestOnlineAccount:
         chat.send_text("message1")
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         assert msg2.text == "message1"
 
         lp.sec("create new chat with contact and send back (encrypted) message")
@@ -1221,6 +1221,40 @@ class TestOnlineAccount:
         assert not msg.is_encrypted()
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    def test_gossip_optimization(self, acfactory, lp):
+        """Test that gossip timestamp is updated when someone else sends gossip,
+        so we don't have to send gossip ourselves.
+        """
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+
+        acfactory.introduce_each_other([ac1, ac2])
+        acfactory.introduce_each_other([ac2, ac3])
+
+        lp.sec("ac1 creates a group chat with ac2")
+        group_chat = ac1.create_group_chat("hello")
+        group_chat.add_contact(ac2)
+        msg = group_chat.send_text("hi")
+
+        # No Autocrypt gossip was sent yet.
+        gossiped_timestamp = msg.chat.get_summary()["gossiped_timestamp"]
+        assert gossiped_timestamp == 0
+
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+        assert msg.text == "hi"
+
+        lp.sec("ac2 adds ac3 to the group")
+        msg.chat.add_contact(ac3)
+
+        lp.sec("ac1 receives message from ac2 and updates gossip timestamp")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+
+        # ac1 has updated the gossip timestamp even though no gossip was sent by ac1.
+        # ac1 does not need to send gossip because ac2 already did it.
+        gossiped_timestamp = msg.chat.get_summary()["gossiped_timestamp"]
+        assert gossiped_timestamp == int(msg.time_sent.timestamp())
+
     def test_gossip_encryption_preference(self, acfactory, lp):
         """Test that encryption preference of group members is gossiped to new members.
         This is a Delta Chat extension to Autocrypt 1.1.0, which Autocrypt-Gossip headers
@@ -1468,7 +1502,7 @@ class TestOnlineAccount:
         assert not msg1.is_encrypted()
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         assert msg2.text == "message1"
         assert not msg2.is_encrypted()
 
@@ -1517,7 +1551,7 @@ class TestOnlineAccount:
         chat1.send_text("hi")
 
         lp.sec("ac2 receives contact request from ac1")
-        received_message = ac2._evtracker.wait_next_messages_changed()
+        received_message = ac2._evtracker.wait_next_incoming_message()
         assert received_message.text == "hi"
 
         basename = "attachment.txt"
@@ -1552,7 +1586,7 @@ class TestOnlineAccount:
         assert msg_out.get_mime_headers() is None
 
         lp.sec("wait for ac2 to receive message")
-        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
         in_id = ev.data2
         mime = ac2.get_message_by_id(in_id).get_mime_headers()
         assert mime.get_all("From")
@@ -1851,7 +1885,7 @@ class TestOnlineAccount:
         ac1.create_chat(ac2).send_text("with avatar!")
 
         lp.sec("ac2: wait for receiving message and avatar from ac1")
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         assert msg2.chat.is_contact_request()
         received_path = msg2.get_sender_contact().get_profile_image()
         assert open(received_path, "rb").read() == open(p, "rb").read()
@@ -2163,7 +2197,7 @@ class TestOnlineAccount:
                 break  # DC is done with reading messages
 
     def test_send_receive_locations(self, acfactory, lp):
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         lp.sec("ac1: create chat with ac2")
@@ -2681,7 +2715,7 @@ class TestOnlineAccount:
 
         lp.sec("receive a message")
         ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
-        ac1._evtracker.wait_next_messages_changed()
+        ac1._evtracker.wait_next_incoming_message()
 
         lp.sec("send out message with bcc to ourselves")
         ac1.direct_imap.idle_start()
@@ -2712,6 +2746,22 @@ class TestOnlineAccount:
         # We can't decrypt the message in this chat, so the chat is empty:
         assert len(private_messages) == 0
 
+    def test_delete_deltachat_folder(self, acfactory):
+        """Test that DeltaChat folder is recreated if user deletes it manually."""
+        ac1 = acfactory.get_online_configuring_account(mvbox=True)
+        ac2 = acfactory.get_online_configuring_account()
+        acfactory.wait_configure(ac1)
+
+        ac1.direct_imap.conn.delete_folder("DeltaChat")
+        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 0
+        acfactory.wait_configure_and_start_io()
+
+        ac2.create_chat(ac1).send_text("hello")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 1
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):

python/tests/test_increation.py

@@ -6,8 +6,6 @@ import shutil
 import pytest
 from filecmp import cmp
 
-from deltachat import const
-
 
 def wait_msg_delivered(account, msg_list):
     """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
@@ -102,14 +100,10 @@ class TestOnlineInCreation:
         ])
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1.data2)
+        received_original = ac2._evtracker.wait_next_incoming_message()
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2.data1 != ev1.data1
-        received_copy = ac2.get_message_by_id(ev2.data2)
+        received_copy = ac2._evtracker.wait_next_incoming_message()
+        assert received_copy.id != received_original.id
         assert cmp(received_copy.filename, orig, shallow=False)

python/tox.ini

@@ -45,9 +45,8 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-    # Pin dependencies to the versions which actually work with Python 3.5.
-    sphinx==3.4.3
-    breathe==4.28.0
+    sphinx
+    breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 

scripts/coverage.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-if ! which grcov 2>/dev/null 1>&2; then
+if ! command -v grcov >/dev/null; then
     echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
     echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
     exit 1

scripts/set_core_version.py

@@ -86,6 +86,8 @@ def main():
     print("")
     print("   git tag -a {}".format(newversion))
     print("   git push origin {}".format(newversion))
+    print("   git tag -a py-{}".format(newversion))
+    print("   git push origin py-{}".format(newversion))
     print("")
 
 

src/accounts.rs

@@ -2,7 +2,7 @@
 
 use std::collections::BTreeMap;
 
-use async_std::channel::{Receiver, Sender};
+use async_std::channel::{self, Receiver, Sender};
 use async_std::fs;
 use async_std::path::PathBuf;
 use async_std::prelude::*;
@@ -13,15 +13,18 @@ use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
-use crate::events::Event;
+use crate::events::{Event, EventType, Events};
 
 /// Account manager, that can handle multiple accounts in a single place.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct Accounts {
     dir: PathBuf,
     config: Config,
-    accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
+    accounts: BTreeMap<u32, Context>,
     emitter: EventEmitter,
+
+    /// Event channel to emit account manager errors.
+    events: Events,
 }
 
 impl Accounts {
@@ -57,6 +60,11 @@ impl Accounts {
         let accounts = config.load_accounts().await?;
 
         let emitter = EventEmitter::new();
+
+        let events = Events::default();
+
+        emitter.sender.send(events.get_emitter()).await?;
+
         for account in accounts.values() {
             emitter.add_account(account).await?;
         }
@@ -64,20 +72,21 @@ impl Accounts {
         Ok(Self {
             dir,
             config,
-            accounts: Arc::new(RwLock::new(accounts)),
+            accounts,
             emitter,
+            events,
         })
     }
 
     /// Get an account by its `id`:
     pub async fn get_account(&self, id: u32) -> Option<Context> {
-        self.accounts.read().await.get(&id).cloned()
+        self.accounts.get(&id).cloned()
     }
 
     /// Get the currently selected account.
     pub async fn get_selected_account(&self) -> Option<Context> {
         let id = self.config.get_selected_account().await;
-        self.accounts.read().await.get(&id).cloned()
+        self.accounts.get(&id).cloned()
     }
 
     /// Returns the currently selected account's id or None if no account is selected.
@@ -89,27 +98,27 @@ impl Accounts {
     }
 
     /// Select the given account.
-    pub async fn select_account(&self, id: u32) -> Result<()> {
+    pub async fn select_account(&mut self, id: u32) -> Result<()> {
         self.config.select_account(id).await?;
 
         Ok(())
     }
 
     /// Add a new account.
-    pub async fn add_account(&self) -> Result<u32> {
+    pub async fn add_account(&mut self) -> Result<u32> {
         let os_name = self.config.os_name().await;
         let account_config = self.config.new_account(&self.dir).await?;
 
         let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
         self.emitter.add_account(&ctx).await?;
-        self.accounts.write().await.insert(account_config.id, ctx);
+        self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
     }
 
     /// Remove an account.
-    pub async fn remove_account(&self, id: u32) -> Result<()> {
-        let ctx = self.accounts.write().await.remove(&id);
+    pub async fn remove_account(&mut self, id: u32) -> Result<()> {
+        let ctx = self.accounts.remove(&id);
         ensure!(ctx.is_some(), "no account with this id: {}", id);
         let ctx = ctx.unwrap();
         ctx.stop_io().await;
@@ -126,7 +135,7 @@ impl Accounts {
     }
 
     /// Migrate an existing account into this structure.
-    pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
+    pub async fn migrate_account(&mut self, dbfile: PathBuf) -> Result<u32> {
         let blobdir = Context::derive_blobdir(&dbfile);
         let walfile = Context::derive_walfile(&dbfile);
 
@@ -182,7 +191,7 @@ impl Accounts {
                 )
                 .await?;
                 self.emitter.add_account(&ctx).await?;
-                self.accounts.write().await.insert(account_config.id, ctx);
+                self.accounts.insert(account_config.id, ctx);
                 Ok(account_config.id)
             }
             Err(err) => {
@@ -203,7 +212,7 @@ impl Accounts {
 
     /// Get a list of all account ids.
     pub async fn get_all(&self) -> Vec<u32> {
-        self.accounts.read().await.keys().copied().collect()
+        self.accounts.keys().copied().collect()
     }
 
     /// This is meant especially for iOS, because iOS needs to tell the system when its background work is done.
@@ -217,7 +226,7 @@ impl Accounts {
     /// - while dc_accounts_all_work_done() returns false:
     ///   -  Wait for DC_EVENT_CONNECTIVITY_CHANGED
     pub async fn all_work_done(&self) -> bool {
-        for account in self.accounts.read().await.values() {
+        for account in self.accounts.values() {
             if !account.all_work_done().await {
                 return false;
             }
@@ -226,33 +235,34 @@ impl Accounts {
     }
 
     pub async fn start_io(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
+        for account in self.accounts.values() {
             account.start_io().await;
         }
     }
 
     pub async fn stop_io(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
+        for account in self.accounts.values() {
             account.stop_io().await;
         }
     }
 
     pub async fn maybe_network(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
+        for account in self.accounts.values() {
             account.maybe_network().await;
         }
     }
 
     pub async fn maybe_network_lost(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
+        for account in self.accounts.values() {
             account.maybe_network_lost().await;
         }
     }
 
+    /// Emits a single event.
+    pub fn emit_event(&self, event: EventType) {
+        self.events.emit(Event { id: 0, typ: event })
+    }
+
     /// Returns unified event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
         self.emitter.clone()
@@ -274,7 +284,7 @@ pub struct EventEmitter {
 
 impl EventEmitter {
     pub fn new() -> Self {
-        let (sender, receiver) = async_std::channel::unbounded();
+        let (sender, receiver) = channel::unbounded();
         Self {
             stream: Arc::new(RwLock::new(futures::stream::SelectAll::new())),
             sender,
@@ -328,12 +338,15 @@ pub const CONFIG_NAME: &str = "accounts.toml";
 pub const DB_NAME: &str = "dc.db";
 
 /// Account manager configuration file.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct Config {
     file: PathBuf,
-    inner: Arc<RwLock<InnerConfig>>,
+    inner: InnerConfig,
 }
 
+/// Account manager configuration file contents.
+///
+/// This is serialized into TOML.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 struct InnerConfig {
     pub os_name: String,
@@ -345,14 +358,15 @@ struct InnerConfig {
 
 impl Config {
     pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
+        let inner = InnerConfig {
+            os_name,
+            accounts: Vec::new(),
+            selected_account: 0,
+            next_id: 1,
+        };
         let cfg = Config {
             file: dir.join(CONFIG_NAME),
-            inner: Arc::new(RwLock::new(InnerConfig {
-                os_name,
-                accounts: Vec::new(),
-                selected_account: 0,
-                next_id: 1,
-            })),
+            inner,
         };
 
         cfg.sync().await?;
@@ -361,17 +375,14 @@ impl Config {
     }
 
     pub async fn os_name(&self) -> String {
-        self.inner.read().await.os_name.clone()
+        self.inner.os_name.clone()
     }
 
     /// Sync the inmemory representation to disk.
     async fn sync(&self) -> Result<()> {
-        fs::write(
-            &self.file,
-            toml::to_string_pretty(&*self.inner.read().await)?,
-        )
-        .await
-        .context("failed to write config")
+        fs::write(&self.file, toml::to_string_pretty(&self.inner)?)
+            .await
+            .context("failed to write config")
     }
 
     /// Read a configuration from the given file into memory.
@@ -379,18 +390,14 @@ impl Config {
         let bytes = fs::read(&file).await.context("failed to read file")?;
         let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
 
-        Ok(Config {
-            file,
-            inner: Arc::new(RwLock::new(inner)),
-        })
+        Ok(Config { file, inner })
     }
 
     pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
-        let cfg = &*self.inner.read().await;
         let mut accounts = BTreeMap::new();
-        for account_config in &cfg.accounts {
+        for account_config in &self.inner.accounts {
             let ctx = Context::new(
-                cfg.os_name.clone(),
+                self.inner.os_name.clone(),
                 account_config.dbfile().into(),
                 account_config.id,
             )
@@ -402,19 +409,18 @@ impl Config {
     }
 
     /// Create a new account in the given root directory.
-    async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
+    async fn new_account(&mut self, dir: &PathBuf) -> Result<AccountConfig> {
         let id = {
-            let inner = &mut self.inner.write().await;
-            let id = inner.next_id;
+            let id = self.inner.next_id;
             let uuid = Uuid::new_v4();
             let target_dir = dir.join(uuid.to_simple_ref().to_string());
 
-            inner.accounts.push(AccountConfig {
+            self.inner.accounts.push(AccountConfig {
                 id,
                 dir: target_dir.into(),
                 uuid,
             });
-            inner.next_id += 1;
+            self.inner.next_id += 1;
             id
         };
 
@@ -426,16 +432,16 @@ impl Config {
     }
 
     /// Removes an existing acccount entirely.
-    pub async fn remove_account(&self, id: u32) -> Result<()> {
+    pub async fn remove_account(&mut self, id: u32) -> Result<()> {
         {
-            let inner = &mut *self.inner.write().await;
-            if let Some(idx) = inner.accounts.iter().position(|e| e.id == id) {
+            if let Some(idx) = self.inner.accounts.iter().position(|e| e.id == id) {
                 // remove account from the configs
-                inner.accounts.remove(idx);
+                self.inner.accounts.remove(idx);
             }
-            if inner.selected_account == id {
+            if self.inner.selected_account == id {
                 // reset selected account
-                inner.selected_account = inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+                self.inner.selected_account =
+                    self.inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
             }
         }
 
@@ -443,29 +449,22 @@ impl Config {
     }
 
     async fn get_account(&self, id: u32) -> Option<AccountConfig> {
-        self.inner
-            .read()
-            .await
-            .accounts
-            .iter()
-            .find(|e| e.id == id)
-            .cloned()
+        self.inner.accounts.iter().find(|e| e.id == id).cloned()
     }
 
     pub async fn get_selected_account(&self) -> u32 {
-        self.inner.read().await.selected_account
+        self.inner.selected_account
     }
 
-    pub async fn select_account(&self, id: u32) -> Result<()> {
+    pub async fn select_account(&mut self, id: u32) -> Result<()> {
         {
-            let inner = &mut *self.inner.write().await;
             ensure!(
-                inner.accounts.iter().any(|e| e.id == id),
+                self.inner.accounts.iter().any(|e| e.id == id),
                 "invalid account id: {}",
                 id
             );
 
-            inner.selected_account = id;
+            self.inner.selected_account = id;
         }
 
         self.sync().await?;
@@ -499,23 +498,17 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts1").into();
 
-        let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let mut accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
         accounts1.add_account().await.unwrap();
 
         let accounts2 = Accounts::open(p).await.unwrap();
 
-        assert_eq!(accounts1.accounts.read().await.len(), 1);
+        assert_eq!(accounts1.accounts.len(), 1);
         assert_eq!(accounts1.config.get_selected_account().await, 1);
 
         assert_eq!(accounts1.dir, accounts2.dir);
-        assert_eq!(
-            &*accounts1.config.inner.read().await,
-            &*accounts2.config.inner.read().await,
-        );
-        assert_eq!(
-            accounts1.accounts.read().await.len(),
-            accounts2.accounts.read().await.len()
-        );
+        assert_eq!(accounts1.config, accounts2.config,);
+        assert_eq!(accounts1.accounts.len(), accounts2.accounts.len());
     }
 
     #[async_std::test]
@@ -523,26 +516,26 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 0);
+        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
         let id = accounts.add_account().await.unwrap();
         assert_eq!(id, 1);
-        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.accounts.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         let id = accounts.add_account().await.unwrap();
         assert_eq!(id, 2);
         assert_eq!(accounts.config.get_selected_account().await, id);
-        assert_eq!(accounts.accounts.read().await.len(), 2);
+        assert_eq!(accounts.accounts.len(), 2);
 
         accounts.select_account(1).await.unwrap();
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         accounts.remove_account(1).await.unwrap();
         assert_eq!(accounts.config.get_selected_account().await, 2);
-        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.accounts.len(), 1);
     }
 
     #[async_std::test]
@@ -550,14 +543,14 @@ mod tests {
         let dir = tempfile::tempdir()?;
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+        let mut accounts = Accounts::new("my_os".into(), p.clone()).await?;
         assert!(accounts.get_selected_account().await.is_none());
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
         let id = accounts.add_account().await?;
         assert!(accounts.get_selected_account().await.is_some());
         assert_eq!(id, 1);
-        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.accounts.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, id);
 
         accounts.remove_account(id).await?;
@@ -571,8 +564,8 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 0);
+        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
         let extern_dbfile: PathBuf = dir.path().join("other").into();
@@ -589,7 +582,7 @@ mod tests {
             .migrate_account(extern_dbfile.clone())
             .await
             .unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.accounts.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         let ctx = accounts.get_selected_account().await.unwrap();
@@ -608,7 +601,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
         for expected_id in 1..10 {
             let id = accounts.add_account().await.unwrap();
@@ -628,7 +621,7 @@ mod tests {
         let dummy_accounts = 10;
 
         let (id0, id1, id2) = {
-            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+            let mut accounts = Accounts::new("my_os".into(), p.clone()).await?;
             accounts.add_account().await?;
             let ids = accounts.get_all().await;
             assert_eq!(ids.len(), 1);
@@ -702,4 +695,30 @@ mod tests {
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_no_accounts_event_emitter() -> Result<()> {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+
+        // Make sure there are no accounts.
+        assert_eq!(accounts.accounts.len(), 0);
+
+        // Create event emitter.
+        let mut event_emitter = accounts.get_event_emitter().await;
+
+        // Test that event emitter does not return `None` immediately.
+        let duration = std::time::Duration::from_millis(1);
+        assert!(async_std::future::timeout(duration, event_emitter.recv())
+            .await
+            .is_err());
+
+        // When account manager is dropped, event emitter is exhausted.
+        drop(accounts);
+        assert_eq!(event_emitter.recv().await?, None);
+
+        Ok(())
+    }
 }

src/aheader.rs

@@ -2,12 +2,12 @@
 //!
 //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
 
+use anyhow::{bail, format_err, Error, Result};
 use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
 use crate::contact::addr_cmp;
-use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};
 
@@ -37,13 +37,13 @@ impl fmt::Display for EncryptPreference {
 }
 
 impl str::FromStr for EncryptPreference {
-    type Err = ();
+    type Err = Error;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    fn from_str(s: &str) -> Result<Self> {
         match s {
             "mutual" => Ok(EncryptPreference::Mutual),
             "nopreference" => Ok(EncryptPreference::NoPreference),
-            _ => Err(()),
+            _ => bail!("Cannot parse encryption preference {}", s),
         }
     }
 }
@@ -70,28 +70,27 @@ impl Aheader {
         }
     }
 
+    /// Tries to parse Autocrypt header.
+    ///
+    /// If there is none, returns None. If the header is present but cannot be parsed, returns an
+    /// error.
     pub fn from_headers(
-        context: &Context,
         wanted_from: &str,
         headers: &[mailparse::MailHeader<'_>],
-    ) -> Option<Self> {
+    ) -> Result<Option<Self>> {
         if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) {
-            match Self::from_str(&value) {
-                Ok(header) => {
-                    if addr_cmp(&header.addr, wanted_from) {
-                        return Some(header);
-                    }
-                }
-                Err(err) => {
-                    warn!(
-                        context,
-                        "found invalid autocrypt header {}: {:?}", value, err
-                    );
-                }
+            let header = Self::from_str(&value)?;
+            if !addr_cmp(&header.addr, wanted_from) {
+                bail!(
+                    "Autocrypt header address {:?} is not {:?}",
+                    header.addr,
+                    wanted_from
+                );
             }
+            Ok(Some(header))
+        } else {
+            Ok(None)
         }
-
-        None
     }
 }
 
@@ -120,9 +119,9 @@ impl fmt::Display for Aheader {
 }
 
 impl str::FromStr for Aheader {
-    type Err = ();
+    type Err = Error;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    fn from_str(s: &str) -> Result<Self> {
         let mut attributes: BTreeMap<String, String> = s
             .split(';')
             .filter_map(|a| {
@@ -136,15 +135,20 @@ impl str::FromStr for Aheader {
 
         let addr = match attributes.remove("addr") {
             Some(addr) => addr,
-            None => {
-                return Err(());
-            }
+            None => bail!("Autocrypt header has no addr"),
         };
         let public_key: SignedPublicKey = attributes
             .remove("keydata")
-            .ok_or(())
-            .and_then(|raw| SignedPublicKey::from_base64(&raw).or(Err(())))
-            .and_then(|key| key.verify().and(Ok(key)).or(Err(())))?;
+            .ok_or_else(|| format_err!("keydata attribute is not found"))
+            .and_then(|raw| {
+                SignedPublicKey::from_base64(&raw)
+                    .map_err(|_| format_err!("Autocrypt key cannot be decoded"))
+            })
+            .and_then(|key| {
+                key.verify()
+                    .and(Ok(key))
+                    .map_err(|_| format_err!("Autocrypt key cannot be verified"))
+            })?;
 
         let prefer_encrypt = attributes
             .remove("prefer-encrypt")
@@ -154,7 +158,7 @@ impl str::FromStr for Aheader {
         // Autocrypt-Level0: unknown attributes starting with an underscore can be safely ignored
         // Autocrypt-Level0: unknown attribute, treat the header as invalid
         if attributes.keys().any(|k| !k.starts_with('_')) {
-            return Err(());
+            bail!("Unknown Autocrypt attribute found");
         }
 
         Ok(Aheader {
@@ -172,38 +176,40 @@ mod tests {
     const RAWKEY: &str = "xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=";
 
     #[test]
-    fn test_from_str() {
+    fn test_from_str() -> Result<()> {
         let h: Aheader = format!(
             "addr=me@mail.com; prefer-encrypt=mutual; keydata={}",
             RAWKEY
         )
-        .parse()
-        .expect("failed to parse");
+        .parse()?;
 
         assert_eq!(h.addr, "me@mail.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::Mutual);
+        Ok(())
     }
 
     // EncryptPreference::Reset is an internal value, parser should never return it
     #[test]
-    fn test_from_str_reset() {
+    fn test_from_str_reset() -> Result<()> {
         let raw = format!(
             "addr=reset@example.com; prefer-encrypt=reset; keydata={}",
             RAWKEY
         );
-        let h: Aheader = raw.parse().expect("failed to parse");
+        let h: Aheader = raw.parse()?;
 
         assert_eq!(h.addr, "reset@example.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
+        Ok(())
     }
 
     #[test]
-    fn test_from_str_non_critical() {
+    fn test_from_str_non_critical() -> Result<()> {
         let raw = format!("addr=me@mail.com; _foo=one; _bar=two; keydata={}", RAWKEY);
-        let h: Aheader = raw.parse().expect("failed to parse");
+        let h: Aheader = raw.parse()?;
 
         assert_eq!(h.addr, "me@mail.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
+        Ok(())
     }
 
     #[test]
@@ -216,7 +222,7 @@ mod tests {
     }
 
     #[test]
-    fn test_good_headers() {
+    fn test_good_headers() -> Result<()> {
         let fixed_header = concat!(
             "addr=a@b.example.org; prefer-encrypt=mutual; ",
             "keydata=xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJg",
@@ -242,7 +248,7 @@ mod tests {
             " wm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc="
         );
 
-        let ah = Aheader::from_str(fixed_header).expect("failed to parse");
+        let ah = Aheader::from_str(fixed_header)?;
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
         assert_eq!(format!("{}", ah), fixed_header);
@@ -250,18 +256,17 @@ mod tests {
         let rendered = ah.to_string();
         assert_eq!(rendered, fixed_header);
 
-        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY)).expect("failed to parse");
+        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY))?;
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
 
         Aheader::from_str(&format!(
             "addr=a@b.example.org; prefer-encrypt=ignoreUnknownValues; keydata={}",
             RAWKEY
-        ))
-        .expect("failed to parse");
+        ))?;
 
-        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))
-            .expect("failed to parse");
+        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))?;
+        Ok(())
     }
 
     #[test]

src/blob.rs

@@ -621,7 +621,10 @@ mod tests {
 
     use super::*;
 
-    use crate::{message::Message, test_utils::TestContext};
+    use crate::{
+        message::Message,
+        test_utils::{self, TestContext},
+    };
     use image::Pixel;
 
     #[async_std::test]
@@ -870,11 +873,10 @@ mod tests {
     async fn test_selfavatar_in_blobdir() {
         let t = TestContext::new().await;
         let avatar_src = t.get_blobdir().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
             .await
             .unwrap()
-            .write_all(avatar_bytes)
+            .write_all(test_utils::AVATAR_900x900_BYTES)
             .await
             .unwrap();
 

src/chatlist.rs

@@ -11,9 +11,9 @@ use crate::constants::{
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::ephemeral::delete_expired_messages;
-use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock_str;
+use crate::summary::Summary;
 
 /// An object representing a single chatlist in memory.
 ///
@@ -288,26 +288,13 @@ impl Chatlist {
         }
     }
 
-    /// Get a summary for a chatlist index.
-    ///
-    /// The summary is returned by a dc_lot_t object with the following fields:
-    ///
-    /// - dc_lot_t::text1: contains the username or the strings "Me", "Draft" and so on.
-    ///   The string may be colored by having a look at text1_meaning.
-    ///   If there is no such name or it should not be displayed, the element is NULL.
-    /// - dc_lot_t::text1_meaning: one of DC_TEXT1_USERNAME, DC_TEXT1_SELF or DC_TEXT1_DRAFT.
-    ///   Typically used to show dc_lot_t::text1 with different colors. 0 if not applicable.
-    /// - dc_lot_t::text2: contains an excerpt of the message text or strings as
-    ///   "No messages".  May be NULL of there is no such text (eg. for the archive link)
-    /// - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
-    /// - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
-    //    0 if not applicable.
+    /// Returns a summary for a given chatlist index.
     pub async fn get_summary(
         &self,
         context: &Context,
         index: usize,
         chat: Option<&Chat>,
-    ) -> Result<Lot> {
+    ) -> Result<Summary> {
         // The summary is created by the chat, not by the last message.
         // This is because we may want to display drafts here or stuff as
         // "is typing".
@@ -320,14 +307,13 @@ impl Chatlist {
         Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
     }
 
+    /// Returns a summary for a given chatlist item.
     pub async fn get_summary2(
         context: &Context,
         chat_id: ChatId,
         lastmsg_id: Option<MsgId>,
         chat: Option<&Chat>,
-    ) -> Result<Lot> {
-        let mut ret = Lot::new();
-
+    ) -> Result<Summary> {
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
@@ -343,10 +329,9 @@ impl Chatlist {
                 (Some(lastmsg), None)
             } else {
                 match chat.typ {
-                    Chattype::Group | Chattype::Mailinglist => {
-                        let lastcontact =
-                            Contact::load_from_db(context, lastmsg.from_id).await.ok();
-                        (Some(lastmsg), lastcontact)
+                    Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
+                        let lastcontact = Contact::load_from_db(context, lastmsg.from_id).await?;
+                        (Some(lastmsg), Some(lastcontact))
                     }
                     Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
                 }
@@ -356,17 +341,15 @@ impl Chatlist {
         };
 
         if chat.id.is_archived_link() {
-            ret.text2 = None;
-        } else if let Some(mut lastmsg) =
-            lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED)
-        {
-            ret.fill(&mut lastmsg, chat, lastcontact.as_ref(), context)
-                .await;
+            Ok(Default::default())
+        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED) {
+            Ok(Summary::new(context, &lastmsg, chat, lastcontact.as_ref()).await)
         } else {
-            ret.text2 = Some(stock_str::no_messages(context).await);
+            Ok(Summary {
+                text: stock_str::no_messages(context).await,
+                ..Default::default()
+            })
         }
-
-        Ok(ret)
     }
 
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
@@ -380,7 +363,7 @@ pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
         .sql
         .count(
             "SELECT COUNT(*) FROM chats WHERE blocked!=? AND archived=?;",
-            paramsv![Blocked::Manually, ChatVisibility::Archived],
+            paramsv![Blocked::Yes, ChatVisibility::Archived],
         )
         .await?;
     Ok(count)
@@ -637,6 +620,6 @@ mod tests {
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         let summary = chats.get_summary(&t, 0, None).await.unwrap();
-        assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
+        assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/config.rs

@@ -1,6 +1,6 @@
 //! # Key-value configuration management.
 
-use anyhow::Result;
+use anyhow::{ensure, Result};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
@@ -156,6 +156,11 @@ pub enum Config {
     #[strum(props(default = "0"))]
     NotifyAboutWrongPw,
 
+    /// If a warning about exceeding quota was shown recently,
+    /// this is the percentage of quota at the time the warning was given.
+    /// Unset, when quota falls below minimal warning threshold again.
+    QuotaExceeding,
+
     /// address to webrtc instance to use for videochats
     WebrtcInstance,
 
@@ -165,6 +170,16 @@ pub enum Config {
     /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
     #[strum(props(default = "60"))]
     ScanAllFoldersDebounceSecs,
+
+    /// Defines the max. size (in bytes) of messages downloaded automatically.
+    /// 0 = no limit.
+    #[strum(props(default = "0"))]
+    DownloadLimit,
+
+    /// Send sync messages, requires `BccSelf` to be set as well.
+    /// In a future versions, this switch may be removed.
+    #[strum(props(default = "0"))]
+    SendSyncMsgs,
 }
 
 impl Context {
@@ -269,13 +284,13 @@ impl Context {
                         let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
                         blob.recode_to_avatar_size(self).await?;
                         self.sql.set_raw_config(key, Some(blob.as_name())).await?;
-                        Ok(())
                     }
                     None => {
                         self.sql.set_raw_config(key, None).await?;
-                        Ok(())
                     }
                 }
+                self.emit_event(EventType::SelfavatarChanged);
+                Ok(())
             }
             Config::Selfstatus => {
                 let def = stock_str::status_line(self).await;
@@ -312,7 +327,7 @@ impl Context {
                     .set_raw_config(key, value)
                     .await
                     .map_err(Into::into);
-                job::schedule_resync(self).await;
+                job::schedule_resync(self).await?;
                 ret
             }
             _ => {
@@ -327,6 +342,21 @@ impl Context {
             .await?;
         Ok(())
     }
+
+    /// Sets an ui-specific key-value pair.
+    /// Keys must be prefixed by `ui.`
+    /// and should be followed by the name of the system and maybe subsystem,
+    /// eg. `ui.desktop.linux.foo`, `ui.desktop.macos.bar`, `ui.ios.foobar`.
+    pub async fn set_ui_config(&self, key: &str, value: Option<&str>) -> Result<()> {
+        ensure!(key.starts_with("ui."), "set_ui_config(): prefix missing.");
+        self.sql.set_raw_config(key, value).await
+    }
+
+    /// Gets an ui-specific value set by set_ui_config().
+    pub async fn get_ui_config(&self, key: &str) -> Result<Option<String>> {
+        ensure!(key.starts_with("ui."), "get_ui_config(): prefix missing.");
+        self.sql.get_raw_config(key).await
+    }
 }
 
 /// Returns all available configuration keys concated together.
@@ -379,4 +409,25 @@ mod tests {
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
         assert_eq!(media_quality, constants::MediaQuality::Worse);
     }
+
+    #[async_std::test]
+    async fn test_ui_config() -> Result<()> {
+        let t = TestContext::new().await;
+
+        assert_eq!(t.get_ui_config("ui.desktop.linux.systray").await?, None);
+
+        t.set_ui_config("ui.android.screen_security", Some("safe"))
+            .await?;
+        assert_eq!(
+            t.get_ui_config("ui.android.screen_security").await?,
+            Some("safe".to_string())
+        );
+
+        t.set_ui_config("ui.android.screen_security", None).await?;
+        assert_eq!(t.get_ui_config("ui.android.screen_security").await?, None);
+
+        assert!(t.set_ui_config("configured", Some("bar")).await.is_err());
+
+        Ok(())
+    }
 }

src/configure.rs

@@ -14,8 +14,7 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
-use crate::login_param::Socks5Config;
-use crate::login_param::{LoginParam, ServerLoginParam};
+use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
 use crate::message::Message;
 use crate::oauth2::dc_get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
@@ -250,6 +249,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                                         }
                                     }
                                 },
+                                strict_tls: Some(provider.strict_tls),
                             })
                             .collect();
 
@@ -290,6 +290,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.imap.port,
             socket: param.imap.security,
             username: param.imap.user.clone(),
+            strict_tls: None,
         })
     }
     if !servers
@@ -302,8 +303,24 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.smtp.port,
             socket: param.smtp.security,
             username: param.smtp.user.clone(),
+            strict_tls: None,
         })
     }
+
+    // respect certificate setting from function parameters
+    for mut server in &mut servers {
+        let certificate_checks = match server.protocol {
+            Protocol::Imap => param.imap.certificate_checks,
+            Protocol::Smtp => param.smtp.certificate_checks,
+        };
+        server.strict_tls = match certificate_checks {
+            CertificateChecks::AcceptInvalidCertificates
+            | CertificateChecks::AcceptInvalidCertificates2 => Some(false),
+            CertificateChecks::Strict => Some(true),
+            CertificateChecks::Automatic => server.strict_tls,
+        };
+    }
+
     let servers = expand_param_vector(servers, &param.addr, &param_domain);
 
     progress!(ctx, 550);
@@ -319,7 +336,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         .filter(|params| params.protocol == Protocol::Smtp)
         .cloned()
         .collect();
-    let provider_strict_tls = param.provider.map_or(false, |provider| provider.strict_tls);
+    let provider_strict_tls = param
+        .provider
+        .map_or(socks5_config.is_some(), |provider| provider.strict_tls);
 
     let smtp_config_task = task::spawn(async move {
         let mut smtp_configured = false;
@@ -329,6 +348,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             smtp_param.server = smtp_server.hostname.clone();
             smtp_param.port = smtp_server.port;
             smtp_param.security = smtp_server.socket;
+            smtp_param.certificate_checks = match smtp_server.strict_tls {
+                Some(true) => CertificateChecks::Strict,
+                Some(false) => CertificateChecks::AcceptInvalidCertificates,
+                None => CertificateChecks::Automatic,
+            };
 
             match try_smtp_one_param(
                 &context_smtp,
@@ -372,6 +396,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         param.imap.server = imap_server.hostname.clone();
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
+        param.imap.certificate_checks = match imap_server.strict_tls {
+            Some(true) => CertificateChecks::Strict,
+            Some(false) => CertificateChecks::AcceptInvalidCertificates,
+            None => CertificateChecks::Automatic,
+        };
 
         match try_imap_one_param(
             ctx,
@@ -442,7 +471,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         ctx,
         job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
     )
-    .await;
+    .await?;
 
     progress!(ctx, 940);
     update_device_chats_handle.await?;

src/configure/auto_mozilla.rs

@@ -243,6 +243,7 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
                 hostname: server.hostname,
                 port: server.port,
                 username: server.username,
+                strict_tls: None,
             })
         })
         .collect();

src/configure/auto_outlook.rs

@@ -180,6 +180,7 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
                 hostname: protocol.server,
                 port: protocol.port,
                 username: String::new(),
+                strict_tls: None,
             })
         })
         .collect()

src/configure/server_params.rs

@@ -22,6 +22,9 @@ pub(crate) struct ServerParams {
 
     /// Username, empty if unknown.
     pub username: String,
+
+    /// Whether TLS certificates should be strictly checked or not, `None` for automatic.
+    pub strict_tls: Option<bool>,
 }
 
 impl ServerParams {
@@ -128,6 +131,23 @@ impl ServerParams {
             vec![self]
         }
     }
+
+    fn expand_strict_tls(self) -> Vec<ServerParams> {
+        if self.strict_tls.is_none() {
+            vec![
+                Self {
+                    strict_tls: Some(true), // Strict.
+                    ..self.clone()
+                },
+                Self {
+                    strict_tls: None, // Automatic.
+                    ..self
+                },
+            ]
+        } else {
+            vec![self]
+        }
+    }
 }
 
 /// Expands vector of `ServerParams`, replacing placeholders with
@@ -138,10 +158,32 @@ pub(crate) fn expand_param_vector(
     domain: &str,
 ) -> Vec<ServerParams> {
     v.into_iter()
-        // The order of expansion is important: ports are expanded the
-        // last, so they are changed the first. Username is only
-        // changed if default value (address with domain) didn't work
-        // for all available hosts and ports.
+        .map(|params| {
+            if params.socket == Socket::Plain {
+                ServerParams {
+                    // Avoid expanding plaintext configuration into configuration with and without
+                    // `strict_tls` if `strict_tls` is set to `None` as `strict_tls` is not used for
+                    // plaintext connections. Always setting it to "enabled", just in case.
+                    strict_tls: Some(true),
+                    ..params
+                }
+            } else {
+                params
+            }
+        })
+        // The order of expansion is important.
+        //
+        // Ports are expanded the last, so they are changed the first.  Username is only changed if
+        // default value (address with domain) didn't work for all available hosts and ports.
+        //
+        // Strict TLS must be expanded first, so we try all configurations with strict TLS first
+        // and only then try again without strict TLS. Otherwise we may lock to wrong hostname
+        // without strict TLS when another hostname with strict TLS is available.  For example, if
+        // both smtp.example.net and mail.example.net are running an SMTP server, but both use a
+        // certificate that is only valid for mail.example.net, we want to skip smtp.example.net
+        // and use mail.example.net with strict TLS instead of using smtp.example.net without
+        // strict TLS.
+        .flat_map(|params| params.expand_strict_tls().into_iter())
         .flat_map(|params| params.expand_usernames(addr).into_iter())
         .flat_map(|params| params.expand_hostnames(domain).into_iter())
         .flat_map(|params| params.expand_ports().into_iter())
@@ -161,6 +203,7 @@ mod tests {
                 port: 0,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
+                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -174,6 +217,7 @@ mod tests {
                 port: 993,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
+                strict_tls: Some(true)
             }],
         );
 
@@ -184,6 +228,7 @@ mod tests {
                 port: 123,
                 socket: Socket::Automatic,
                 username: "foobar".to_string(),
+                strict_tls: None,
             }],
             "foobar@example.net",
             "example.net",
@@ -197,16 +242,59 @@ mod tests {
                     hostname: "example.net".to_string(),
                     port: 123,
                     socket: Socket::Ssl,
-                    username: "foobar".to_string()
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true),
                 },
                 ServerParams {
                     protocol: Protocol::Smtp,
                     hostname: "example.net".to_string(),
                     port: 123,
                     socket: Socket::Starttls,
-                    username: "foobar".to_string()
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                },
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: None,
+                },
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Starttls,
+                    username: "foobar".to_string(),
+                    strict_tls: None
                 }
             ],
         );
+
+        // Test that strict_tls is not expanded for plaintext connections.
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Smtp,
+                hostname: "example.net".to_string(),
+                port: 123,
+                socket: Socket::Plain,
+                username: "foobar".to_string(),
+                strict_tls: None,
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+        assert_eq!(
+            v,
+            vec![ServerParams {
+                protocol: Protocol::Smtp,
+                hostname: "example.net".to_string(),
+                port: 123,
+                socket: Socket::Plain,
+                username: "foobar".to_string(),
+                strict_tls: Some(true)
+            }],
+        );
     }
 }

src/constants.rs

@@ -24,7 +24,7 @@ pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION")
 #[repr(i8)]
 pub enum Blocked {
     Not = 0,
-    Manually = 1,
+    Yes = 1,
     Request = 2,
 }
 
@@ -153,6 +153,7 @@ pub enum Chattype {
     Single = 100,
     Group = 120,
     Mailinglist = 140,
+    Broadcast = 160,
 }
 
 impl Default for Chattype {
@@ -348,6 +349,7 @@ mod tests {
         assert_eq!(Chattype::Single, Chattype::from_i32(100).unwrap());
         assert_eq!(Chattype::Group, Chattype::from_i32(120).unwrap());
         assert_eq!(Chattype::Mailinglist, Chattype::from_i32(140).unwrap());
+        assert_eq!(Chattype::Broadcast, Chattype::from_i32(160).unwrap());
     }
 
     #[test]
@@ -383,7 +385,7 @@ mod tests {
         // values may be written to disk and must not change
         assert_eq!(Blocked::Not, Blocked::default());
         assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
-        assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
+        assert_eq!(Blocked::Yes, Blocked::from_i32(1).unwrap());
         assert_eq!(Blocked::Request, Blocked::from_i32(2).unwrap());
     }
 

src/contact.rs

@@ -2,7 +2,7 @@
 
 use std::convert::{TryFrom, TryInto};
 
-use anyhow::{bail, ensure, format_err, Result};
+use anyhow::{bail, ensure, Context as _, Result};
 use async_std::path::PathBuf;
 use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
@@ -174,6 +174,12 @@ pub enum VerifiedStatus {
     BidirectVerified = 2,
 }
 
+impl Default for VerifiedStatus {
+    fn default() -> Self {
+        Self::Unverified
+    }
+}
+
 impl Contact {
     pub async fn load_from_db(context: &Context, contact_id: u32) -> Result<Self> {
         let mut contact = context
@@ -229,11 +235,9 @@ impl Contact {
     }
 
     /// Check if a contact is blocked.
-    pub async fn is_blocked_load(context: &Context, id: u32) -> bool {
-        Self::load_from_db(context, id)
-            .await
-            .map(|contact| contact.blocked)
-            .unwrap_or_default()
+    pub async fn is_blocked_load(context: &Context, id: u32) -> Result<bool> {
+        let blocked = Self::load_from_db(context, id).await?.blocked;
+        Ok(blocked)
     }
 
     /// Block the given contact.
@@ -263,7 +267,7 @@ impl Contact {
 
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated).await?;
-        let blocked = Contact::is_blocked_load(context, contact_id).await;
+        let blocked = Contact::is_blocked_load(context, contact_id).await?;
         match sth_modified {
             Modifier::None => {}
             Modifier::Modified | Modifier::Created => {
@@ -707,7 +711,7 @@ impl Contact {
             .sql
             .query_map(
                 "SELECT name, grpid FROM chats WHERE type=? AND blocked=?;",
-                paramsv![Chattype::Mailinglist, Blocked::Manually],
+                paramsv![Chattype::Mailinglist, Blocked::Yes],
                 |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
                 |rows| {
                     rows.collect::<std::result::Result<Vec<_>, _>>()
@@ -754,12 +758,9 @@ impl Contact {
 
     /// Get blocked contacts.
     pub async fn get_all_blocked(context: &Context) -> Result<Vec<u32>> {
-        if let Err(e) = Contact::update_blocked_mailinglist_contacts(context).await {
-            warn!(
-                context,
-                "Cannot update blocked mailinglist contacts: {:?}", e
-            );
-        }
+        Contact::update_blocked_mailinglist_contacts(context)
+            .await
+            .context("cannot update blocked mailinglist contacts")?;
 
         let list = context
             .sql
@@ -1018,7 +1019,7 @@ impl Contact {
     ///
     /// The UI may draw a checkbox or something like that beside verified contacts.
     ///
-    pub async fn is_verified(&self, context: &Context) -> VerifiedStatus {
+    pub async fn is_verified(&self, context: &Context) -> Result<VerifiedStatus> {
         self.is_verified_ex(context, None).await
     }
 
@@ -1029,54 +1030,46 @@ impl Contact {
         &self,
         context: &Context,
         peerstate: Option<&Peerstate>,
-    ) -> VerifiedStatus {
+    ) -> Result<VerifiedStatus> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
         if self.id == DC_CONTACT_ID_SELF {
-            return VerifiedStatus::BidirectVerified;
+            return Ok(VerifiedStatus::BidirectVerified);
         }
 
         if let Some(peerstate) = peerstate {
             if peerstate.verified_key.is_some() {
-                return VerifiedStatus::BidirectVerified;
+                return Ok(VerifiedStatus::BidirectVerified);
             }
         }
 
-        let peerstate = match Peerstate::from_addr(context, &self.addr).await {
-            Ok(peerstate) => peerstate,
-            Err(err) => {
-                warn!(
-                    context,
-                    "Failed to load peerstate for address {}: {}", self.addr, err
-                );
-                return VerifiedStatus::Unverified;
-            }
-        };
-
-        if let Some(ps) = peerstate {
-            if ps.verified_key.is_some() {
-                return VerifiedStatus::BidirectVerified;
+        if let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? {
+            if peerstate.verified_key.is_some() {
+                return Ok(VerifiedStatus::BidirectVerified);
             }
         }
 
-        VerifiedStatus::Unverified
+        Ok(VerifiedStatus::Unverified)
     }
 
-    pub async fn addr_equals_contact(context: &Context, addr: &str, contact_id: u32) -> bool {
+    pub async fn addr_equals_contact(
+        context: &Context,
+        addr: &str,
+        contact_id: u32,
+    ) -> Result<bool> {
         if addr.is_empty() {
-            return false;
+            return Ok(false);
         }
 
-        if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            if !contact.addr.is_empty() {
-                let normalized_addr = addr_normalize(addr);
-                if contact.addr == normalized_addr {
-                    return true;
-                }
+        let contact = Contact::load_from_db(context, contact_id).await?;
+        if !contact.addr.is_empty() {
+            let normalized_addr = addr_normalize(addr);
+            if contact.addr == normalized_addr {
+                return Ok(true);
             }
         }
 
-        false
+        Ok(false)
     }
 
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
@@ -1094,30 +1087,34 @@ impl Contact {
         Ok(count)
     }
 
-    pub async fn real_exists_by_id(context: &Context, contact_id: u32) -> bool {
-        if !context.sql.is_open().await || contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
-            return false;
+    pub async fn real_exists_by_id(context: &Context, contact_id: u32) -> Result<bool> {
+        if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+            return Ok(false);
         }
 
-        context
+        let exists = context
             .sql
             .exists(
                 "SELECT COUNT(*) FROM contacts WHERE id=?;",
                 paramsv![contact_id as i32],
             )
-            .await
-            .unwrap_or_default()
+            .await?;
+        Ok(exists)
     }
 
-    pub async fn scaleup_origin_by_id(context: &Context, contact_id: u32, origin: Origin) -> bool {
+    pub async fn scaleup_origin_by_id(
+        context: &Context,
+        contact_id: u32,
+        origin: Origin,
+    ) -> Result<()> {
         context
             .sql
             .execute(
                 "UPDATE contacts SET origin=? WHERE id=? AND origin<?;",
                 paramsv![origin, contact_id as i32, origin],
             )
-            .await
-            .is_ok()
+            .await?;
+        Ok(())
     }
 }
 
@@ -1342,12 +1339,11 @@ fn cat_fingerprint(
 impl Context {
     /// determine whether the specified addr maps to the/a self addr
     pub async fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        let self_addr = self
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .ok_or_else(|| format_err!("Not configured"))?;
-
-        Ok(addr_cmp(self_addr, addr))
+        if let Some(self_addr) = self.get_config(Config::ConfiguredAddr).await? {
+            Ok(addr_cmp(self_addr, addr))
+        } else {
+            Ok(false)
+        }
     }
 }
 
@@ -1375,11 +1371,14 @@ fn split_address_book(book: &str) -> Vec<(&str, &str)> {
 
 #[cfg(test)]
 mod tests {
+    use async_std::fs::File;
+    use async_std::io::WriteExt;
+
     use super::*;
 
     use crate::chat::send_text_msg;
     use crate::message::Message;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::{self, TestContext};
 
     #[test]
     fn test_may_be_valid_addr() {
@@ -1494,7 +1493,7 @@ mod tests {
     #[async_std::test]
     async fn test_is_self_addr() -> Result<()> {
         let t = TestContext::new().await;
-        assert!(t.is_self_addr("me@me.org").await.is_err());
+        assert_eq!(t.is_self_addr("me@me.org").await?, false);
 
         let addr = t.configure_alice().await;
         assert_eq!(t.is_self_addr("me@me.org").await?, false);
@@ -1975,4 +1974,64 @@ CCCB 5AA9 F6E1 141C 9431
 
         Ok(())
     }
+
+    /// Tests that DC_EVENT_SELFAVATAR_CHANGED is emitted on avatar changes.
+    #[async_std::test]
+    async fn test_selfavatar_changed_event() -> Result<()> {
+        // Alice has two devices.
+        let alice1 = TestContext::new_alice().await;
+        let alice2 = TestContext::new_alice().await;
+
+        // Bob has one device.
+        let bob = TestContext::new_bob().await;
+
+        assert_eq!(alice1.get_config(Config::Selfavatar).await?, None);
+
+        let avatar_src = alice1.get_blobdir().join("avatar.png");
+        File::create(&avatar_src)
+            .await?
+            .write_all(test_utils::AVATAR_900x900_BYTES)
+            .await?;
+
+        alice1
+            .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await?;
+
+        alice1
+            .evtracker
+            .get_matching(|e| e == EventType::SelfavatarChanged)
+            .await;
+
+        // Bob sends a message so that Alice can encrypt to him.
+        let chat = bob
+            .create_chat_with_contact("Alice", "alice@example.com")
+            .await;
+
+        send_text_msg(&bob, chat.id, "Reply".to_string()).await?;
+        let sent_msg = bob.pop_sent_msg().await;
+        alice1.recv_msg(&sent_msg).await;
+        alice2.recv_msg(&sent_msg).await;
+
+        // Alice sends a message.
+        let alice1_chat_id = alice1.get_last_msg().await.chat_id;
+        alice1_chat_id.accept(&alice1).await?;
+        send_text_msg(&alice1, alice1_chat_id, "Hello".to_string()).await?;
+        let sent_msg = alice1.pop_sent_msg().await;
+
+        // The message is encrypted.
+        let message = Message::load_from_db(&alice1, sent_msg.sender_msg_id).await?;
+        assert!(message.get_showpadlock());
+
+        // Alice's second device receives a copy of the outgoing message.
+        alice2.recv_msg(&sent_msg).await;
+
+        // Alice's second device applies the selfavatar.
+        assert!(alice2.get_config(Config::Selfavatar).await?.is_some());
+        alice2
+            .evtracker
+            .get_matching(|e| e == EventType::SelfavatarChanged)
+            .await;
+
+        Ok(())
+    }
 }

src/context.rs

@@ -76,6 +76,11 @@ pub struct InnerContext {
     pub(crate) id: u32,
 
     creation_time: SystemTime,
+
+    /// The text of the last error logged and emitted as an event.
+    /// If the ui wants to display an error after a failure,
+    /// `last_error` should be used to avoid races with the event thread.
+    pub(crate) last_error: RwLock<String>,
 }
 
 #[derive(Debug)]
@@ -147,6 +152,7 @@ impl Context {
             quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
+            last_error: RwLock::new("".to_string()),
         };
 
         let ctx = Context {
@@ -303,6 +309,7 @@ impl Context {
         let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await?;
         let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await?;
         let bcc_self = self.get_config_int(Config::BccSelf).await?;
+        let send_sync_msgs = self.get_config_int(Config::SendSyncMsgs).await?;
 
         let prv_key_cnt = self
             .sql
@@ -371,6 +378,12 @@ impl Context {
             "show_emails",
             self.get_config_int(Config::ShowEmails).await?.to_string(),
         );
+        res.insert(
+            "download_limit",
+            self.get_config_int(Config::DownloadLimit)
+                .await?
+                .to_string(),
+        );
         res.insert("inbox_watch", inbox_watch.to_string());
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_watch", mvbox_watch.to_string());
@@ -386,6 +399,7 @@ impl Context {
             self.get_config_int(Config::KeyGenType).await?.to_string(),
         );
         res.insert("bcc_self", bcc_self.to_string());
+        res.insert("send_sync_msgs", send_sync_msgs.to_string());
         res.insert("private_key_count", prv_key_cnt.to_string());
         res.insert("public_key_count", pub_key_cnt.to_string());
         res.insert("fingerprint", fingerprint_str);
@@ -423,6 +437,12 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "quota_exceeding",
+            self.get_config_int(Config::QuotaExceeding)
+                .await?
+                .to_string(),
+        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));

src/dc_tools.rs

@@ -84,9 +84,9 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
 //   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
 const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 
-// returns the currently smeared timestamp,
-// may be used to check if call to dc_create_smeared_timestamp() is needed or not.
-// the returned timestamp MUST NOT be used to be sent out or saved in the database!
+/// Returns the current smeared timestamp,
+///
+/// The returned timestamp MUST NOT be sent out.
 pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
     let mut now = time();
     let ts = *context.last_smeared_timestamp.read().await;
@@ -97,7 +97,7 @@ pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
     now
 }
 
-// returns a timestamp that is guaranteed to be unique.
+/// Returns a timestamp that is guaranteed to be unique.
 pub(crate) async fn dc_create_smeared_timestamp(context: &Context) -> i64 {
     let now = time();
     let mut ret = now;
@@ -835,8 +835,8 @@ mod tests {
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
-    use crate::chat;
     use crate::chatlist::Chatlist;
+    use crate::{chat, test_utils};
     use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
     use proptest::prelude::*;
 
@@ -844,22 +844,18 @@ mod tests {
         #[test]
         fn test_dc_truncate(
             buf: String,
-            approx_chars in 0..10000usize
+            approx_chars in 0..100usize
         ) {
             let res = dc_truncate(&buf, approx_chars);
             let el_len = 5;
             let l = res.chars().count();
-            if approx_chars > 0 {
-                assert!(
-                    l <= approx_chars + el_len,
-                    "buf: '{}' - res: '{}' - len {}, approx {}",
-                    &buf, &res, res.len(), approx_chars
-                );
-            } else {
-                assert_eq!(&res, &buf);
-            }
+            assert!(
+                l <= approx_chars + el_len,
+                "buf: '{}' - res: '{}' - len {}, approx {}",
+                &buf, &res, res.len(), approx_chars
+            );
 
-            if approx_chars > 0 && buf.chars().count() > approx_chars + el_len {
+            if buf.chars().count() > approx_chars + el_len {
                 let l = res.len();
                 assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
             }
@@ -986,8 +982,7 @@ mod tests {
 
     #[test]
     fn test_get_filemeta() {
-        let data = include_bytes!("../test-data/image/avatar900x900.png");
-        let (w, h) = dc_get_filemeta(data).unwrap();
+        let (w, h) = dc_get_filemeta(test_utils::AVATAR_900x900_BYTES).unwrap();
         assert_eq!(w, 900);
         assert_eq!(h, 900);
 

src/download.rs

@@ -0,0 +1,346 @@
+//! # Download large messages manually.
+
+use anyhow::{anyhow, Result};
+use deltachat_derive::{FromSql, ToSql};
+use serde::{Deserialize, Serialize};
+
+use crate::config::Config;
+use crate::constants::Viewtype;
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::imap::{Imap, ImapActionResult};
+use crate::job::{self, Action, Job, Status};
+use crate::message::{Message, MsgId};
+use crate::mimeparser::{MimeMessage, Part};
+use crate::param::Params;
+use crate::{job_try, stock_str, EventType};
+use std::cmp::max;
+
+/// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`.
+///
+/// Some messages as non-delivery-reports (NDN) or read-receipts (MDN)
+/// need to be downloaded completely to handle them correctly,
+/// eg. to assign them to the correct chat.
+/// As these messages are typically small,
+/// they're catched by `MIN_DOWNLOAD_LIMIT`.
+const MIN_DOWNLOAD_LIMIT: u32 = 32768;
+
+/// If a message is downloaded only partially
+/// and `delete_server_after` is set to small timeouts (eg. "at once"),
+/// the user might have no chance to actually download that message.
+/// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case.
+pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
+
+#[derive(
+    Debug,
+    Display,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    FromPrimitive,
+    ToPrimitive,
+    FromSql,
+    ToSql,
+    Serialize,
+    Deserialize,
+)]
+#[repr(u32)]
+pub enum DownloadState {
+    Done = 0,
+    Available = 10,
+    Failure = 20,
+    InProgress = 1000,
+}
+
+impl Default for DownloadState {
+    fn default() -> Self {
+        DownloadState::Done
+    }
+}
+
+impl Context {
+    // Returns validated download limit or `None` for "no limit".
+    pub(crate) async fn download_limit(&self) -> Result<Option<u32>> {
+        let download_limit = self.get_config_int(Config::DownloadLimit).await?;
+        if download_limit <= 0 {
+            Ok(None)
+        } else {
+            Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32)))
+        }
+    }
+}
+
+impl MsgId {
+    /// Schedules full message download for partially downloaded message.
+    pub async fn download_full(self, context: &Context) -> Result<()> {
+        let msg = Message::load_from_db(context, self).await?;
+        match msg.download_state() {
+            DownloadState::Done => return Err(anyhow!("Nothing to download.")),
+            DownloadState::InProgress => return Err(anyhow!("Download already in progress.")),
+            DownloadState::Available | DownloadState::Failure => {
+                self.update_download_state(context, DownloadState::InProgress)
+                    .await?;
+                job::add(
+                    context,
+                    Job::new(Action::DownloadMsg, self.to_u32(), Params::new(), 0),
+                )
+                .await?;
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) async fn update_download_state(
+        self,
+        context: &Context,
+        download_state: DownloadState,
+    ) -> Result<()> {
+        let msg = Message::load_from_db(context, self).await?;
+        context
+            .sql
+            .execute(
+                "UPDATE msgs SET download_state=? WHERE id=?;",
+                paramsv![download_state, self],
+            )
+            .await?;
+        context.emit_event(EventType::MsgsChanged {
+            chat_id: msg.chat_id,
+            msg_id: self,
+        });
+        Ok(())
+    }
+}
+
+impl Message {
+    /// Returns the download state of the message.
+    pub fn download_state(&self) -> DownloadState {
+        self.download_state
+    }
+}
+
+impl Job {
+    /// Actually download a message.
+    /// Called in response to `Action::DownloadMsg`.
+    pub(crate) async fn download_msg(&self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.prepare(context).await {
+            warn!(context, "download: could not connect: {:?}", err);
+            return Status::RetryNow;
+        }
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let server_folder = msg.server_folder.unwrap_or_default();
+        match imap
+            .fetch_single_msg(context, &server_folder, msg.server_uid)
+            .await
+        {
+            ImapActionResult::RetryLater | ImapActionResult::Failed => {
+                job_try!(
+                    msg.id
+                        .update_download_state(context, DownloadState::Failure)
+                        .await
+                );
+                Status::Finished(Err(anyhow!("Call download_full() again to try over.")))
+            }
+            ImapActionResult::Success | ImapActionResult::AlreadyDone => {
+                // update_download_state() not needed as receive_imf() already
+                // set the state and emitted the event.
+                Status::Finished(Ok(()))
+            }
+        }
+    }
+}
+
+impl Imap {
+    /// Download a single message and pipe it to receive_imf().
+    ///
+    /// receive_imf() is not directly aware that this is a result of a call to download_msg(),
+    /// however, implicitly knows that as the existing message is flagged as being partly.
+    async fn fetch_single_msg(
+        &mut self,
+        context: &Context,
+        folder: &str,
+        uid: u32,
+    ) -> ImapActionResult {
+        if let Some(imapresult) = self
+            .prepare_imap_operation_on_msg(context, folder, uid)
+            .await
+        {
+            return imapresult;
+        }
+
+        // we are connected, and the folder is selected
+        info!(context, "Downloading message {}/{} fully...", folder, uid);
+
+        let (_, error_cnt) = self
+            .fetch_many_msgs(context, folder, vec![uid], false, false)
+            .await;
+        if error_cnt > 0 {
+            return ImapActionResult::Failed;
+        }
+
+        ImapActionResult::Success
+    }
+}
+
+impl MimeMessage {
+    /// Creates a placeholder part and add that to `parts`.
+    ///
+    /// To create the placeholder, only the outermost header can be used,
+    /// the mime-structure itself is not available.
+    ///
+    /// The placeholder part currently contains a text with size and availability of the message;
+    /// in the future, we may do more advanced things as previews here.
+    pub(crate) async fn create_stub_from_partial_download(
+        &mut self,
+        context: &Context,
+        org_bytes: u32,
+    ) -> Result<()> {
+        let mut text = format!(
+            "[{}]",
+            stock_str::partial_download_msg_body(context, org_bytes).await
+        );
+        if let Some(delete_server_after) = context.get_config_delete_server_after().await? {
+            let until = stock_str::download_availability(
+                context,
+                time() + max(delete_server_after, MIN_DELETE_SERVER_AFTER),
+            )
+            .await;
+            text += format!(" [{}]", until).as_str();
+        };
+
+        info!(context, "Partial download: {}", text);
+
+        self.parts.push(Part {
+            typ: Viewtype::Text,
+            msg: text,
+            ..Default::default()
+        });
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chat::send_msg;
+    use crate::constants::Viewtype;
+    use crate::dc_receive_imf::dc_receive_imf_inner;
+    use crate::test_utils::TestContext;
+    use num_traits::FromPrimitive;
+
+    #[test]
+    fn test_downloadstate_values() {
+        // values may be written to disk and must not change
+        assert_eq!(DownloadState::Done, DownloadState::default());
+        assert_eq!(DownloadState::Done, DownloadState::from_i32(0).unwrap());
+        assert_eq!(
+            DownloadState::Available,
+            DownloadState::from_i32(10).unwrap()
+        );
+        assert_eq!(DownloadState::Failure, DownloadState::from_i32(20).unwrap());
+        assert_eq!(
+            DownloadState::InProgress,
+            DownloadState::from_i32(1000).unwrap()
+        );
+    }
+
+    #[async_std::test]
+    async fn test_download_limit() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        assert_eq!(t.download_limit().await?, None);
+
+        t.set_config(Config::DownloadLimit, Some("200000")).await?;
+        assert_eq!(t.download_limit().await?, Some(200000));
+
+        t.set_config(Config::DownloadLimit, Some("20000")).await?;
+        assert_eq!(t.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
+
+        t.set_config(Config::DownloadLimit, None).await?;
+        assert_eq!(t.download_limit().await?, None);
+
+        for val in &["0", "-1", "-100", "", "foo"] {
+            t.set_config(Config::DownloadLimit, Some(val)).await?;
+            assert_eq!(t.download_limit().await?, None);
+        }
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_update_download_state() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat = t.create_chat_with_contact("Bob", "bob@example.org").await;
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("Hi Bob".to_owned()));
+        let msg_id = send_msg(&t, chat.id, &mut msg).await?;
+        let msg = Message::load_from_db(&t, msg_id).await?;
+        assert_eq!(msg.download_state(), DownloadState::Done);
+
+        for s in &[
+            DownloadState::Available,
+            DownloadState::InProgress,
+            DownloadState::Failure,
+            DownloadState::Done,
+        ] {
+            msg_id.update_download_state(&t, *s).await?;
+            let msg = Message::load_from_db(&t, msg_id).await?;
+            assert_eq!(msg.download_state(), *s);
+        }
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_partial_receive_imf() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        let header =
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+             From: bob@example.com\n\
+             To: alice@example.org\n\
+             Subject: foo\n\
+             Message-ID: <Mr.12345678901@example.com>\n\
+             Chat-Version: 1.0\n\
+             Date: Sun, 22 Mar 2020 22:37:57 +0000\
+             Content-Type: text/plain";
+
+        dc_receive_imf_inner(
+            &t,
+            header.as_bytes(),
+            "INBOX",
+            1,
+            false,
+            Some(100000),
+            false,
+        )
+        .await?;
+        let msg = t.get_last_msg().await;
+        assert_eq!(msg.download_state(), DownloadState::Available);
+        assert_eq!(msg.get_subject(), "foo");
+        assert!(msg
+            .get_text()
+            .unwrap()
+            .contains(&stock_str::partial_download_msg_body(&t, 100000).await));
+
+        dc_receive_imf_inner(
+            &t,
+            format!("{}\n\n100k text...", header).as_bytes(),
+            "INBOX",
+            1,
+            false,
+            None,
+            false,
+        )
+        .await?;
+        let msg = t.get_last_msg().await;
+        assert_eq!(msg.download_state(), DownloadState::Done);
+        assert_eq!(msg.get_subject(), "foo");
+        assert_eq!(msg.get_text(), Some("100k text...".to_string()));
+
+        Ok(())
+    }
+}

src/e2ee.rs

@@ -161,15 +161,19 @@ pub async fn try_decrypt(
     let mut peerstate = Peerstate::from_addr(context, &from).await?;
 
     // Apply Autocrypt header
-    if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
-        if let Some(ref mut peerstate) = peerstate {
-            peerstate.apply_header(header, message_time);
-            peerstate.save_to_db(&context.sql, false).await?;
-        } else {
-            let p = Peerstate::from_header(header, message_time);
-            p.save_to_db(&context.sql, true).await?;
-            peerstate = Some(p);
+    match Aheader::from_headers(&from, &mail.headers) {
+        Ok(Some(ref header)) => {
+            if let Some(ref mut peerstate) = peerstate {
+                peerstate.apply_header(header, message_time);
+                peerstate.save_to_db(&context.sql, false).await?;
+            } else {
+                let p = Peerstate::from_header(header, message_time);
+                p.save_to_db(&context.sql, true).await?;
+                peerstate = Some(p);
+            }
         }
+        Ok(None) => {}
+        Err(err) => warn!(context, "Failed to parse Autocrypt header: {}", err),
     }
 
     // Possibly perform decryption

src/ephemeral.rs

@@ -71,11 +71,13 @@ use crate::constants::{
 };
 use crate::context::Context;
 use crate::dc_tools::time;
+use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
 use crate::job;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::stock_str;
+use std::cmp::max;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -279,7 +281,7 @@ impl MsgId {
     /// Starts ephemeral message timer for the message if it is not started yet.
     pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> anyhow::Result<()> {
         if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
-            let ephemeral_timestamp = time() + i64::from(duration);
+            let ephemeral_timestamp = time().saturating_add(duration.into());
 
             context
                 .sql
@@ -416,24 +418,18 @@ pub async fn schedule_ephemeral_task(context: &Context) {
             let context1 = context.clone();
             let ephemeral_task = task::spawn(async move {
                 async_std::task::sleep(duration).await;
-                emit_event!(
-                    context1,
-                    EventType::MsgsChanged {
-                        chat_id: ChatId::new(0),
-                        msg_id: MsgId::new(0)
-                    }
-                );
+                context1.emit_event(EventType::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0),
+                });
             });
             *context.ephemeral_task.write().await = Some(ephemeral_task);
         } else {
             // Emit event immediately
-            emit_event!(
-                context,
-                EventType::MsgsChanged {
-                    chat_id: ChatId::new(0),
-                    msg_id: MsgId::new(0)
-                }
-            );
+            context.emit_event(EventType::MsgsChanged {
+                chat_id: ChatId::new(0),
+                msg_id: MsgId::new(0),
+            });
         }
     }
 }
@@ -445,23 +441,32 @@ pub async fn schedule_ephemeral_task(context: &Context) {
 pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
     let now = time();
 
-    let threshold_timestamp = match context.get_config_delete_server_after().await? {
-        None => 0,
-        Some(delete_server_after) => now - delete_server_after,
-    };
+    let (threshold_timestamp, threshold_timestamp_extended) =
+        match context.get_config_delete_server_after().await? {
+            None => (0, 0),
+            Some(delete_server_after) => (
+                now - delete_server_after,
+                now - max(delete_server_after, MIN_DELETE_SERVER_AFTER),
+            ),
+        };
 
     context
         .sql
         .query_row_optional(
             "SELECT id FROM msgs \
          WHERE ( \
-         timestamp < ? \
+         ((download_state = 0 AND timestamp < ?) OR (download_state != 0 AND timestamp < ?)) \
          OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
          AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
          LIMIT 1",
-            paramsv![threshold_timestamp, now, job::Action::DeleteMsgOnImap],
+            paramsv![
+                threshold_timestamp,
+                threshold_timestamp_extended,
+                now,
+                job::Action::DeleteMsgOnImap
+            ],
             |row| {
                 let msg_id: MsgId = row.get(0)?;
                 Ok(msg_id)
@@ -506,6 +511,9 @@ mod tests {
     use async_std::task::sleep;
 
     use super::*;
+    use crate::config::Config;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
         chat::{self, Chat, ChatItem},
@@ -641,8 +649,83 @@ mod tests {
         );
     }
 
+    /// Test enabling and disabling ephemeral timer remotely.
     #[async_std::test]
-    async fn test_ephemeral_timer() -> anyhow::Result<()> {
+    async fn test_ephemeral_enable_disable() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
+
+        chat_alice
+            .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
+            .await?;
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        chat_alice
+            .set_ephemeral_timer(&alice.ctx, Timer::Disabled)
+            .await?;
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Disabled
+        );
+
+        Ok(())
+    }
+
+    /// Test that timer is enabled even if the message explicitly enabling the timer is lost.
+    #[async_std::test]
+    async fn test_ephemeral_enable_lost() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
+
+        // Alice enables the timer.
+        chat_alice
+            .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
+            .await?;
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+        // The message enabling the timer is lost.
+        let _sent = alice.pop_sent_msg().await;
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Disabled,
+        );
+
+        // Alice sends a text message.
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        // Bob receives text message and enables the timer, even though explicit timer update was
+        // lost previously.
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        Ok(())
+    }
+
+    /// Test that Alice replying to the chat without a timer at the same time as Bob enables the
+    /// timer does not result in disabling the timer on the Bob's side.
+    #[async_std::test]
+    async fn test_ephemeral_timer_rollback() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
@@ -700,6 +783,18 @@ mod tests {
             Timer::Enabled { duration: 60 }
         );
 
+        // Bob disables the chat timer.
+        // Note that the last message in the Bob's chat is from Alice and has no timer,
+        // but the chat timer is enabled.
+        chat_bob
+            .set_ephemeral_timer(&bob.ctx, Timer::Disabled)
+            .await?;
+        alice.recv_msg(&bob.pop_sent_msg().await).await;
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Disabled
+        );
+
         Ok(())
     }
 
@@ -777,4 +872,149 @@ mod tests {
             assert!(rawtxt.is_none_or_empty(), "{:?}", rawtxt);
         }
     }
+
+    #[async_std::test]
+    async fn test_load_imap_deletion_msgid() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        const HOUR: i64 = 60 * 60;
+        let now = time();
+        for (id, timestamp, ephemeral_timestamp) in &[
+            (900, now - 2 * HOUR, 0),
+            (1000, now - 23 * HOUR - MIN_DELETE_SERVER_AFTER, 0),
+            (1010, now - 23 * HOUR, 0),
+            (1020, now - 21 * HOUR, 0),
+            (1030, now - 19 * HOUR, 0),
+            (2000, now - 18 * HOUR, now - HOUR),
+            (2020, now - 17 * HOUR, now + HOUR),
+        ] {
+            t.sql
+                .execute(
+                    "INSERT INTO msgs (id, server_uid, timestamp, ephemeral_timestamp) VALUES (?,?,?,?);",
+                    paramsv![id, id, timestamp, ephemeral_timestamp],
+                )
+                .await?;
+        }
+
+        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(2000)));
+
+        MsgId::new(2000).delete_from_db(&t).await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
+
+        t.set_config(Config::DeleteServerAfter, Some(&*(25 * HOUR).to_string()))
+            .await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1000)));
+
+        MsgId::new(1000)
+            .update_download_state(&t, DownloadState::Available)
+            .await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1000))); // delete downloadable anyway
+
+        MsgId::new(1000).delete_from_db(&t).await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
+
+        t.set_config(Config::DeleteServerAfter, Some(&*(22 * HOUR).to_string()))
+            .await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1010)));
+
+        MsgId::new(1010)
+            .update_download_state(&t, DownloadState::Available)
+            .await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, None); // keep downloadable for now
+
+        MsgId::new(1010).delete_from_db(&t).await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
+
+        Ok(())
+    }
+
+    // Regression test for a bug in the timer rollback protection.
+    #[async_std::test]
+    async fn test_ephemeral_timer_references() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        // Message with Message-ID <first@example.com> and no timer is received.
+        dc_receive_imf(
+            &alice,
+            b"From: Bob <bob@example.com>\n\
+                    To: Alice <alice@example.com>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: Subject\n\
+                    Message-ID: <first@example.com>\n\
+                    Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
+                    \n\
+                    hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await?;
+
+        let msg = alice.get_last_msg().await;
+        let chat_id = msg.chat_id;
+        assert_eq!(chat_id.get_ephemeral_timer(&alice).await?, Timer::Disabled);
+
+        // Message with Message-ID <second@example.com> is received.
+        dc_receive_imf(
+            &alice,
+            b"From: Bob <bob@example.com>\n\
+                    To: Alice <alice@example.com>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: Subject\n\
+                    Message-ID: <second@example.com>\n\
+                    Date: Sun, 22 Mar 2020 00:11:00 +0000\n\
+                    Ephemeral-Timer: 60\n\
+                    \n\
+                    second message\n",
+            "INBOX",
+            2,
+            false,
+        )
+        .await?;
+        assert_eq!(
+            chat_id.get_ephemeral_timer(&alice).await?,
+            Timer::Enabled { duration: 60 }
+        );
+        let msg = alice.get_last_msg().await;
+
+        // Message is deleted from the database when its timer expires.
+        msg.id.delete_from_db(&alice).await?;
+
+        // Message with Message-ID <third@example.com>, referencing <first@example.com> and
+        // <second@example.com>, is received.  The message <second@example.come> is not in the
+        // database anymore, so the timer should be applied unconditionally without rollback
+        // protection.
+        //
+        // Previously Delta Chat fallen back to using <first@example.com> in this case and
+        // compared received timer value to the timer value of the <first@examle.com>. Because
+        // their timer values are the same ("disabled"), Delta Chat assumed that the timer was not
+        // changed explicitly and the change should be ignored.
+        //
+        // The message also contains a quote of the first message to test that only References:
+        // header and not In-Reply-To: is consulted by the rollback protection.
+        dc_receive_imf(
+            &alice,
+            b"From: Bob <bob@example.com>\n\
+                    To: Alice <alice@example.com>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: Subject\n\
+                    Message-ID: <third@example.com>\n\
+                    Date: Sun, 22 Mar 2020 00:12:00 +0000\n\
+                    References: <first@example.com> <second@example.com>\n\
+                    In-Reply-To: <first@example.com>\n\
+                    \n\
+                    > hello\n",
+            "INBOX",
+            3,
+            false,
+        )
+        .await?;
+
+        let msg = alice.get_last_msg().await;
+        assert_eq!(
+            msg.chat_id.get_ephemeral_timer(&alice).await?,
+            Timer::Disabled
+        );
+
+        Ok(())
+    }
 }

src/events.rs

@@ -199,7 +199,8 @@ pub enum EventType {
     /// - Chats created, deleted or archived
     /// - A draft has been set
     ///
-    /// The `chat_id` and `msg_id` values will be 0 if more than one message is changed.
+    /// `chat_id` is set if only a single chat is affected by the changes, otherwise 0.
+    /// `msg_id` is set if only a single message is affected by the changes, otherwise 0.
     #[strum(props(id = "2000"))]
     MsgsChanged { chat_id: ChatId, msg_id: MsgId },
 
@@ -322,4 +323,7 @@ pub enum EventType {
     /// dc_get_connectivity_html() for details.
     #[strum(props(id = "2100"))]
     ConnectivityChanged,
+
+    #[strum(props(id = "2110"))]
+    SelfavatarChanged,
 }

src/headerdef.rs

@@ -1,9 +1,8 @@
 //! # List of email headers.
 
-use crate::strum::AsStaticRef;
 use mailparse::{MailHeader, MailHeaderMap};
 
-#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
+#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, IntoStaticStr)]
 #[strum(serialize_all = "kebab_case")]
 pub enum HeaderDef {
     MessageId,
@@ -67,9 +66,9 @@ pub enum HeaderDef {
 }
 
 impl HeaderDef {
-    /// Returns the corresponding Event id.
+    /// Returns the corresponding header string.
     pub fn get_headername(&self) -> &'static str {
-        self.as_static()
+        self.into()
     }
 }
 

src/imap.rs

@@ -6,9 +6,8 @@
 use std::{cmp, cmp::max, collections::BTreeMap};
 
 use anyhow::{anyhow, bail, format_err, Context as _, Result};
-use async_imap::{
-    error::Result as ImapResult,
-    types::{Fetch, Flag, Mailbox, Name, NameAttribute, Quota, QuotaRoot, UnsolicitedResponse},
+use async_imap::types::{
+    Fetch, Flag, Mailbox, Name, NameAttribute, Quota, QuotaRoot, UnsolicitedResponse,
 };
 use async_std::channel::Receiver;
 use async_std::prelude::*;
@@ -66,7 +65,7 @@ pub enum ImapActionResult {
 /// - Chat-Version to check if a message is a chat message
 /// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message,
 ///   not necessarily sent by Delta Chat.
-const PREFETCH_FLAGS: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
+const PREFETCH_FLAGS: &str = "(UID RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\
                               MESSAGE-ID \
                               FROM \
                               IN-REPLY-TO REFERENCES \
@@ -82,15 +81,14 @@ const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
                              X-MICROSOFT-ORIGINAL-MESSAGE-ID\
                              )])";
 const JUST_UID: &str = "(UID)";
-const BODY_FLAGS: &str = "(FLAGS BODY.PEEK[])";
+const BODY_FULL: &str = "(FLAGS BODY.PEEK[])";
+const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])";
 
 #[derive(Debug)]
 pub struct Imap {
     idle_interrupt: Receiver<InterruptInfo>,
     config: ImapConfig,
     session: Option<Session>,
-    connected: bool,
-    interrupt: Option<stop_token::StopSource>,
     should_reconnect: bool,
     login_failed_once: bool,
 
@@ -201,8 +199,6 @@ impl Imap {
             idle_interrupt,
             config,
             session: None,
-            connected: false,
-            interrupt: None,
             should_reconnect: false,
             login_failed_once: false,
             connectivity: Default::default(),
@@ -229,7 +225,11 @@ impl Imap {
             param.socks5_config.clone(),
             &param.addr,
             param.server_flags & DC_LP_AUTH_OAUTH2 != 0,
-            param.provider.map_or(false, |provider| provider.strict_tls),
+            param
+                .provider
+                .map_or(param.socks5_config.is_some(), |provider| {
+                    provider.strict_tls
+                }),
             idle_interrupt,
         )
         .await?;
@@ -251,7 +251,7 @@ impl Imap {
         if self.should_reconnect() {
             self.disconnect(context).await;
             self.should_reconnect = false;
-        } else if self.is_connected() {
+        } else if self.session.is_some() {
             return Ok(());
         }
 
@@ -259,7 +259,7 @@ impl Imap {
 
         let oauth2 = self.config.oauth2;
 
-        let connection_res: ImapResult<Client> = if self.config.lp.security == Socket::Starttls
+        let connection_res: Result<Client> = if self.config.lp.security == Socket::Starttls
             || self.config.lp.security == Socket::Plain
         {
             let config = &mut self.config;
@@ -344,17 +344,16 @@ impl Imap {
         match login_res {
             Ok(session) => {
                 // needs to be set here to ensure it is set on reconnects.
-                self.connected = true;
                 self.session = Some(session);
                 self.login_failed_once = false;
-                emit_event!(
-                    context,
-                    EventType::ImapConnected(format!("IMAP-LOGIN as {}", self.config.lp.user))
-                );
+                context.emit_event(EventType::ImapConnected(format!(
+                    "IMAP-LOGIN as {}",
+                    self.config.lp.user
+                )));
                 Ok(())
             }
 
-            Err((err, _)) => {
+            Err(err) => {
                 let imap_user = self.config.lp.user.to_owned();
                 let message = stock_str::cannot_login(context, &imap_user).await;
 
@@ -439,16 +438,11 @@ impl Imap {
                 warn!(context, "failed to logout: {:?}", err);
             }
         }
-        self.connected = false;
         self.capabilities_determined = false;
         self.config.selected_folder = None;
         self.config.selected_mailbox = None;
     }
 
-    pub fn is_connected(&self) -> bool {
-        self.connected
-    }
-
     pub fn should_reconnect(&self) -> bool {
         self.should_reconnect
     }
@@ -466,7 +460,7 @@ impl Imap {
         self.prepare(context).await?;
 
         while self
-            .fetch_new_messages(context, &watch_folder, false)
+            .fetch_new_messages(context, watch_folder, false)
             .await?
         {
             // We fetch until no more new messages are there.
@@ -556,7 +550,7 @@ impl Imap {
         context: &Context,
         folder: &str,
     ) -> Result<bool> {
-        let newly_selected = self.select_folder(context, Some(folder)).await?;
+        let newly_selected = self.select_or_create_folder(context, folder).await?;
 
         let mailbox = &mut self.config.selected_mailbox.as_ref();
         let mailbox =
@@ -584,7 +578,7 @@ impl Imap {
                         folder, old_uid_next, uid_next, new_uid_validity,
                     );
                     set_uid_next(context, folder, uid_next).await?;
-                    job::schedule_resync(context).await;
+                    job::schedule_resync(context).await?;
                 }
                 uid_next != old_uid_next // If uid_next changed, there are new emails
             } else {
@@ -637,7 +631,7 @@ impl Imap {
         set_uid_next(context, folder, new_uid_next).await?;
         set_uidvalidity(context, folder, new_uid_validity).await?;
         if old_uid_validity != 0 || old_uid_next != 0 {
-            job::schedule_resync(context).await;
+            job::schedule_resync(context).await?;
         }
         info!(
             context,
@@ -651,25 +645,24 @@ impl Imap {
         Ok(false)
     }
 
-    pub(crate) async fn fetch_new_messages<S: AsRef<str>>(
+    pub(crate) async fn fetch_new_messages(
         &mut self,
         context: &Context,
-        folder: S,
+        folder: &str,
         fetch_existing_msgs: bool,
     ) -> Result<bool> {
         let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
             .unwrap_or_default();
+        let download_limit = context.download_limit().await?;
 
-        let new_emails = self
-            .select_with_uidvalidity(context, folder.as_ref())
-            .await?;
+        let new_emails = self.select_with_uidvalidity(context, folder).await?;
 
         if !new_emails && !fetch_existing_msgs {
-            info!(context, "No new emails in folder {}", folder.as_ref());
+            info!(context, "No new emails in folder {}", folder);
             return Ok(false);
         }
 
-        let old_uid_next = get_uid_next(context, folder.as_ref()).await?;
+        let old_uid_next = get_uid_next(context, folder).await?;
 
         let msgs = if fetch_existing_msgs {
             self.prefetch_existing_msgs().await?
@@ -677,10 +670,10 @@ impl Imap {
             self.prefetch(old_uid_next).await?
         };
         let read_cnt = msgs.len();
-        let folder: &str = folder.as_ref();
 
         let mut read_errors = 0;
-        let mut uids = Vec::with_capacity(msgs.len());
+        let mut uids_fetch_fully = Vec::with_capacity(msgs.len());
+        let mut uids_fetch_partially = Vec::with_capacity(msgs.len());
         let mut largest_uid_skipped = None;
 
         for (current_uid, msg) in msgs.into_iter() {
@@ -707,7 +700,16 @@ impl Imap {
             )
             .await
             {
-                uids.push(current_uid);
+                match download_limit {
+                    Some(download_limit) => {
+                        if msg.size.unwrap_or_default() > download_limit {
+                            uids_fetch_partially.push(current_uid);
+                        } else {
+                            uids_fetch_fully.push(current_uid)
+                        }
+                    }
+                    None => uids_fetch_fully.push(current_uid),
+                }
             } else if read_errors == 0 {
                 // If there were errors (`read_errors != 0`), stop updating largest_uid_skipped so that uid_next will
                 // not be updated and we will retry prefetching next time
@@ -715,12 +717,29 @@ impl Imap {
             }
         }
 
-        if !uids.is_empty() {
+        if !uids_fetch_fully.is_empty() || !uids_fetch_partially.is_empty() {
             self.connectivity.set_working(context).await;
         }
 
-        let (largest_uid_processed, error_cnt) = self
-            .fetch_many_msgs(context, folder, uids, fetch_existing_msgs)
+        let (largest_uid_fully_fetched, error_cnt) = self
+            .fetch_many_msgs(
+                context,
+                folder,
+                uids_fetch_fully,
+                false,
+                fetch_existing_msgs,
+            )
+            .await;
+        read_errors += error_cnt;
+
+        let (largest_uid_partially_fetched, error_cnt) = self
+            .fetch_many_msgs(
+                context,
+                folder,
+                uids_fetch_partially,
+                true,
+                fetch_existing_msgs,
+            )
             .await;
         read_errors += error_cnt;
 
@@ -731,7 +750,10 @@ impl Imap {
         // So: Update the uid_next to the largest uid that did NOT recoverably fail. Not perfect because if there was
         // another message afterwards that succeeded, we will not retry. The upside is that we will not retry an infinite amount of times.
         let largest_uid_without_errors = max(
-            largest_uid_processed.unwrap_or(0),
+            max(
+                largest_uid_fully_fetched.unwrap_or(0),
+                largest_uid_partially_fetched.unwrap_or(0),
+            ),
             largest_uid_skipped.unwrap_or(0),
         );
         let new_uid_next = largest_uid_without_errors + 1;
@@ -868,30 +890,25 @@ impl Imap {
     /// Fetches a list of messages by server UID.
     ///
     /// Returns the last uid fetch successfully and an error count.
-    async fn fetch_many_msgs(
+    pub(crate) async fn fetch_many_msgs(
         &mut self,
         context: &Context,
         folder: &str,
         server_uids: Vec<u32>,
+        fetch_partially: bool,
         fetching_existing_messages: bool,
     ) -> (Option<u32>, usize) {
         if server_uids.is_empty() {
             return (None, 0);
         }
 
-        if !self.is_connected() {
-            warn!(context, "Not connected");
-            return (None, server_uids.len());
-        }
-
-        if self.session.is_none() {
-            // we could not get a valid imap session, this should be retried
-            self.trigger_reconnect(context).await;
-            warn!(context, "Could not get IMAP session");
-            return (None, server_uids.len());
-        }
-
-        let session = self.session.as_mut().unwrap();
+        let session = match self.session.as_mut() {
+            Some(session) => session,
+            None => {
+                warn!(context, "Not connected");
+                return (None, server_uids.len());
+            }
+        };
 
         let sets = build_sequence_sets(server_uids.clone());
         let mut read_errors = 0;
@@ -899,7 +916,17 @@ impl Imap {
         let mut last_uid = None;
 
         for set in sets.iter() {
-            let mut msgs = match session.uid_fetch(&set, BODY_FLAGS).await {
+            let mut msgs = match session
+                .uid_fetch(
+                    &set,
+                    if fetch_partially {
+                        BODY_PARTIAL
+                    } else {
+                        BODY_FULL
+                    },
+                )
+                .await
+            {
                 Ok(msgs) => msgs,
                 Err(err) => {
                     // TODO: maybe differentiate between IO and input/parsing problems
@@ -934,7 +961,13 @@ impl Imap {
                 count += 1;
 
                 let is_deleted = msg.flags().any(|flag| flag == Flag::Deleted);
-                if is_deleted || msg.body().is_none() {
+                let (body, partial) = if fetch_partially {
+                    (msg.header(), msg.size) // `BODY.PEEK[HEADER]` goes to header() ...
+                } else {
+                    (msg.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header()
+                };
+
+                if is_deleted || body.is_none() {
                     info!(
                         context,
                         "Not processing deleted or empty msg {}", server_uid
@@ -948,7 +981,7 @@ impl Imap {
                 let folder = folder.clone();
 
                 // safe, as we checked above that there is a body.
-                let body = msg.body().unwrap();
+                let body = body.unwrap();
                 let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
                 match dc_receive_imf_inner(
@@ -957,6 +990,7 @@ impl Imap {
                     &folder,
                     server_uid,
                     is_seen,
+                    partial,
                     fetching_existing_messages,
                 )
                 .await
@@ -1012,13 +1046,10 @@ impl Imap {
             if let Some(ref mut session) = &mut self.session {
                 match session.uid_mv(&set, &dest_folder).await {
                     Ok(_) => {
-                        emit_event!(
-                            context,
-                            EventType::ImapMessageMoved(format!(
-                                "IMAP Message {} moved to {}",
-                                display_folder_id, dest_folder
-                            ))
-                        );
+                        context.emit_event(EventType::ImapMessageMoved(format!(
+                            "IMAP Message {} moved to {}",
+                            display_folder_id, dest_folder
+                        )));
                         return ImapActionResult::Success;
                     }
                     Err(err) => {
@@ -1056,23 +1087,17 @@ impl Imap {
 
         if !self.add_flag_finalized(context, uid, "\\Deleted").await {
             warn!(context, "Cannot mark {} as \"Deleted\" after copy.", uid);
-            emit_event!(
-                context,
-                EventType::ImapMessageMoved(format!(
-                    "IMAP Message {} copied to {} (delete FAILED)",
-                    display_folder_id, dest_folder
-                ))
-            );
+            context.emit_event(EventType::ImapMessageMoved(format!(
+                "IMAP Message {} copied to {} (delete FAILED)",
+                display_folder_id, dest_folder
+            )));
             ImapActionResult::Failed
         } else {
             self.config.selected_folder_needs_expunge = true;
-            emit_event!(
-                context,
-                EventType::ImapMessageMoved(format!(
-                    "IMAP Message {} copied to {} (delete successfull)",
-                    display_folder_id, dest_folder
-                ))
-            );
+            context.emit_event(EventType::ImapMessageMoved(format!(
+                "IMAP Message {} copied to {} (delete successfull)",
+                display_folder_id, dest_folder
+            )));
             ImapActionResult::Success
         }
     }
@@ -1129,7 +1154,7 @@ impl Imap {
         if uid == 0 {
             return Some(ImapActionResult::RetryLater);
         }
-        if !self.is_connected() {
+        if self.session.is_none() {
             // currently jobs are only performed on the INBOX thread
             // TODO: make INBOX/SENT/MVBOX perform the jobs on their
             // respective folders to avoid select_folder network traffic
@@ -1266,13 +1291,10 @@ impl Imap {
             );
             ImapActionResult::RetryLater
         } else {
-            emit_event!(
-                context,
-                EventType::ImapMessageDeleted(format!(
-                    "IMAP Message {} marked as deleted [{}]",
-                    display_imap_id, message_id
-                ))
-            );
+            context.emit_event(EventType::ImapMessageDeleted(format!(
+                "IMAP Message {} marked as deleted [{}]",
+                display_imap_id, message_id
+            )));
             self.config.selected_folder_needs_expunge = true;
             ImapActionResult::Success
         }
@@ -1292,115 +1314,114 @@ impl Imap {
     }
 
     pub async fn configure_folders(&mut self, context: &Context, create_mvbox: bool) -> Result<()> {
-        if !self.is_connected() {
-            bail!("IMAP No Connection established");
-        }
+        let session = match self.session {
+            Some(ref mut session) => session,
+            None => bail!("no IMAP connection established"),
+        };
 
-        if let Some(ref mut session) = &mut self.session {
-            let mut folders = match session.list(Some(""), Some("*")).await {
-                Ok(f) => f,
-                Err(err) => {
-                    bail!("list_folders failed: {}", err);
-                }
-            };
+        let mut folders = match session.list(Some(""), Some("*")).await {
+            Ok(f) => f,
+            Err(err) => {
+                bail!("list_folders failed: {}", err);
+            }
+        };
 
-            let mut delimiter = ".".to_string();
-            let mut delimiter_is_default = true;
-            let mut mvbox_folder = None;
-            let mut folder_configs = BTreeMap::new();
-            let mut fallback_folder = get_fallback_folder(&delimiter);
+        let mut delimiter = ".".to_string();
+        let mut delimiter_is_default = true;
+        let mut mvbox_folder = None;
+        let mut folder_configs = BTreeMap::new();
+        let mut fallback_folder = get_fallback_folder(&delimiter);
 
-            while let Some(folder) = folders.next().await {
-                let folder = folder?;
-                info!(context, "Scanning folder: {:?}", folder);
+        while let Some(folder) = folders.next().await {
+            let folder = folder?;
+            info!(context, "Scanning folder: {:?}", folder);
 
-                // Update the delimiter iff there is a different one, but only once.
-                if let Some(d) = folder.delimiter() {
-                    if delimiter_is_default && !d.is_empty() && delimiter != d {
-                        delimiter = d.to_string();
-                        fallback_folder = get_fallback_folder(&delimiter);
-                        delimiter_is_default = false;
-                    }
-                }
-
-                let folder_meaning = get_folder_meaning(&folder);
-                let folder_name_meaning = get_folder_meaning_by_name(folder.name());
-                if folder.name() == "DeltaChat" {
-                    // Always takes precedence
-                    mvbox_folder = Some(folder.name().to_string());
-                } else if folder.name() == fallback_folder {
-                    // only set if none has been already set
-                    if mvbox_folder.is_none() {
-                        mvbox_folder = Some(folder.name().to_string());
-                    }
-                } else if let Some(config) = folder_meaning.to_config() {
-                    // Always takes precedence
-                    folder_configs.insert(config, folder.name().to_string());
-                } else if let Some(config) = folder_name_meaning.to_config() {
-                    // only set if none has been already set
-                    folder_configs
-                        .entry(config)
-                        .or_insert_with(|| folder.name().to_string());
+            // Update the delimiter iff there is a different one, but only once.
+            if let Some(d) = folder.delimiter() {
+                if delimiter_is_default && !d.is_empty() && delimiter != d {
+                    delimiter = d.to_string();
+                    fallback_folder = get_fallback_folder(&delimiter);
+                    delimiter_is_default = false;
                 }
             }
-            drop(folders);
 
-            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
+            let folder_meaning = get_folder_meaning(&folder);
+            let folder_name_meaning = get_folder_meaning_by_name(folder.name());
+            if folder.name() == "DeltaChat" {
+                // Always takes precedence
+                mvbox_folder = Some(folder.name().to_string());
+            } else if folder.name() == fallback_folder {
+                // only set if none has been already set
+                if mvbox_folder.is_none() {
+                    mvbox_folder = Some(folder.name().to_string());
+                }
+            } else if let Some(config) = folder_meaning.to_config() {
+                // Always takes precedence
+                folder_configs.insert(config, folder.name().to_string());
+            } else if let Some(config) = folder_name_meaning.to_config() {
+                // only set if none has been already set
+                folder_configs
+                    .entry(config)
+                    .or_insert_with(|| folder.name().to_string());
+            }
+        }
+        drop(folders);
 
-            if mvbox_folder.is_none() && create_mvbox {
-                info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
+        info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
 
-                match session.create("DeltaChat").await {
-                    Ok(_) => {
-                        mvbox_folder = Some("DeltaChat".into());
-                        info!(context, "MVBOX-folder created.",);
-                    }
-                    Err(err) => {
-                        warn!(
-                            context,
-                            "Cannot create MVBOX-folder, trying to create INBOX subfolder. ({})",
-                            err
-                        );
+        if mvbox_folder.is_none() && create_mvbox {
+            info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
 
-                        match session.create(&fallback_folder).await {
-                            Ok(_) => {
-                                mvbox_folder = Some(fallback_folder);
-                                info!(
-                                    context,
-                                    "MVBOX-folder created as INBOX subfolder. ({})", err
-                                );
-                            }
-                            Err(err) => {
-                                warn!(context, "Cannot create MVBOX-folder. ({})", err);
-                            }
+            match session.create("DeltaChat").await {
+                Ok(_) => {
+                    mvbox_folder = Some("DeltaChat".into());
+                    info!(context, "MVBOX-folder created.",);
+                }
+                Err(err) => {
+                    warn!(
+                        context,
+                        "Cannot create MVBOX-folder, trying to create INBOX subfolder. ({})", err
+                    );
+
+                    match session.create(&fallback_folder).await {
+                        Ok(_) => {
+                            mvbox_folder = Some(fallback_folder);
+                            info!(
+                                context,
+                                "MVBOX-folder created as INBOX subfolder. ({})", err
+                            );
+                        }
+                        Err(err) => {
+                            warn!(context, "Cannot create MVBOX-folder. ({})", err);
                         }
                     }
                 }
-                // SUBSCRIBE is needed to make the folder visible to the LSUB command
-                // that may be used by other MUAs to list folders.
-                // for the LIST command, the folder is always visible.
-                if let Some(ref mvbox) = mvbox_folder {
-                    if let Err(err) = session.subscribe(mvbox).await {
-                        warn!(context, "could not subscribe to {:?}: {:?}", mvbox, err);
-                    }
+            }
+            // SUBSCRIBE is needed to make the folder visible to the LSUB command
+            // that may be used by other MUAs to list folders.
+            // for the LIST command, the folder is always visible.
+            if let Some(ref mvbox) = mvbox_folder {
+                if let Err(err) = session.subscribe(mvbox).await {
+                    warn!(context, "could not subscribe to {:?}: {:?}", mvbox, err);
                 }
             }
+        }
+        context
+            .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
+            .await?;
+        if let Some(ref mvbox_folder) = mvbox_folder {
             context
-                .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
-                .await?;
-            if let Some(ref mvbox_folder) = mvbox_folder {
-                context
-                    .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
-                    .await?;
-            }
-            for (config, name) in folder_configs {
-                context.set_config(config, Some(&name)).await?;
-            }
-            context
-                .sql
-                .set_raw_config_int("folders_configured", DC_FOLDERS_CONFIGURED_VERSION)
+                .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
                 .await?;
         }
+        for (config, name) in folder_configs {
+            context.set_config(config, Some(&name)).await?;
+        }
+        context
+            .sql
+            .set_raw_config_int("folders_configured", DC_FOLDERS_CONFIGURED_VERSION)
+            .await?;
+
         info!(context, "FINISHED configuring IMAP-folders.");
         Ok(())
     }
@@ -1588,13 +1609,13 @@ async fn precheck_imf(
                         context,
                         job::Job::new(Action::MoveMsg, msg_id.to_u32(), Params::new(), 0),
                     )
-                    .await;
+                    .await?;
                 } else {
                     job::add(
                         context,
                         job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
                     )
-                    .await;
+                    .await?;
                 }
             }
         } else if old_server_folder != server_folder {
@@ -1637,7 +1658,7 @@ async fn precheck_imf(
                         context,
                         job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
                     )
-                    .await;
+                    .await?;
                 }
             }
         }

src/imap/client.rs

@@ -3,10 +3,9 @@ use std::{
     time::Duration,
 };
 
-use async_imap::{
-    error::{Error as ImapError, Result as ImapResult},
-    Client as ImapClient,
-};
+use anyhow::{Context as _, Result};
+
+use async_imap::Client as ImapClient;
 
 use async_smtp::ServerAddress;
 use async_std::net::{self, TcpStream};
@@ -40,24 +39,12 @@ impl DerefMut for Client {
 }
 
 impl Client {
-    pub async fn login(
-        self,
-        username: &str,
-        password: &str,
-    ) -> std::result::Result<Session, (ImapError, Self)> {
-        let Client { inner, is_secure } = self;
+    pub async fn login(self, username: &str, password: &str) -> Result<Session> {
+        let Client { inner, .. } = self;
         let session = inner
             .login(username, password)
             .await
-            .map_err(|(err, client)| {
-                (
-                    err,
-                    Client {
-                        is_secure,
-                        inner: client,
-                    },
-                )
-            })?;
+            .map_err(|(err, _client)| err)?;
         Ok(Session { inner: session })
     }
 
@@ -65,21 +52,12 @@ impl Client {
         self,
         auth_type: &str,
         authenticator: impl async_imap::Authenticator,
-    ) -> std::result::Result<Session, (ImapError, Self)> {
-        let Client { inner, is_secure } = self;
-        let session =
-            inner
-                .authenticate(auth_type, authenticator)
-                .await
-                .map_err(|(err, client)| {
-                    (
-                        err,
-                        Client {
-                            is_secure,
-                            inner: client,
-                        },
-                    )
-                })?;
+    ) -> Result<Session> {
+        let Client { inner, .. } = self;
+        let session = inner
+            .authenticate(auth_type, authenticator)
+            .await
+            .map_err(|(err, _client)| err)?;
         Ok(Session { inner: session })
     }
 
@@ -87,7 +65,7 @@ impl Client {
         addr: impl net::ToSocketAddrs,
         domain: &str,
         strict_tls: bool,
-    ) -> ImapResult<Self> {
+    ) -> Result<Self> {
         let stream = TcpStream::connect(addr).await?;
         let tls = dc_build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> = Box::new(tls.connect(domain, stream).await?);
@@ -96,7 +74,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .context("failed to read greeting")?;
 
         Ok(Client {
             is_secure: true,
@@ -104,14 +82,14 @@ impl Client {
         })
     }
 
-    pub async fn connect_insecure(addr: impl net::ToSocketAddrs) -> ImapResult<Self> {
+    pub async fn connect_insecure(addr: impl net::ToSocketAddrs) -> Result<Self> {
         let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .context("failed to read greeting")?;
 
         Ok(Client {
             is_secure: false,
@@ -123,15 +101,11 @@ impl Client {
         target_addr: &ServerAddress,
         strict_tls: bool,
         socks5_config: Socks5Config,
-    ) -> ImapResult<Self> {
+    ) -> Result<Self> {
         let socks5_stream: Box<dyn SessionStream> = Box::new(
-            match socks5_config
+            socks5_config
                 .connect(target_addr, Some(Duration::from_secs(IMAP_TIMEOUT)))
-                .await
-            {
-                Ok(s) => s,
-                Err(e) => return ImapResult::Err(async_imap::error::Error::Bad(e.to_string())),
-            },
+                .await?,
         );
 
         let tls = dc_build_tls(strict_tls);
@@ -142,7 +116,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .context("failed to read greeting")?;
 
         Ok(Client {
             is_secure: true,
@@ -153,22 +127,18 @@ impl Client {
     pub async fn connect_insecure_socks5(
         target_addr: &ServerAddress,
         socks5_config: Socks5Config,
-    ) -> ImapResult<Self> {
+    ) -> Result<Self> {
         let socks5_stream: Box<dyn SessionStream> = Box::new(
-            match socks5_config
+            socks5_config
                 .connect(target_addr, Some(Duration::from_secs(IMAP_TIMEOUT)))
-                .await
-            {
-                Ok(s) => s,
-                Err(e) => return ImapResult::Err(async_imap::error::Error::Bad(e.to_string())),
-            },
+                .await?,
         );
 
         let mut client = ImapClient::new(socks5_stream);
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .context("failed to read greeting")?;
 
         Ok(Client {
             is_secure: false,
@@ -176,7 +146,7 @@ impl Client {
         })
     }
 
-    pub async fn secure(self, domain: &str, strict_tls: bool) -> ImapResult<Client> {
+    pub async fn secure(self, domain: &str, strict_tls: bool) -> Result<Client> {
         if self.is_secure {
             Ok(self)
         } else {

src/imap/select_folder.rs

@@ -15,6 +15,9 @@ pub enum Error {
     #[error("IMAP Folder name invalid: {0}")]
     BadFolderName(String),
 
+    #[error("IMAP folder does not exist: {0}")]
+    NoFolder(String),
+
     #[error("IMAP close/expunge failed")]
     CloseExpungeFailed(#[from] async_imap::error::Error),
 
@@ -110,6 +113,9 @@ impl Imap {
                     Err(async_imap::error::Error::Validate(_)) => {
                         Err(Error::BadFolderName(folder.to_string()))
                     }
+                    Err(async_imap::error::Error::No(_)) => {
+                        Err(Error::NoFolder(folder.to_string()))
+                    }
                     Err(err) => {
                         self.config.selected_folder = None;
                         self.trigger_reconnect(context).await;
@@ -123,6 +129,28 @@ impl Imap {
             Ok(NewlySelected::No)
         }
     }
+
+    /// Selects a folder. Tries to create it once and select again if the folder does not exist.
+    pub(super) async fn select_or_create_folder(
+        &mut self,
+        context: &Context,
+        folder: &str,
+    ) -> Result<NewlySelected> {
+        match self.select_folder(context, Some(folder)).await {
+            Ok(newly_selected) => Ok(newly_selected),
+            Err(err) => match err {
+                Error::NoFolder(_) => {
+                    if let Some(ref mut session) = self.session {
+                        session.create(folder).await?;
+                    } else {
+                        return Err(Error::NoSession);
+                    }
+                    self.select_folder(context, Some(folder)).await
+                }
+                _ => Err(err),
+            },
+        }
+    }
 }
 #[derive(PartialEq, Debug, Copy, Clone, Eq)]
 pub(super) enum NewlySelected {

src/imex.rs

@@ -234,7 +234,7 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     msg = Message::default();
     msg.viewtype = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
-
+    msg.subject = stock_str::ac_setup_msg_subject(context).await;
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
@@ -507,14 +507,16 @@ async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()>
     let archive = Archive::new(backup_file);
 
     let mut entries = archive.entries()?;
+    let mut last_progress = 0;
     while let Some(file) = entries.next().await {
         let f = &mut file?;
 
         let current_pos = f.raw_file_position();
         let progress = 1000 * current_pos / file_size;
-        if progress > 10 && progress < 1000 {
+        if progress != last_progress && progress > 10 && progress < 1000 {
             // We already emitted ImexProgress(10) above
             context.emit_event(EventType::ImexProgress(progress as usize));
+            last_progress = progress;
         }
 
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
@@ -737,6 +739,7 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
     let count = read_dir.len();
     let mut written_files = 0;
 
+    let mut last_progress = 0;
     for entry in read_dir.into_iter() {
         let entry = entry?;
         let name = entry.file_name();
@@ -754,9 +757,10 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
 
         written_files += 1;
         let progress = 1000 * written_files / count;
-        if progress > 10 && progress < 1000 {
+        if progress != last_progress && progress > 10 && progress < 1000 {
             // We already emitted ImexProgress(10) above
-            emit_event!(context, EventType::ImexProgress(progress));
+            context.emit_event(EventType::ImexProgress(progress));
+            last_progress = progress;
         }
     }
 

src/job.rs

@@ -2,12 +2,11 @@
 //!
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
+use std::fmt;
 use std::future::Future;
-use std::{fmt, time::Duration};
 
 use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
 use async_smtp::smtp::response::{Category, Code, Detail};
-use async_std::task::sleep;
 use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
@@ -103,6 +102,12 @@ pub enum Action {
     MoveMsg = 200,
     DeleteMsgOnImap = 210,
 
+    // This job will download partially downloaded messages completely
+    // and is added when download_full() is called.
+    // Most messages are downloaded automatically on fetch
+    // and do not go through this job.
+    DownloadMsg = 250,
+
     // UID synchronization is high-priority to make sure correct UIDs
     // are used by message moving/deletion.
     ResyncFolders = 300,
@@ -134,6 +139,7 @@ impl From<Action> for Thread {
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
             UpdateRecentQuota => Thread::Imap,
+            DownloadMsg => Thread::Imap,
 
             MaybeSendLocations => Thread::Smtp,
             MaybeSendLocationsEnded => Thread::Smtp,
@@ -427,6 +433,13 @@ impl Job {
                 }
                 // now also delete the generated file
                 dc_delete_file(context, filename).await;
+
+                // finally, create another send-job if there are items to be synced.
+                // triggering sync-job after msg-send-job guarantees, the recipient has grpid etc.
+                // once the sync message arrives.
+                // if there are no items to sync, this function returns fast.
+                context.send_sync_msg().await?;
+
                 Ok(())
             }
         })
@@ -709,7 +722,7 @@ impl Job {
                 Config::ConfiguredSentboxFolder,
             ] {
                 if let Some(folder) = job_try!(context.get_config(*config).await) {
-                    if let Err(e) = imap.fetch_new_messages(context, folder, true).await {
+                    if let Err(e) = imap.fetch_new_messages(context, &folder, true).await {
                         // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
                         warn!(context, "Could not fetch messages, retrying: {:#}", e);
                         return Status::RetryLater;
@@ -815,12 +828,12 @@ impl Job {
 }
 
 /// Delete all pending jobs with the given action.
-pub async fn kill_action(context: &Context, action: Action) -> bool {
+pub async fn kill_action(context: &Context, action: Action) -> Result<()> {
     context
         .sql
         .execute("DELETE FROM jobs WHERE action=?;", paramsv![action])
-        .await
-        .is_ok()
+        .await?;
+    Ok(())
 }
 
 /// Remove jobs with specified IDs.
@@ -836,15 +849,15 @@ async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
     Ok(())
 }
 
-pub async fn action_exists(context: &Context, action: Action) -> bool {
-    context
+pub async fn action_exists(context: &Context, action: Action) -> Result<bool> {
+    let exists = context
         .sql
         .exists(
             "SELECT COUNT(*) FROM jobs WHERE action=?;",
             paramsv![action],
         )
-        .await
-        .unwrap_or_default()
+        .await?;
+    Ok(exists)
 }
 
 async fn set_delivered(context: &Context, msg_id: MsgId) -> Result<()> {
@@ -978,7 +991,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     }
 
     if rendered_msg.is_gossiped {
-        chat::set_gossiped_timestamp(context, msg.chat_id, time()).await?;
+        msg.chat_id.set_gossiped_timestamp(context, time()).await?;
     }
 
     if 0 != rendered_msg.last_added_location_id {
@@ -995,6 +1008,12 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
         }
     }
 
+    if let Some(sync_ids) = rendered_msg.sync_ids_to_delete {
+        if let Err(err) = context.delete_sync_ids(sync_ids).await {
+            error!(context, "Failed to delete sync ids: {:?}", err);
+        }
+    }
+
     if attach_selfavatar {
         if let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, time()).await {
             error!(context, "Failed to set selfavatar timestamp: {:?}", err);
@@ -1078,7 +1097,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
                     "{} thread increases job {} tries to {}", &connection, job, tries
                 );
                 job.tries = tries;
-                let time_offset = get_backoff_time_offset(tries);
+                let time_offset = get_backoff_time_offset(tries, job.action);
                 job.desired_timestamp = time() + time_offset;
                 info!(
                     context,
@@ -1152,7 +1171,11 @@ async fn perform_job_action(
             sql::housekeeping(context).await.ok_or_log(context);
             Status::Finished(Ok(()))
         }
-        Action::UpdateRecentQuota => context.update_recent_quota(connection.inbox()).await,
+        Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
+            Ok(status) => status,
+            Err(err) => Status::Finished(Err(err)),
+        },
+        Action::DownloadMsg => job.download_msg(context, connection.inbox()).await,
     };
 
     info!(context, "Finished immediate try {} of job {}", tries, job);
@@ -1160,33 +1183,43 @@ async fn perform_job_action(
     try_res
 }
 
-fn get_backoff_time_offset(tries: u32) -> i64 {
-    let n = 2_i32.pow(tries - 1) * 60;
-    let mut rng = thread_rng();
-    let r: i32 = rng.gen();
-    let mut seconds = r % (n + 1);
-    if seconds < 1 {
-        seconds = 1;
+fn get_backoff_time_offset(tries: u32, action: Action) -> i64 {
+    match action {
+        // Just try every 10s to update the quota
+        // If all retries are exhausted, a new job will be created when the quota information is needed
+        Action::UpdateRecentQuota => 10,
+
+        _ => {
+            // Exponential backoff
+            let n = 2_i32.pow(tries - 1) * 60;
+            let mut rng = thread_rng();
+            let r: i32 = rng.gen();
+            let mut seconds = r % (n + 1);
+            if seconds < 1 {
+                seconds = 1;
+            }
+            seconds as i64
+        }
     }
-    seconds as i64
 }
 
 async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     let mut param = Params::new();
     param.set(Param::MsgId, msg.id.to_u32().to_string());
 
-    add(context, Job::new(Action::SendMdn, msg.from_id, param, 0)).await;
+    add(context, Job::new(Action::SendMdn, msg.from_id, param, 0)).await?;
 
     Ok(())
 }
 
-pub(crate) async fn schedule_resync(context: &Context) {
-    kill_action(context, Action::ResyncFolders).await;
+pub(crate) async fn schedule_resync(context: &Context) -> Result<()> {
+    kill_action(context, Action::ResyncFolders).await?;
     add(
         context,
         Job::new(Action::ResyncFolders, 0, Params::new(), 0),
     )
-    .await;
+    .await?;
+    Ok(())
 }
 
 /// Creates a job.
@@ -1200,12 +1233,10 @@ pub fn create(action: Action, foreign_id: u32, param: Params, delay_seconds: i64
 }
 
 /// Adds a job to the database, scheduling it.
-pub async fn add(context: &Context, job: Job) {
+pub async fn add(context: &Context, job: Job) -> Result<()> {
     let action = job.action;
     let delay_seconds = job.delay_seconds();
-    job.save(context).await.unwrap_or_else(|err| {
-        error!(context, "failed to save job: {}", err);
-    });
+    job.save(context).await.context("failed to save job")?;
 
     if delay_seconds == 0 {
         match action {
@@ -1216,7 +1247,8 @@ pub async fn add(context: &Context, job: Job) {
             | Action::MarkseenMsgOnImap
             | Action::FetchExistingMsgs
             | Action::MoveMsg
-            | Action::UpdateRecentQuota => {
+            | Action::UpdateRecentQuota
+            | Action::DownloadMsg => {
                 info!(context, "interrupt: imap");
                 context
                     .interrupt_inbox(InterruptInfo::new(false, None))
@@ -1233,23 +1265,18 @@ pub async fn add(context: &Context, job: Job) {
             }
         }
     }
+    Ok(())
 }
 
-async fn load_housekeeping_job(context: &Context) -> Option<Job> {
-    let last_time = match context.get_config_i64(Config::LastHousekeeping).await {
-        Ok(last_time) => last_time,
-        Err(err) => {
-            warn!(context, "failed to load housekeeping config: {:?}", err);
-            return None;
-        }
-    };
+async fn load_housekeeping_job(context: &Context) -> Result<Option<Job>> {
+    let last_time = context.get_config_i64(Config::LastHousekeeping).await?;
 
     let next_time = last_time + (60 * 60 * 24);
     if next_time <= time() {
-        kill_action(context, Action::Housekeeping).await;
-        Some(Job::new(Action::Housekeeping, 0, Params::new(), 0))
+        kill_action(context, Action::Housekeeping).await?;
+        Ok(Some(Job::new(Action::Housekeeping, 0, Params::new(), 0)))
     } else {
-        None
+        Ok(None)
     }
 }
 
@@ -1263,20 +1290,9 @@ pub(crate) async fn load_next(
     context: &Context,
     thread: Thread,
     info: &InterruptInfo,
-) -> Option<Job> {
+) -> Result<Option<Job>> {
     info!(context, "loading job for {}-thread", thread);
 
-    while !context.sql.is_open().await {
-        // The db is closed, which means that this thread should not be running.
-        // Wait until the db is re-opened (if we returned None, this thread might do further damage)
-        warn!(
-            context,
-            "{}: load_next() was called but the db was not opened, THIS SHOULD NOT HAPPEN. Waiting...",
-            thread
-        );
-        sleep(Duration::from_millis(500)).await;
-    }
-
     let query;
     let params;
     let t = time();
@@ -1343,51 +1359,38 @@ LIMIT 1;
                 info!(context, "cleaning up job, because of {}", err);
 
                 // TODO: improve by only doing a single query
-                match context
+                let id = context
                     .sql
                     .query_row(query, params.clone(), |row| row.get::<_, i32>(0))
                     .await
-                {
-                    Ok(id) => {
-                        if let Err(err) = context
-                            .sql
-                            .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
-                            .await
-                        {
-                            warn!(context, "failed to delete job {}: {:?}", id, err);
-                        }
-                    }
-                    Err(err) => {
-                        error!(context, "failed to retrieve invalid job from DB: {}", err);
-                        break None;
-                    }
-                }
+                    .context("Failed to retrieve invalid job ID from the database")?;
+                context
+                    .sql
+                    .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
+                    .await
+                    .with_context(|| format!("Failed to delete invalid job {}", id))?;
             }
         }
     };
 
     match thread {
         Thread::Unknown => {
-            error!(context, "unknown thread for job");
-            None
+            bail!("unknown thread for job")
         }
         Thread::Imap => {
             if let Some(job) = job {
                 if job.action < Action::DeleteMsgOnImap {
-                    load_imap_deletion_job(context)
-                        .await
-                        .unwrap_or_default()
-                        .or(Some(job))
+                    Ok(load_imap_deletion_job(context).await?.or(Some(job)))
                 } else {
-                    Some(job)
+                    Ok(Some(job))
                 }
-            } else if let Some(job) = load_imap_deletion_job(context).await.unwrap_or_default() {
-                Some(job)
+            } else if let Some(job) = load_imap_deletion_job(context).await? {
+                Ok(Some(job))
             } else {
-                load_housekeeping_job(context).await
+                Ok(load_housekeeping_job(context).await?)
             }
         }
-        Thread::Smtp => job,
+        Thread::Smtp => Ok(job),
     }
 }
 
@@ -1419,7 +1422,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_load_next_job_two() {
+    async fn test_load_next_job_two() -> Result<()> {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
         // all jobs.
@@ -1430,7 +1433,7 @@ mod tests {
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
-        .await;
+        .await?;
         // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
         assert_eq!(jobs.unwrap().action, Action::Housekeeping);
 
@@ -1440,12 +1443,13 @@ mod tests {
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
-        .await;
+        .await?;
         assert!(jobs.is_some());
+        Ok(())
     }
 
     #[async_std::test]
-    async fn test_load_next_job_one() {
+    async fn test_load_next_job_one() -> Result<()> {
         let t = TestContext::new().await;
 
         insert_job(&t, 1, true).await;
@@ -1455,7 +1459,8 @@ mod tests {
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
-        .await;
+        .await?;
         assert!(jobs.is_some());
+        Ok(())
     }
 }

src/lib.rs

@@ -55,6 +55,7 @@ mod configure;
 pub mod constants;
 pub mod contact;
 pub mod context;
+pub mod download;
 mod e2ee;
 pub mod ephemeral;
 mod imap;
@@ -67,7 +68,6 @@ pub mod key;
 mod keyring;
 pub mod location;
 mod login_param;
-pub mod lot;
 pub mod message;
 mod mimefactory;
 pub mod mimeparser;
@@ -82,12 +82,15 @@ pub mod securejoin;
 mod simplify;
 mod smtp;
 pub mod stock_str;
+mod sync;
 mod token;
+mod update_helper;
 #[macro_use]
 mod dehtml;
 mod color;
 pub mod html;
 pub mod plaintext;
+pub mod summary;
 
 pub mod dc_receive_imf;
 pub mod dc_tools;

src/location.rs

@@ -1,7 +1,7 @@
 //! Location handling.
 use std::convert::TryFrom;
 
-use anyhow::{ensure, Error};
+use anyhow::{ensure, Result};
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
@@ -63,7 +63,7 @@ impl Kml {
         Default::default()
     }
 
-    pub fn parse(context: &Context, to_parse: &[u8]) -> Result<Self, Error> {
+    pub fn parse(context: &Context, to_parse: &[u8]) -> Result<Self> {
         ensure!(to_parse.len() <= 1024 * 1024, "kml-file is too large");
 
         let mut reader = quick_xml::Reader::from_reader(to_parse);
@@ -191,89 +191,97 @@ impl Kml {
 }
 
 /// Enables location streaming in chat identified by `chat_id` for `seconds` seconds.
-pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
+pub async fn send_locations_to_chat(
+    context: &Context,
+    chat_id: ChatId,
+    seconds: i64,
+) -> Result<()> {
+    ensure!(seconds >= 0);
+    ensure!(!chat_id.is_special());
     let now = time();
-    if !(seconds < 0 || chat_id.is_special()) {
-        let is_sending_locations_before =
-            is_sending_locations_to_chat(context, Some(chat_id)).await;
-        if context
-            .sql
-            .execute(
-                "UPDATE chats    \
-             SET locations_send_begin=?,        \
-             locations_send_until=?  \
-             WHERE id=?",
-                paramsv![
-                    if 0 != seconds { now } else { 0 },
-                    if 0 != seconds { now + seconds } else { 0 },
-                    chat_id,
-                ],
-            )
+    let is_sending_locations_before = is_sending_locations_to_chat(context, Some(chat_id)).await?;
+    context
+        .sql
+        .execute(
+            "UPDATE chats    \
+         SET locations_send_begin=?,        \
+         locations_send_until=?  \
+         WHERE id=?",
+            paramsv![
+                if 0 != seconds { now } else { 0 },
+                if 0 != seconds { now + seconds } else { 0 },
+                chat_id,
+            ],
+        )
+        .await?;
+    if 0 != seconds && !is_sending_locations_before {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(stock_str::msg_location_enabled(context).await);
+        msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
+        chat::send_msg(context, chat_id, &mut msg)
             .await
-            .is_ok()
-        {
-            if 0 != seconds && !is_sending_locations_before {
-                let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some(stock_str::msg_location_enabled(context).await);
-                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
-                chat::send_msg(context, chat_id, &mut msg)
-                    .await
-                    .unwrap_or_default();
-            } else if 0 == seconds && is_sending_locations_before {
-                let stock_str = stock_str::msg_location_disabled(context).await;
-                chat::add_info_msg(context, chat_id, stock_str, now).await;
-            }
-            context.emit_event(EventType::ChatModified(chat_id));
-            if 0 != seconds {
-                schedule_maybe_send_locations(context, false).await;
-                job::add(
-                    context,
-                    job::Job::new(
-                        job::Action::MaybeSendLocationsEnded,
-                        chat_id.to_u32(),
-                        Params::new(),
-                        seconds + 1,
-                    ),
-                )
-                .await;
-            }
-        }
+            .unwrap_or_default();
+    } else if 0 == seconds && is_sending_locations_before {
+        let stock_str = stock_str::msg_location_disabled(context).await;
+        chat::add_info_msg(context, chat_id, stock_str, now).await?;
     }
+    context.emit_event(EventType::ChatModified(chat_id));
+    if 0 != seconds {
+        schedule_maybe_send_locations(context, false).await?;
+        job::add(
+            context,
+            job::Job::new(
+                job::Action::MaybeSendLocationsEnded,
+                chat_id.to_u32(),
+                Params::new(),
+                seconds + 1,
+            ),
+        )
+        .await?;
+    }
+    Ok(())
 }
 
-async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) {
-    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await {
+async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) -> Result<()> {
+    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await? {
         job::add(
             context,
             job::Job::new(job::Action::MaybeSendLocations, 0, Params::new(), 60),
         )
-        .await;
+        .await?;
     };
+    Ok(())
 }
 
 /// Returns whether `chat_id` or any chat is sending locations.
 ///
 /// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
 /// is sending locations.
-pub async fn is_sending_locations_to_chat(context: &Context, chat_id: Option<ChatId>) -> bool {
-    match chat_id {
-        Some(chat_id) => context
-            .sql
-            .exists(
-                "SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;",
-                paramsv![chat_id, time()],
-            )
-            .await
-            .unwrap_or_default(),
-        None => context
-            .sql
-            .exists(
-                "SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;",
-                paramsv![time()],
-            )
-            .await
-            .unwrap_or_default(),
-    }
+pub async fn is_sending_locations_to_chat(
+    context: &Context,
+    chat_id: Option<ChatId>,
+) -> Result<bool> {
+    let exists = match chat_id {
+        Some(chat_id) => {
+            context
+                .sql
+                .exists(
+                    "SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;",
+                    paramsv![chat_id, time()],
+                )
+                .await?
+        }
+        None => {
+            context
+                .sql
+                .exists(
+                    "SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;",
+                    paramsv![time()],
+                )
+                .await?
+        }
+    };
+    Ok(exists)
 }
 
 pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
@@ -288,7 +296,11 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
             "SELECT id FROM chats WHERE locations_send_until>?;",
             paramsv![time()],
             |row| row.get::<_, i32>(0),
-            |chats| chats.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+            |chats| {
+                chats
+                    .collect::<std::result::Result<Vec<_>, _>>()
+                    .map_err(Into::into)
+            },
         )
         .await
     {
@@ -313,7 +325,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
         if continue_streaming {
             context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
-        schedule_maybe_send_locations(context, false).await;
+        schedule_maybe_send_locations(context, false).await.ok();
     }
 
     continue_streaming
@@ -325,7 +337,7 @@ pub async fn get_range(
     contact_id: Option<u32>,
     timestamp_from: i64,
     mut timestamp_to: i64,
-) -> Result<Vec<Location>, Error> {
+) -> Result<Vec<Location>> {
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
@@ -400,7 +412,7 @@ fn is_marker(txt: &str) -> bool {
 }
 
 /// Deletes all locations from the database.
-pub async fn delete_all(context: &Context) -> Result<(), Error> {
+pub async fn delete_all(context: &Context) -> Result<()> {
     context
         .sql
         .execute("DELETE FROM locations;", paramsv![])
@@ -409,7 +421,7 @@ pub async fn delete_all(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Error> {
+pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)> {
     let mut last_added_location_id = 0;
 
     let self_addr = context
@@ -517,7 +529,7 @@ pub async fn set_kml_sent_timestamp(
     context: &Context,
     chat_id: ChatId,
     timestamp: i64,
-) -> Result<(), Error> {
+) -> Result<()> {
     context
         .sql
         .execute(
@@ -528,11 +540,7 @@ pub async fn set_kml_sent_timestamp(
     Ok(())
 }
 
-pub async fn set_msg_location_id(
-    context: &Context,
-    msg_id: MsgId,
-    location_id: u32,
-) -> Result<(), Error> {
+pub async fn set_msg_location_id(context: &Context, msg_id: MsgId, location_id: u32) -> Result<()> {
     context
         .sql
         .execute(
@@ -544,17 +552,20 @@ pub async fn set_msg_location_id(
     Ok(())
 }
 
-pub async fn save(
+/// Saves given locations to the database.
+///
+/// Returns the database row ID of the location with the highest timestamp.
+pub(crate) async fn save(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
     locations: &[Location],
     independent: bool,
-) -> Result<u32, Error> {
+) -> Result<Option<u32>> {
     ensure!(!chat_id.is_special(), "Invalid chat id");
 
     let mut newest_timestamp = 0;
-    let mut newest_location_id = 0;
+    let mut newest_location_id = None;
 
     let stmt_insert = "INSERT INTO locations\
              (timestamp, from_id, chat_id, latitude, longitude, accuracy, independent) \
@@ -592,12 +603,12 @@ pub async fn save(
                 drop(stmt_test);
                 drop(stmt_insert);
                 newest_timestamp = timestamp;
-                newest_location_id = conn.last_insert_rowid();
+                newest_location_id = Some(u32::try_from(conn.last_insert_rowid())?);
             }
         }
     }
 
-    Ok(u32::try_from(newest_location_id)?)
+    Ok(newest_location_id)
 }
 
 pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> job::Status {
@@ -630,7 +641,7 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
             },
             |rows| {
                 rows.filter_map(|v| v.transpose())
-                    .collect::<Result<Vec<_>, _>>()
+                    .collect::<std::result::Result<Vec<_>, _>>()
                     .map_err(Into::into)
             },
         )
@@ -690,7 +701,7 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
     }
 
     if continue_streaming {
-        schedule_maybe_send_locations(context, true).await;
+        job_try!(schedule_maybe_send_locations(context, true).await);
     }
     job::Status::Finished(Ok(()))
 }
@@ -736,7 +747,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
             );
 
             let stock_str = stock_str::msg_location_disabled(context).await;
-            chat::add_info_msg(context, chat_id, stock_str, now).await;
+            job_try!(chat::add_info_msg(context, chat_id, stock_str, now).await);
             context.emit_event(EventType::ChatModified(chat_id));
         }
     }

src/log.rs

@@ -1,6 +1,7 @@
 //! # Logging.
 
 use crate::context::Context;
+use async_std::task::block_on;
 
 #[macro_export]
 macro_rules! info {
@@ -13,7 +14,7 @@ macro_rules! info {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::EventType::Info(full));
+        $ctx.emit_event($crate::EventType::Info(full));
     }};
 }
 
@@ -28,7 +29,7 @@ macro_rules! warn {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::EventType::Warning(full));
+        $ctx.emit_event($crate::EventType::Warning(full));
     }};
 }
 
@@ -39,15 +40,26 @@ macro_rules! error {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        emit_event!($ctx, $crate::EventType::Error(formatted));
+        $ctx.set_last_error(&formatted);
+        $ctx.emit_event($crate::EventType::Error(formatted));
     }};
 }
 
-#[macro_export]
-macro_rules! emit_event {
-    ($ctx:expr, $event:expr) => {
-        $ctx.emit_event($event);
-    };
+impl Context {
+    /// Set last error string.
+    /// Implemented as blocking as used from macros in different, not always async blocks.
+    pub fn set_last_error(&self, error: &str) {
+        block_on(async move {
+            let mut last_error = self.last_error.write().await;
+            *last_error = error.to_string();
+        });
+    }
+
+    /// Get last error string.
+    pub async fn get_last_error(&self) -> String {
+        let last_error = &*self.last_error.read().await;
+        last_error.clone()
+    }
 }
 
 pub trait LogExt<T, E>
@@ -136,8 +148,36 @@ impl<T, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
             );
             // We can't use the warn!() macro here as the file!() and line!() macros
             // don't work with #[track_caller]
-            emit_event!(context, crate::EventType::Warning(full));
+            context.emit_event(crate::EventType::Warning(full));
         };
         self
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::test_utils::TestContext;
+    use anyhow::Result;
+
+    #[async_std::test]
+    async fn test_get_last_error() -> Result<()> {
+        let t = TestContext::new().await;
+
+        assert_eq!(t.get_last_error().await, "");
+
+        error!(t, "foo-error");
+        assert_eq!(t.get_last_error().await, "foo-error");
+
+        warn!(t, "foo-warning");
+        assert_eq!(t.get_last_error().await, "foo-error");
+
+        info!(t, "foo-info");
+        assert_eq!(t.get_last_error().await, "foo-error");
+
+        error!(t, "bar-error");
+        error!(t, "baz-error");
+        assert_eq!(t.get_last_error().await, "baz-error");
+
+        Ok(())
+    }
+}

src/login_param.rs

@@ -11,8 +11,10 @@ use anyhow::Result;
 use async_std::io;
 use async_std::net::TcpStream;
 
+use async_native_tls::Certificate;
 pub use async_smtp::ServerAddress;
 use fast_socks5::client::Socks5Stream;
+use once_cell::sync::Lazy;
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(u32)]
@@ -368,8 +370,18 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
+// this certificate is missing on older android devices (eg. lg with android6 from 2017)
+// certificate downloaded from https://letsencrypt.org/certificates/
+static LETSENCRYPT_ROOT: Lazy<Certificate> = Lazy::new(|| {
+    Certificate::from_der(include_bytes!(
+        "../assets/root-certificates/letsencrypt/isrgrootx1.der"
+    ))
+    .unwrap()
+});
+
 pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
-    let tls_builder = async_native_tls::TlsConnector::new();
+    let tls_builder =
+        async_native_tls::TlsConnector::new().add_root_certificate(LETSENCRYPT_ROOT.clone());
 
     if strict_tls {
         tls_builder
@@ -430,4 +442,13 @@ mod tests {
         assert_eq!(param, loaded);
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_build_tls() -> Result<()> {
+        // we are using some additional root certificates.
+        // make sure, they do not break construction of TlsConnector
+        let _ = dc_build_tls(true);
+        let _ = dc_build_tls(false);
+        Ok(())
+    }
 }

src/lot.rs

@@ -1,141 +0,0 @@
-//! # Legacy generic return values for C API.
-
-use deltachat_derive::{FromSql, ToSql};
-
-use crate::key::Fingerprint;
-
-/// An object containing a set of values.
-/// The meaning of the values is defined by the function returning the object.
-/// Lot objects are created
-/// eg. by chatlist.get_summary() or dc_msg_get_summary().
-///
-/// *Lot* is used in the meaning *heap* here.
-#[derive(Default, Debug, Clone)]
-pub struct Lot {
-    pub(crate) text1_meaning: Meaning,
-    pub(crate) text1: Option<String>,
-    pub(crate) text2: Option<String>,
-    pub(crate) timestamp: i64,
-    pub(crate) state: LotState,
-    pub(crate) id: u32,
-    pub(crate) fingerprint: Option<Fingerprint>,
-    pub(crate) invitenumber: Option<String>,
-    pub(crate) auth: Option<String>,
-}
-
-#[repr(u8)]
-#[derive(
-    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
-)]
-pub enum Meaning {
-    None = 0,
-    Text1Draft = 1,
-    Text1Username = 2,
-    Text1Self = 3,
-}
-
-impl Default for Meaning {
-    fn default() -> Self {
-        Meaning::None
-    }
-}
-
-impl Lot {
-    pub fn new() -> Self {
-        Default::default()
-    }
-
-    pub fn get_text1(&self) -> Option<&str> {
-        self.text1.as_deref()
-    }
-
-    pub fn get_text2(&self) -> Option<&str> {
-        self.text2.as_deref()
-    }
-
-    pub fn get_text1_meaning(&self) -> Meaning {
-        self.text1_meaning
-    }
-
-    pub fn get_state(&self) -> LotState {
-        self.state
-    }
-
-    pub fn get_id(&self) -> u32 {
-        self.id
-    }
-
-    pub fn get_timestamp(&self) -> i64 {
-        self.timestamp
-    }
-}
-
-#[repr(u32)]
-#[derive(
-    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
-)]
-pub enum LotState {
-    // Default
-    Undefined = 0,
-
-    // Qr States
-    /// id=contact
-    QrAskVerifyContact = 200,
-
-    /// text1=groupname
-    QrAskVerifyGroup = 202,
-
-    /// id=contact
-    QrFprOk = 210,
-
-    /// id=contact
-    QrFprMismatch = 220,
-
-    /// test1=formatted fingerprint
-    QrFprWithoutAddr = 230,
-
-    /// text1=domain
-    QrAccount = 250,
-
-    /// text1=domain, text2=instance pattern
-    QrWebrtcInstance = 260,
-
-    /// id=contact
-    QrAddr = 320,
-
-    /// text1=text
-    QrText = 330,
-
-    /// text1=URL
-    QrUrl = 332,
-
-    /// text1=error string
-    QrError = 400,
-
-    QrWithdrawVerifyContact = 500,
-
-    /// text1=groupname
-    QrWithdrawVerifyGroup = 502,
-
-    QrReviveVerifyContact = 510,
-
-    /// text1=groupname
-    QrReviveVerifyGroup = 512,
-
-    // Message States
-    MsgInFresh = 10,
-    MsgInNoticed = 13,
-    MsgInSeen = 16,
-    MsgOutPreparing = 18,
-    MsgOutDraft = 19,
-    MsgOutPending = 20,
-    MsgOutFailed = 24,
-    MsgOutDelivered = 26,
-    MsgOutMdnRcvd = 28,
-}
-
-impl Default for LotState {
-    fn default() -> Self {
-        LotState::Undefined
-    }
-}

src/message.rs

@@ -3,10 +3,9 @@
 use std::collections::BTreeMap;
 use std::convert::TryInto;
 
-use anyhow::{ensure, format_err, Result};
+use anyhow::{ensure, format_err, Context as _, Result};
 use async_std::path::{Path, PathBuf};
 use deltachat_derive::{FromSql, ToSql};
-use itertools::Itertools;
 use rusqlite::types::ValueRef;
 use serde::{Deserialize, Serialize};
 
@@ -22,19 +21,16 @@ use crate::dc_tools::{
     dc_create_smeared_timestamp, dc_get_filebytes, dc_get_filemeta, dc_gm2local_offset,
     dc_read_file, dc_timestamp_to_str, dc_truncate, time,
 };
+use crate::download::DownloadState;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::events::EventType;
 use crate::job::{self, Action};
 use crate::log::LogExt;
-use crate::lot::{Lot, LotState, Meaning};
 use crate::mimeparser::{parse_message_id, FailureReport, SystemMessage};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
 use crate::stock_str;
-
-// In practice, the user additionally cuts the string themselves
-// pixel-accurate.
-const SUMMARY_CHARACTERS: usize = 160;
+use crate::summary::Summary;
 
 /// Message ID, including reserved IDs.
 ///
@@ -301,6 +297,7 @@ pub struct Message {
     pub(crate) chat_id: ChatId,
     pub(crate) viewtype: Viewtype,
     pub(crate) state: MessageState,
+    pub(crate) download_state: DownloadState,
     pub(crate) hidden: bool,
     pub(crate) timestamp_sort: i64,
     pub(crate) timestamp_sent: i64,
@@ -355,6 +352,7 @@ impl Message {
                     "    m.ephemeral_timestamp AS ephemeral_timestamp,",
                     "    m.type AS type,",
                     "    m.state AS state,",
+                    "    m.download_state AS download_state,",
                     "    m.error AS error,",
                     "    m.msgrmsg AS msgrmsg,",
                     "    m.mime_modified AS mime_modified,",
@@ -406,6 +404,7 @@ impl Message {
                         ephemeral_timestamp: row.get("ephemeral_timestamp")?,
                         viewtype: row.get("type")?,
                         state: row.get("state")?,
+                        download_state: row.get("download_state")?,
                         error: Some(row.get::<_, String>("error")?)
                             .filter(|error| !error.is_empty()),
                         is_dc_message: row.get("msgrmsg")?,
@@ -592,23 +591,21 @@ impl Message {
         self.ephemeral_timestamp
     }
 
-    pub async fn get_summary(&mut self, context: &Context, chat: Option<&Chat>) -> Lot {
-        let mut ret = Lot::new();
-
+    /// Returns message summary for display in the search results.
+    pub async fn get_summary(&self, context: &Context, chat: Option<&Chat>) -> Result<Summary> {
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, self.chat_id).await {
+        } else {
+            let chat = Chat::load_from_db(context, self.chat_id).await?;
             chat_loaded = chat;
             &chat_loaded
-        } else {
-            return ret;
         };
 
         let contact = if self.from_id != DC_CONTACT_ID_SELF {
             match chat.typ {
-                Chattype::Group | Chattype::Mailinglist => {
-                    Contact::get_by_id(context, self.from_id).await.ok()
+                Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
+                    Some(Contact::get_by_id(context, self.from_id).await?)
                 }
                 Chattype::Single | Chattype::Undefined => None,
             }
@@ -616,21 +613,7 @@ impl Message {
             None
         };
 
-        ret.fill(self, chat, contact.as_ref(), context).await;
-
-        ret
-    }
-
-    pub async fn get_summarytext(&self, context: &Context, approx_characters: usize) -> String {
-        get_summarytext_by_raw(
-            self.viewtype,
-            self.text.as_ref(),
-            self.is_forwarded(),
-            &self.param,
-            approx_characters,
-            context,
-        )
-        .await
+        Ok(Summary::new(context, self, chat, contact.as_ref()).await)
     }
 
     // It's a little unfortunate that the UI has to first call dc_msg_get_override_sender_name() and then if it was NULL, call
@@ -876,7 +859,11 @@ impl Message {
             Param::Quote,
             if text.is_empty() {
                 // Use summary, similar to "Image" to avoid sending empty quote.
-                quote.get_summarytext(context, 500).await
+                quote
+                    .get_summary(context, None)
+                    .await?
+                    .truncated_text(500)
+                    .to_string()
             } else {
                 text
             },
@@ -1029,24 +1016,6 @@ impl std::fmt::Display for MessageState {
     }
 }
 
-impl From<MessageState> for LotState {
-    fn from(s: MessageState) -> Self {
-        use MessageState::*;
-        match s {
-            Undefined => LotState::Undefined,
-            InFresh => LotState::MsgInFresh,
-            InNoticed => LotState::MsgInNoticed,
-            InSeen => LotState::MsgInSeen,
-            OutPreparing => LotState::MsgOutPreparing,
-            OutDraft => LotState::MsgOutDraft,
-            OutPending => LotState::MsgOutPending,
-            OutFailed => LotState::MsgOutFailed,
-            OutDelivered => LotState::MsgOutDelivered,
-            OutMdnRcvd => LotState::MsgOutMdnRcvd,
-        }
-    }
-}
-
 impl MessageState {
     pub fn can_fail(self) -> bool {
         use MessageState::*;
@@ -1064,68 +1033,6 @@ impl MessageState {
     }
 }
 
-impl Lot {
-    /* library-internal */
-    /* in practice, the user additionally cuts the string himself pixel-accurate */
-    pub async fn fill(
-        &mut self,
-        msg: &mut Message,
-        chat: &Chat,
-        contact: Option<&Contact>,
-        context: &Context,
-    ) {
-        if msg.state == MessageState::OutDraft {
-            self.text1 = Some(stock_str::draft(context).await);
-            self.text1_meaning = Meaning::Text1Draft;
-        } else if msg.from_id == DC_CONTACT_ID_SELF {
-            if msg.is_info() || chat.is_self_talk() {
-                self.text1 = None;
-                self.text1_meaning = Meaning::None;
-            } else {
-                self.text1 = Some(stock_str::self_msg(context).await);
-                self.text1_meaning = Meaning::Text1Self;
-            }
-        } else {
-            match chat.typ {
-                Chattype::Group | Chattype::Mailinglist => {
-                    if msg.is_info() || contact.is_none() {
-                        self.text1 = None;
-                        self.text1_meaning = Meaning::None;
-                    } else {
-                        self.text1 = msg
-                            .get_override_sender_name()
-                            .or_else(|| contact.map(|contact| msg.get_sender_name(contact)));
-                        self.text1_meaning = Meaning::Text1Username;
-                    }
-                }
-                Chattype::Single | Chattype::Undefined => {
-                    self.text1 = None;
-                    self.text1_meaning = Meaning::None;
-                }
-            }
-        }
-
-        let mut text2 = get_summarytext_by_raw(
-            msg.viewtype,
-            msg.text.as_ref(),
-            msg.is_forwarded(),
-            &msg.param,
-            SUMMARY_CHARACTERS,
-            context,
-        )
-        .await;
-
-        if text2.is_empty() && msg.quoted_text().is_some() {
-            text2 = stock_str::reply_noun(context).await
-        }
-
-        self.text2 = Some(text2);
-
-        self.timestamp = msg.get_timestamp();
-        self.state = msg.state.into();
-    }
-}
-
 pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
     let msg = Message::load_from_db(context, msg_id).await?;
     let rawtxt: Option<String> = context
@@ -1366,21 +1273,21 @@ pub async fn get_mime_headers(context: &Context, msg_id: MsgId) -> Result<Vec<u8
     Ok(headers)
 }
 
-pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
+pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     for msg_id in msg_ids.iter() {
-        if let Ok(msg) = Message::load_from_db(context, *msg_id).await {
-            if msg.location_id > 0 {
-                delete_poi_location(context, msg.location_id).await;
-            }
-        }
-        if let Err(err) = msg_id.trash(context).await {
-            error!(context, "Unable to trash message {}: {}", msg_id, err);
+        let msg = Message::load_from_db(context, *msg_id).await?;
+        if msg.location_id > 0 {
+            delete_poi_location(context, msg.location_id).await;
         }
+        msg_id
+            .trash(context)
+            .await
+            .with_context(|| format!("Unable to trash message {}", msg_id))?;
         job::add(
             context,
             job::Job::new(Action::DeleteMsgOnImap, msg_id.to_u32(), Params::new(), 0),
         )
-        .await;
+        .await?;
     }
 
     if !msg_ids.is_empty() {
@@ -1388,13 +1295,14 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
-        job::kill_action(context, Action::Housekeeping).await;
+        job::kill_action(context, Action::Housekeeping).await?;
         job::add(
             context,
             job::Job::new(Action::Housekeeping, 0, Params::new(), 10),
         )
-        .await;
+        .await?;
     }
+    Ok(())
 }
 
 async fn delete_poi_location(context: &Context, location_id: u32) -> bool {
@@ -1467,7 +1375,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
                 context,
                 job::Job::new(Action::MarkseenMsgOnImap, id.to_u32(), Params::new(), 0),
             )
-            .await;
+            .await?;
             updated_chat_ids.insert(curr_chat_id, true);
         }
     }
@@ -1490,88 +1398,6 @@ pub async fn update_msg_state(context: &Context, msg_id: MsgId, state: MessageSt
         .is_ok()
 }
 
-/// Returns a summary text.
-pub async fn get_summarytext_by_raw(
-    viewtype: Viewtype,
-    text: Option<impl AsRef<str>>,
-    was_forwarded: bool,
-    param: &Params,
-    approx_characters: usize,
-    context: &Context,
-) -> String {
-    let mut append_text = true;
-    let prefix = match viewtype {
-        Viewtype::Image => stock_str::image(context).await,
-        Viewtype::Gif => stock_str::gif(context).await,
-        Viewtype::Sticker => stock_str::sticker(context).await,
-        Viewtype::Video => stock_str::video(context).await,
-        Viewtype::Voice => stock_str::voice_message(context).await,
-        Viewtype::Audio | Viewtype::File => {
-            if param.get_cmd() == SystemMessage::AutocryptSetupMessage {
-                append_text = false;
-                stock_str::ac_setup_msg_subject(context).await
-            } else {
-                let file_name: String = param
-                    .get_path(Param::File, context)
-                    .unwrap_or(None)
-                    .and_then(|path| {
-                        path.file_name()
-                            .map(|fname| fname.to_string_lossy().into_owned())
-                    })
-                    .unwrap_or_else(|| String::from("ErrFileName"));
-                let label = if viewtype == Viewtype::Audio {
-                    stock_str::audio(context).await
-                } else {
-                    stock_str::file(context).await
-                };
-                format!("{} – {}", label, file_name)
-            }
-        }
-        Viewtype::VideochatInvitation => {
-            append_text = false;
-            stock_str::videochat_invitation(context).await
-        }
-        _ => {
-            if param.get_cmd() != SystemMessage::LocationOnly {
-                "".to_string()
-            } else {
-                append_text = false;
-                stock_str::location(context).await
-            }
-        }
-    };
-
-    if !append_text {
-        return prefix;
-    }
-
-    let summary_content = if let Some(text) = text {
-        if text.as_ref().is_empty() {
-            prefix
-        } else if prefix.is_empty() {
-            dc_truncate(text.as_ref(), approx_characters).to_string()
-        } else {
-            let tmp = format!("{} – {}", prefix, text.as_ref());
-            dc_truncate(&tmp, approx_characters).to_string()
-        }
-    } else {
-        prefix
-    };
-
-    let summary = if was_forwarded {
-        let tmp = format!(
-            "{}: {}",
-            stock_str::forwarded(context).await,
-            summary_content
-        );
-        dc_truncate(&tmp, approx_characters).to_string()
-    } else {
-        summary_content
-    };
-
-    summary.split_whitespace().join(" ")
-}
-
 // as we do not cut inside words, this results in about 32-42 characters.
 // Do not use too long subjects - we add a tag after the subject which gets truncated by the clients otherwise.
 // It should also be very clear, the subject is _not_ the whole message.
@@ -1635,6 +1461,17 @@ pub async fn handle_mdn(
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Result<Option<(ChatId, MsgId)>> {
+    if from_id == DC_CONTACT_ID_SELF {
+        warn!(
+            context,
+            "ignoring MDN sent to self, this is a bug on the sender device"
+        );
+
+        // This is not an error on our side,
+        // we successfully ignored an invalid MDN and return `Ok`.
+        return Ok(None);
+    }
+
     let res = context
         .sql
         .query_row_optional(
@@ -1642,7 +1479,6 @@ pub async fn handle_mdn(
                 "SELECT",
                 "    m.id AS msg_id,",
                 "    c.id AS chat_id,",
-                "    c.type AS type,",
                 "    m.state AS state",
                 " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
                 " WHERE rfc724_mid=? AND from_id=1",
@@ -1653,14 +1489,13 @@ pub async fn handle_mdn(
                 Ok((
                     row.get::<_, MsgId>("msg_id")?,
                     row.get::<_, ChatId>("chat_id")?,
-                    row.get::<_, Chattype>("type")?,
                     row.get::<_, MessageState>("state")?,
                 ))
             },
         )
         .await?;
 
-    let (msg_id, chat_id, chat_type, msg_state) = if let Some(res) = res {
+    let (msg_id, chat_id, msg_state) = if let Some(res) = res {
         res
     } else {
         info!(
@@ -1671,63 +1506,28 @@ pub async fn handle_mdn(
         return Ok(None);
     };
 
-    let mut read_by_all = false;
+    if !context
+        .sql
+        .exists(
+            "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=? AND contact_id=?;",
+            paramsv![msg_id, from_id as i32,],
+        )
+        .await?
+    {
+        context
+            .sql
+            .execute(
+                "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?);",
+                paramsv![msg_id, from_id as i32, timestamp_sent],
+            )
+            .await?;
+    }
+
     if msg_state == MessageState::OutPreparing
         || msg_state == MessageState::OutPending
         || msg_state == MessageState::OutDelivered
     {
-        let mdn_already_in_table = context
-            .sql
-            .exists(
-                "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=? AND contact_id=?;",
-                paramsv![msg_id, from_id as i32,],
-            )
-            .await?;
-
-        if !mdn_already_in_table {
-            context
-                .sql
-                .execute(
-                    "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?);",
-                    paramsv![msg_id, from_id as i32, timestamp_sent],
-                )
-                .await?;
-        }
-
-        // Normal chat? that's quite easy.
-        if chat_type == Chattype::Single {
-            update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await;
-            read_by_all = true;
-        } else {
-            // send event about new state
-            let ist_cnt = context
-                .sql
-                .count(
-                    "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=?;",
-                    paramsv![msg_id],
-                )
-                .await?;
-
-            // Groupsize:  Min. MDNs
-            // 1 S         n/a
-            // 2 SR        1
-            // 3 SRR       2
-            // 4 SRRR      2
-            // 5 SRRRR     3
-            // 6 SRRRRR    3
-            //
-            // (S=Sender, R=Recipient)
-
-            // for rounding, SELF is already included!
-            let soll_cnt = (chat::get_chat_contact_cnt(context, chat_id).await? + 1) / 2;
-            if ist_cnt >= soll_cnt {
-                update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await;
-                read_by_all = true;
-            } // else wait for more receipts
-        }
-    }
-
-    if read_by_all {
+        update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await;
         Ok(Some((chat_id, msg_id)))
     } else {
         Ok(None)
@@ -1791,7 +1591,7 @@ async fn ndn_maybe_add_info_msg(
     chat_type: Chattype,
 ) -> Result<()> {
     match chat_type {
-        Chattype::Group => {
+        Chattype::Group | Chattype::Broadcast => {
             if let Some(failed_recipient) = &failed.failed_recipient {
                 let contact_id =
                     Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown)
@@ -1809,7 +1609,7 @@ async fn ndn_maybe_add_info_msg(
                     text,
                     dc_create_smeared_timestamp(context).await,
                 )
-                .await;
+                .await?;
                 context.emit_event(EventType::ChatModified(chat_id));
             }
         }
@@ -2238,203 +2038,6 @@ mod tests {
         assert!(chat::prepare_msg(ctx, chat.id, &mut msg).await.is_err());
     }
 
-    #[async_std::test]
-    async fn test_get_summarytext_by_raw() {
-        let d = test::TestContext::new().await;
-        let ctx = &d.ctx;
-
-        let some_text = Some(" bla \t\n\tbla\n\t".to_string());
-        let empty_text = Some("".to_string());
-        let no_text: Option<String> = None;
-
-        let mut some_file = Params::new();
-        some_file.set(Param::File, "foo.bar");
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Text,
-                some_text.as_ref(),
-                false,
-                &Params::new(),
-                50,
-                ctx
-            )
-            .await,
-            "bla bla" // for simple text, the type is not added to the summary
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Image,
-                no_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx
-            )
-            .await,
-            "Image" // file names are not added for images
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Video,
-                no_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx
-            )
-            .await,
-            "Video" // file names are not added for videos
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(Viewtype::Gif, no_text.as_ref(), false, &some_file, 50, ctx,)
-                .await,
-            "GIF" // file names are not added for GIFs
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Sticker,
-                no_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx,
-            )
-            .await,
-            "Sticker" // file names are not added for stickers
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Voice,
-                empty_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx,
-            )
-            .await,
-            "Voice message" // file names are not added for voice messages, empty text is skipped
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Voice,
-                no_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx
-            )
-            .await,
-            "Voice message" // file names are not added for voice messages
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Voice,
-                some_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx
-            )
-            .await,
-            "Voice message \u{2013} bla bla" // `\u{2013}` explicitly checks for "EN DASH"
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Audio,
-                no_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx
-            )
-            .await,
-            "Audio \u{2013} foo.bar" // file name is added for audio
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Audio,
-                empty_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx,
-            )
-            .await,
-            "Audio \u{2013} foo.bar" // file name is added for audio, empty text is not added
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Audio,
-                some_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx
-            )
-            .await,
-            "Audio \u{2013} foo.bar \u{2013} bla bla" // file name and text added for audio
-        );
-
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::File,
-                some_text.as_ref(),
-                false,
-                &some_file,
-                50,
-                ctx
-            )
-            .await,
-            "File \u{2013} foo.bar \u{2013} bla bla" // file name is added for files
-        );
-
-        // Forwarded
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::Text,
-                some_text.as_ref(),
-                true,
-                &Params::new(),
-                50,
-                ctx
-            )
-            .await,
-            "Forwarded: bla bla" // for simple text, the type is not added to the summary
-        );
-        assert_eq!(
-            get_summarytext_by_raw(
-                Viewtype::File,
-                some_text.as_ref(),
-                true,
-                &some_file,
-                50,
-                ctx
-            )
-            .await,
-            "Forwarded: File \u{2013} foo.bar \u{2013} bla bla"
-        );
-
-        let mut asm_file = Params::new();
-        asm_file.set(Param::File, "foo.bar");
-        asm_file.set_cmd(SystemMessage::AutocryptSetupMessage);
-        assert_eq!(
-            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), false, &asm_file, 50, ctx)
-                .await,
-            "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
-        );
-    }
-
     #[async_std::test]
     async fn test_parse_webrtc_instance() {
         let (webrtc_type, url) = Message::parse_webrtc_instance("basicwebrtc:https://foo/bar");

src/mimefactory.rs

@@ -68,6 +68,13 @@ pub struct MimeFactory<'a> {
     references: String,
     req_mdn: bool,
     last_added_location_id: u32,
+
+    /// If the created mime-structure contains sync-items,
+    /// the IDs of these items are listed here.
+    /// The IDs are returned via `RenderedEmail`
+    /// and must be deleted if the message is actually queued for sending.
+    sync_ids_to_delete: Option<String>,
+
     attach_selfavatar: bool,
 }
 
@@ -80,6 +87,12 @@ pub struct RenderedEmail {
     pub is_gossiped: bool,
     pub last_added_location_id: u32,
 
+    /// A comma-separated string of sync-IDs that are used by the rendered email
+    /// and must be deleted once the message is actually queued for sending
+    /// (deletion must be done by `delete_sync_ids()`).
+    /// If the rendered email is not queued for sending, the IDs must not be deleted.
+    pub sync_ids_to_delete: Option<String>,
+
     /// Message ID (Message in the sense of Email)
     pub rfc724_mid: String,
     pub subject: String,
@@ -205,6 +218,7 @@ impl<'a> MimeFactory<'a> {
             references,
             req_mdn,
             last_added_location_id: 0,
+            sync_ids_to_delete: None,
             attach_selfavatar,
         };
         Ok(factory)
@@ -249,6 +263,7 @@ impl<'a> MimeFactory<'a> {
             references: String::default(),
             req_mdn: false,
             last_added_location_id: 0,
+            sync_ids_to_delete: None,
             attach_selfavatar: false,
         };
 
@@ -316,6 +331,10 @@ impl<'a> MimeFactory<'a> {
             Loaded::Message { chat } => {
                 if chat.is_protected() {
                     false
+                } else if chat.typ == Chattype::Broadcast {
+                    // encryption may disclose recipients;
+                    // this is probably a worse issue than not opportunistically (!) encrypting
+                    true
                 } else {
                     self.msg
                         .param
@@ -342,7 +361,7 @@ impl<'a> MimeFactory<'a> {
         match &self.loaded {
             Loaded::Message { chat } => {
                 // beside key- and member-changes, force re-gossip every 48 hours
-                let gossiped_timestamp = chat.get_gossiped_timestamp(context).await?;
+                let gossiped_timestamp = chat.id.get_gossiped_timestamp(context).await?;
                 if time() > gossiped_timestamp + (2 * 24 * 60 * 60) {
                     Ok(true)
                 } else {
@@ -388,17 +407,11 @@ impl<'a> MimeFactory<'a> {
 
         let subject = match self.loaded {
             Loaded::Message { ref chat } => {
-                if self.msg.param.get_cmd() == SystemMessage::AutocryptSetupMessage {
-                    return Ok(stock_str::ac_setup_msg_subject(context).await);
-                }
-
                 if !self.msg.subject.is_empty() {
                     return Ok(self.msg.subject.clone());
                 }
 
                 if chat.typ == Chattype::Group && quoted_msg_subject.is_none_or_empty() {
-                    // If we have a `quoted_msg_subject`, we use the subject of the quoted message
-                    // instead of the group name
                     let re = if self.in_reply_to.is_empty() {
                         ""
                     } else {
@@ -407,22 +420,22 @@ impl<'a> MimeFactory<'a> {
                     return Ok(format!("{}{}", re, chat.name));
                 }
 
-                let parent_subject = if quoted_msg_subject.is_none_or_empty() {
-                    chat.param.get(Param::LastSubject)
-                } else {
-                    quoted_msg_subject.as_deref()
-                };
-
-                if let Some(last_subject) = parent_subject {
-                    format!("Re: {}", remove_subject_prefix(last_subject))
-                } else {
-                    let self_name = match context.get_config(Config::Displayname).await? {
-                        Some(name) => name,
-                        None => context.get_config(Config::Addr).await?.unwrap_or_default(),
+                if chat.typ != Chattype::Broadcast {
+                    let parent_subject = if quoted_msg_subject.is_none_or_empty() {
+                        chat.param.get(Param::LastSubject)
+                    } else {
+                        quoted_msg_subject.as_deref()
                     };
-
-                    stock_str::subject_for_new_contact(context, self_name).await
+                    if let Some(last_subject) = parent_subject {
+                        return Ok(format!("Re: {}", remove_subject_prefix(last_subject)));
+                    }
                 }
+
+                let self_name = match context.get_config(Config::Displayname).await? {
+                    Some(name) => name,
+                    None => context.get_config(Config::Addr).await?.unwrap_or_default(),
+                };
+                stock_str::subject_for_new_contact(context, self_name).await
             }
             Loaded::Mdn { .. } => stock_str::read_rcpt(context).await,
         };
@@ -559,9 +572,21 @@ impl<'a> MimeFactory<'a> {
             render_rfc724_mid(&rfc724_mid),
         ));
 
-        headers
-            .unprotected
-            .push(Header::new_with_value("To".into(), to).unwrap());
+        let undisclosed_recipients = match &self.loaded {
+            Loaded::Message { chat } => chat.typ == Chattype::Broadcast,
+            Loaded::Mdn { .. } => false,
+        };
+
+        if undisclosed_recipients {
+            headers
+                .unprotected
+                .push(Header::new("To".into(), "hidden-recipients: ;".to_string()));
+        } else {
+            headers
+                .unprotected
+                .push(Header::new_with_value("To".into(), to).unwrap());
+        }
+
         headers
             .unprotected
             .push(Header::new_with_value("From".into(), vec![from]).unwrap());
@@ -593,12 +618,20 @@ impl<'a> MimeFactory<'a> {
             main_part
         } else {
             // Multiple parts, render as multipart.
-            parts.into_iter().fold(
-                PartBuilder::new()
-                    .message_type(MimeMultipartType::Mixed)
-                    .child(main_part.build()),
-                |message, part| message.child(part.build()),
-            )
+            let part_holder = if self.msg.param.get_cmd() == SystemMessage::MultiDeviceSync {
+                PartBuilder::new().header((
+                    "Content-Type".to_string(),
+                    "multipart/report; report-type=multi-device-sync".to_string(),
+                ))
+            } else {
+                PartBuilder::new().message_type(MimeMultipartType::Mixed)
+            };
+
+            parts
+                .into_iter()
+                .fold(part_holder.child(main_part.build()), |message, part| {
+                    message.child(part.build())
+                })
         };
 
         let outer_message = if is_encrypted {
@@ -719,6 +752,7 @@ impl<'a> MimeFactory<'a> {
             is_encrypted,
             is_gossiped,
             last_added_location_id,
+            sync_ids_to_delete: self.sync_ids_to_delete,
             rfc724_mid,
             subject: subject_str,
         })
@@ -863,7 +897,7 @@ impl<'a> MimeFactory<'a> {
                     "ephemeral-timer-changed".to_string(),
                 ));
             }
-            SystemMessage::LocationOnly => {
+            SystemMessage::LocationOnly | SystemMessage::MultiDeviceSync => {
                 // This should prevent automatic replies,
                 // such as non-delivery reports.
                 //
@@ -1084,7 +1118,7 @@ impl<'a> MimeFactory<'a> {
             parts.push(msg_kml_part);
         }
 
-        if location::is_sending_locations_to_chat(context, Some(self.msg.chat_id)).await {
+        if location::is_sending_locations_to_chat(context, Some(self.msg.chat_id)).await? {
             match self.get_location_kml_part(context).await {
                 Ok(part) => parts.push(part),
                 Err(err) => {
@@ -1093,6 +1127,15 @@ impl<'a> MimeFactory<'a> {
             }
         }
 
+        // we do not piggyback sync-files to other self-sent-messages
+        // to not risk files becoming too larger and being skipped by download-on-demand.
+        if command == SystemMessage::MultiDeviceSync && self.is_e2ee_guaranteed() {
+            let json = self.msg.param.get(Param::Arg).unwrap_or_default();
+            let ids = self.msg.param.get(Param::Arg2).unwrap_or_default();
+            parts.push(context.build_sync_part(json.to_string()).await);
+            self.sync_ids_to_delete = Some(ids.to_string());
+        }
+
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar).await? {
                 Some(path) => match build_selfavatar_file(context, &path) {
@@ -1147,7 +1190,11 @@ impl<'a> MimeFactory<'a> {
         {
             stock_str::encrypted_msg(context).await
         } else {
-            self.msg.get_summarytext(context, 32).await
+            self.msg
+                .get_summary(context, None)
+                .await?
+                .truncated_text(32)
+                .to_string()
         };
         let p2 = stock_str::read_rcpt_mail_body(context, p1).await;
         let message_text = format!("{}\r\n", format_flowed(&p2));
@@ -1609,27 +1656,30 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_subject_in_group() {
+    async fn test_subject_in_group() -> Result<()> {
         async fn send_msg_get_subject(
             t: &TestContext,
             group_id: ChatId,
             quote: Option<&Message>,
-        ) -> String {
+        ) -> Result<String> {
             let mut new_msg = Message::new(Viewtype::Text);
             new_msg.set_text(Some("Hi".to_string()));
             if let Some(q) = quote {
-                new_msg.set_quote(t, q).await.unwrap();
+                new_msg.set_quote(t, q).await?;
             }
             let sent = t.send_msg(group_id, &mut new_msg).await;
             get_subject(t, sent).await
         }
-        async fn get_subject(t: &TestContext, sent: crate::test_utils::SentMessage) -> String {
+        async fn get_subject(
+            t: &TestContext,
+            sent: crate::test_utils::SentMessage,
+        ) -> Result<String> {
             let parsed_subject = t.parse_msg(&sent).await.get_subject().unwrap();
 
-            let sent_msg = Message::load_from_db(t, sent.sender_msg_id).await.unwrap();
+            let sent_msg = Message::load_from_db(t, sent.sender_msg_id).await?;
             assert_eq!(parsed_subject, sent_msg.subject);
 
-            parsed_subject
+            Ok(parsed_subject)
         }
 
         // 6. Test that in a group, replies also take the quoted message's subject, while non-replies use the group title as subject
@@ -1638,13 +1688,13 @@ mod tests {
             chat::create_group_chat(&t, chat::ProtectionStatus::Unprotected, "groupname") // TODO encodings, ä
                 .await
                 .unwrap();
-        let bob = Contact::create(&t, "", "bob@example.org").await.unwrap();
-        chat::add_contact_to_chat(&t, group_id, bob).await;
+        let bob = Contact::create(&t, "", "bob@example.org").await?;
+        chat::add_contact_to_chat(&t, group_id, bob).await?;
 
-        let subject = send_msg_get_subject(&t, group_id, None).await;
+        let subject = send_msg_get_subject(&t, group_id, None).await?;
         assert_eq!(subject, "groupname");
 
-        let subject = send_msg_get_subject(&t, group_id, None).await;
+        let subject = send_msg_get_subject(&t, group_id, None).await?;
         assert_eq!(subject, "Re: groupname");
 
         dc_receive_imf(
@@ -1666,28 +1716,26 @@ mod tests {
             5,
             false,
         )
-        .await
-        .unwrap();
+        .await?;
         let message_from_bob = t.get_last_msg().await;
 
-        let subject = send_msg_get_subject(&t, group_id, None).await;
+        let subject = send_msg_get_subject(&t, group_id, None).await?;
         assert_eq!(subject, "Re: groupname");
 
-        let subject = send_msg_get_subject(&t, group_id, Some(&message_from_bob)).await;
+        let subject = send_msg_get_subject(&t, group_id, Some(&message_from_bob)).await?;
         let outgoing_quoting_msg = t.get_last_msg().await;
         assert_eq!(subject, "Re: Different subject");
 
-        let subject = send_msg_get_subject(&t, group_id, None).await;
+        let subject = send_msg_get_subject(&t, group_id, None).await?;
         assert_eq!(subject, "Re: groupname");
 
-        let subject = send_msg_get_subject(&t, group_id, Some(&outgoing_quoting_msg)).await;
+        let subject = send_msg_get_subject(&t, group_id, Some(&outgoing_quoting_msg)).await?;
         assert_eq!(subject, "Re: Different subject");
 
-        chat::forward_msgs(&t, &[message_from_bob.id], group_id)
-            .await
-            .unwrap();
-        let subject = get_subject(&t, t.pop_sent_msg().await).await;
-        assert_eq!(subject, "Fwd: Different subject");
+        chat::forward_msgs(&t, &[message_from_bob.id], group_id).await?;
+        let subject = get_subject(&t, t.pop_sent_msg().await).await?;
+        assert_eq!(subject, "Re: groupname");
+        Ok(())
     }
 
     async fn first_subject_str(t: TestContext) -> String {

src/mimeparser.rs

@@ -28,6 +28,7 @@ use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::simplify::simplify;
 use crate::stock_str;
+use crate::sync::SyncItems;
 
 /// A parsed MIME message.
 ///
@@ -56,11 +57,14 @@ pub struct MimeMessage {
     /// this set is empty.
     pub signatures: HashSet<Fingerprint>,
 
-    pub gossipped_addr: HashSet<String>,
+    /// The set of mail recipient addresses for which gossip headers were applied, regardless of
+    /// whether they modified any peerstates.
+    pub gossiped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
     pub location_kml: Option<location::Kml>,
     pub message_kml: Option<location::Kml>,
+    pub(crate) sync_items: Option<SyncItems>,
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
     pub(crate) mdn_reports: Vec<Report>,
@@ -124,6 +128,10 @@ pub enum SystemMessage {
     // Chat protection state changed
     ChatProtectionEnabled = 11,
     ChatProtectionDisabled = 12,
+
+    /// Self-sent-message that contains only json used for multi-device-sync;
+    /// if possible, we attach that to other messages as for locations.
+    MultiDeviceSync = 20,
 }
 
 impl Default for SystemMessage {
@@ -136,6 +144,18 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
 
 impl MimeMessage {
     pub async fn from_bytes(context: &Context, body: &[u8]) -> Result<Self> {
+        MimeMessage::from_bytes_with_partial(context, body, None).await
+    }
+
+    /// Parse a mime message.
+    ///
+    /// If `partial` is set, it contains the full message size in bytes
+    /// and `body` contains the header only.
+    pub async fn from_bytes_with_partial(
+        context: &Context,
+        body: &[u8],
+        partial: Option<u32>,
+    ) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
 
         let message_time = mail
@@ -180,7 +200,7 @@ impl MimeMessage {
 
         // Memory location for a possible decrypted message.
         let mut mail_raw = Vec::new();
-        let mut gossipped_addr = Default::default();
+        let mut gossiped_addr = Default::default();
 
         let (mail, signatures, warn_empty_signature) =
             match e2ee::try_decrypt(context, &mail, message_time).await {
@@ -203,7 +223,7 @@ impl MimeMessage {
                         if !signatures.is_empty() {
                             let gossip_headers =
                                 decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
-                            gossipped_addr = update_gossip_peerstates(
+                            gossiped_addr = update_gossip_peerstates(
                                 context,
                                 message_time,
                                 &mail,
@@ -261,12 +281,13 @@ impl MimeMessage {
 
             // only non-empty if it was a valid autocrypt message
             signatures,
-            gossipped_addr,
+            gossiped_addr,
             is_forwarded: false,
             mdn_reports: Vec::new(),
             is_system_message: SystemMessage::Unknown,
             location_kml: None,
             message_kml: None,
+            sync_items: None,
             user_avatar: None,
             group_avatar: None,
             failure_report: None,
@@ -274,7 +295,18 @@ impl MimeMessage {
             is_mime_modified: false,
             decoded_data: Vec::new(),
         };
-        parser.parse_mime_recursive(context, &mail, false).await?;
+
+        match partial {
+            Some(org_bytes) => {
+                parser
+                    .create_stub_from_partial_download(context, org_bytes)
+                    .await?;
+            }
+            None => {
+                parser.parse_mime_recursive(context, &mail, false).await?;
+            }
+        };
+
         parser.maybe_remove_bad_parts();
         parser.maybe_remove_inline_mailinglist_footer();
         parser.heuristically_parse_ndn(context).await;
@@ -790,6 +822,12 @@ impl MimeMessage {
                                 }
                             }
                         }
+                        Some("multi-device-sync") => {
+                            if let Some(second) = mail.subparts.get(1) {
+                                self.add_single_part_if_known(context, second, is_related)
+                                    .await?;
+                            }
+                        }
                         Some(_) => {
                             if let Some(first) = mail.subparts.get(0) {
                                 any_part_added = self
@@ -976,7 +1014,20 @@ impl MimeMessage {
                 }
                 return;
             }
+        } else if filename == "multi-device-sync.json" {
+            let serialized = String::from_utf8_lossy(decoded_data)
+                .parse()
+                .unwrap_or_default();
+            self.sync_items = context
+                .parse_sync_items(serialized)
+                .await
+                .map_err(|err| {
+                    warn!(context, "failed to parse sync data: {}", err);
+                })
+                .ok();
+            return;
         }
+
         /* we have a regular file attachment,
         write decoded data to new blob object */
 
@@ -1141,11 +1192,11 @@ impl MimeMessage {
         report: &mailparse::ParsedMail<'_>,
     ) -> Result<Option<FailureReport>> {
         // parse as mailheaders
-        if let Some(original_msg) = report
-            .subparts
-            .iter()
-            .find(|p| p.ctype.mimetype.contains("rfc822") || p.ctype.mimetype == "message/global")
-        {
+        if let Some(original_msg) = report.subparts.iter().find(|p| {
+            p.ctype.mimetype.contains("rfc822")
+                || p.ctype.mimetype == "message/global"
+                || p.ctype.mimetype == "message/global-headers"
+        }) {
             let report_body = original_msg.get_body_raw()?;
             let (report_fields, _) = mailparse::parse_headers(&report_body)?;
 
@@ -1331,6 +1382,9 @@ impl MimeMessage {
     }
 }
 
+/// Parses `Autocrypt-Gossip` headers from the email and applies them to peerstates.
+///
+/// Returns the set of mail recipient addresses for which valid gossip headers were found.
 async fn update_gossip_peerstates(
     context: &Context,
     message_time: i64,
@@ -1338,42 +1392,46 @@ async fn update_gossip_peerstates(
     gossip_headers: Vec<String>,
 ) -> Result<HashSet<String>> {
     // XXX split the parsing from the modification part
-    let mut gossipped_addr: HashSet<String> = Default::default();
+    let mut gossiped_addr: HashSet<String> = Default::default();
 
     for value in &gossip_headers {
-        let gossip_header = value.parse::<Aheader>();
-
-        if let Ok(ref header) = gossip_header {
-            if get_recipients(&mail.headers)
-                .iter()
-                .any(|info| info.addr == header.addr.to_lowercase())
-            {
-                let mut peerstate = Peerstate::from_addr(context, &header.addr).await?;
-                if let Some(ref mut peerstate) = peerstate {
-                    peerstate.apply_gossip(header, message_time);
-                    peerstate.save_to_db(&context.sql, false).await?;
-                } else {
-                    let p = Peerstate::from_gossip(header, message_time);
-                    p.save_to_db(&context.sql, true).await?;
-                    peerstate = Some(p);
-                }
-                if let Some(peerstate) = peerstate {
-                    peerstate
-                        .handle_fingerprint_change(context, message_time)
-                        .await?;
-                }
-
-                gossipped_addr.insert(header.addr.clone());
-            } else {
-                warn!(
-                    context,
-                    "Ignoring gossipped \"{}\" as the address is not in To/Cc list.", &header.addr,
-                );
+        let header = match value.parse::<Aheader>() {
+            Ok(header) => header,
+            Err(err) => {
+                warn!(context, "Failed parsing Autocrypt-Gossip header: {}", err);
+                continue;
             }
+        };
+
+        if !get_recipients(&mail.headers)
+            .iter()
+            .any(|info| info.addr == header.addr.to_lowercase())
+        {
+            warn!(
+                context,
+                "Ignoring gossiped \"{}\" as the address is not in To/Cc list.", &header.addr,
+            );
+            continue;
         }
+
+        let peerstate;
+        if let Some(mut p) = Peerstate::from_addr(context, &header.addr).await? {
+            p.apply_gossip(&header, message_time);
+            p.save_to_db(&context.sql, false).await?;
+            peerstate = p;
+        } else {
+            let p = Peerstate::from_gossip(&header, message_time);
+            p.save_to_db(&context.sql, true).await?;
+            peerstate = p;
+        };
+        peerstate
+            .handle_fingerprint_change(context, message_time)
+            .await?;
+
+        gossiped_addr.insert(header.addr.clone());
     }
 
-    Ok(gossipped_addr)
+    Ok(gossiped_addr)
 }
 
 #[derive(Debug)]
@@ -1443,9 +1501,9 @@ pub struct Part {
     pub msg_raw: Option<String>,
     pub bytes: usize,
     pub param: Params,
-    org_filename: Option<String>,
+    pub(crate) org_filename: Option<String>,
     pub error: Option<String>,
-    dehtml_failed: bool,
+    pub(crate) dehtml_failed: bool,
 
     /// the part is a child or a descendant of multipart/related.
     /// typically, these are images that are referenced from text/html part
@@ -1453,7 +1511,7 @@ pub struct Part {
     ///
     /// note that multipart/related may contain further multipart nestings
     /// and all of them needs to be marked with `is_related`.
-    is_related: bool,
+    pub(crate) is_related: bool,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -2997,4 +3055,74 @@ Message.
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_ignore_read_receipt_to_self() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        // Alice receives BCC-self copy of a message sent to Bob.
+        dc_receive_imf(
+            &alice,
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                 From: alice@example.com\n\
+                 To: bob@example.net\n\
+                 Subject: foo\n\
+                 Message-ID: first@example.com\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Disposition-Notification-To: alice@example.com\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n"
+                .as_bytes(),
+            "INBOX",
+            1,
+            false,
+        )
+        .await?;
+        let msg = alice.get_last_msg().await;
+        assert_eq!(msg.state, MessageState::OutDelivered);
+
+        // Due to a bug in the old version running on the other device, Alice receives a read
+        // receipt from self.
+        dc_receive_imf(
+            &alice,
+                "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                 From: alice@example.com\n\
+                 To: alice@example.com\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: second@example.com\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Original-Recipient: rfc822;bob@example.com\n\
+                 Final-Recipient: rfc822;bob@example.com\n\
+                 Original-Message-ID: <first@example.com>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n\
+                 \n\
+                 --SNIPP--"
+            .as_bytes(),
+            "INBOX",
+            2,
+            false,
+        )
+        .await?;
+
+        // Check that the state has not changed to `MessageState::OutMdnRcvd`.
+        let msg = Message::load_from_db(&alice, msg.id).await?;
+        assert_eq!(msg.state, MessageState::OutDelivered);
+
+        Ok(())
+    }
 }

src/param.rs

@@ -139,6 +139,27 @@ pub enum Param {
 
     /// For MDN-sending job
     MsgId = b'I',
+
+    /// For Contacts: timestamp of status (aka signature or footer) update.
+    StatusTimestamp = b'j',
+
+    /// For Contacts and Chats: timestamp of avatar update.
+    AvatarTimestamp = b'J',
+
+    /// For Chats: timestamp of status/signature/footer update.
+    EphemeralSettingsTimestamp = b'B',
+
+    /// For Chats: timestamp of subject update.
+    SubjectTimestamp = b'C',
+
+    /// For Chats: timestamp of group name update.
+    GroupNameTimestamp = b'g',
+
+    /// For Chats: timestamp of group name update.
+    MemberListTimestamp = b'k',
+
+    /// For Chats: timestamp of protection settings update.
+    ProtectionSettingsTimestamp = b'L',
 }
 
 /// An object for handling key=value parameter lists.
@@ -245,6 +266,11 @@ impl Params {
         self.get(key).and_then(|s| s.parse().ok())
     }
 
+    /// Get the given parameter and parse as `i64`.
+    pub fn get_i64(&self, key: Param) -> Option<i64> {
+        self.get(key).and_then(|s| s.parse().ok())
+    }
+
     /// Get the given parameter and parse as `bool`.
     pub fn get_bool(&self, key: Param) -> Option<bool> {
         self.get_int(key).map(|v| v != 0)
@@ -346,6 +372,12 @@ impl Params {
         self
     }
 
+    /// Set the given paramter to the passed in `i64`.
+    pub fn set_i64(&mut self, key: Param, value: i64) -> &mut Self {
+        self.set(key, value.to_string());
+        self
+    }
+
     /// Set the given parameter to the passed in `f64` .
     pub fn set_float(&mut self, key: Param, value: f64) -> &mut Self {
         self.set(key, format!("{}", value));

src/peerstate.rs

@@ -277,8 +277,8 @@ impl Peerstate {
 
                 let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
 
-                chat::add_info_msg(context, chat_id, msg, timestamp).await;
-                emit_event!(context, EventType::ChatModified(chat_id));
+                chat::add_info_msg(context, chat_id, msg, timestamp).await?;
+                context.emit_event(EventType::ChatModified(chat_id));
             } else {
                 bail!("contact with peerstate.addr {:?} not found", &self.addr);
             }
@@ -496,6 +496,30 @@ impl Peerstate {
     }
 }
 
+/// Removes duplicate peerstates from `acpeerstates` database table.
+///
+/// Normally there should be no more than one peerstate per address.
+/// However, the database does not enforce this condition.
+///
+/// Previously there were bugs that caused creation of additional
+/// peerstates when existing peerstate could not be read due to a
+/// temporary database error or a failure to parse stored data.  This
+/// procedure fixes the problem by removing duplicate records.
+pub(crate) async fn deduplicate_peerstates(sql: &Sql) -> Result<()> {
+    sql.execute(
+        "DELETE FROM acpeerstates
+         WHERE id NOT IN (
+         SELECT MIN(id)
+         FROM acpeerstates
+         GROUP BY addr
+         )",
+        paramsv![],
+    )
+    .await?;
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/provider.rs

@@ -4,7 +4,7 @@ mod data;
 
 use crate::config::Config;
 use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED};
-use async_std_resolver::{config, resolver};
+use async_std_resolver::resolver_from_system_conf;
 use chrono::{NaiveDateTime, NaiveTime};
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
@@ -118,12 +118,7 @@ pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
 ///
 /// For security reasons, only Gmail can be configured this way.
 pub async fn get_provider_by_mx(domain: &str) -> Option<&'static Provider> {
-    if let Ok(resolver) = resolver(
-        config::ResolverConfig::default(),
-        config::ResolverOpts::default(),
-    )
-    .await
-    {
+    if let Ok(resolver) = resolver_from_system_conf().await {
         let mut fqdn: String = domain.to_string();
         if !fqdn.ends_with('.') {
             fqdn.push('.');

src/provider/data.rs

@@ -131,6 +131,35 @@ static P_AUTISTICI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
+// blindzeln.org.md: delta.blinzeln.de, delta.blindzeln.org
+static P_BLINDZELN_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "blindzeln.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/blindzeln-org",
+    server: vec![
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "webbox222.server-home.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "webbox222.server-home.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    config_defaults: None,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
+    oauth2_authorizer: None,
+});
+
 // bluewin.ch.md: bluewin.ch
 static P_BLUEWIN_CH: Lazy<Provider> = Lazy::new(|| Provider {
     id: "bluewin.ch",
@@ -1363,7 +1392,7 @@ static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// zoho.md: zohomail.eu, zoho.com
+// zoho.md: zohomail.eu, zohomail.com, zoho.com
 static P_ZOHO: Lazy<Provider> = Lazy::new(|| Provider {
     id: "zoho",
     status: Status::Preparation,
@@ -1399,6 +1428,8 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("aol.com", &*P_AOL),
         ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
+        ("delta.blinzeln.de", &*P_BLINDZELN_ORG),
+        ("delta.blindzeln.org", &*P_BLINDZELN_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
         ("buzon.uy", &*P_BUZON_UY),
         ("chello.at", &*P_CHELLO_AT),
@@ -1580,6 +1611,7 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("yggmail", &*P_YGGMAIL),
         ("ziggo.nl", &*P_ZIGGO_NL),
         ("zohomail.eu", &*P_ZOHO),
+        ("zohomail.com", &*P_ZOHO),
         ("zoho.com", &*P_ZOHO),
     ]
     .iter()
@@ -1594,6 +1626,7 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("aol", &*P_AOL),
         ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
+        ("blindzeln.org", &*P_BLINDZELN_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
         ("buzon.uy", &*P_BUZON_UY),
         ("chello.at", &*P_CHELLO_AT),
@@ -1653,4 +1686,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 8, 17));
+    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 9, 29));

src/quota.rs

@@ -4,21 +4,34 @@ use anyhow::{anyhow, Result};
 use async_imap::types::{Quota, QuotaResource};
 use indexmap::IndexMap;
 
+use crate::chat::add_device_msg_with_importance;
+use crate::config::Config;
+use crate::constants::Viewtype;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::imap::scan_folders::get_watched_folders;
 use crate::imap::Imap;
 use crate::job::{Action, Status};
+use crate::message::Message;
 use crate::param::Params;
-use crate::{job, EventType};
+use crate::{job, stock_str, EventType};
 
 /// warn about a nearly full mailbox after this usage percentage is reached.
 /// quota icon is "yellow".
 pub const QUOTA_WARN_THRESHOLD_PERCENTAGE: u64 = 80;
 
-// warning is already issued at QUOTA_WARN_THRESHOLD_PERCENTAGE,
-// this threshold only makes the quota icon "red".
-pub const QUOTA_ERROR_THRESHOLD_PERCENTAGE: u64 = 99;
+// warning again after this usage percentage is reached,
+// quota icon is "red".
+pub const QUOTA_ERROR_THRESHOLD_PERCENTAGE: u64 = 95;
+
+/// if quota is below this value (again),
+/// QuotaExceeding is cleared.
+/// This value should be a bit below QUOTA_WARN_THRESHOLD_PERCENTAGE to
+/// avoid jittering and lots of warnings when quota is exactly at the warning threshold.
+///
+/// We do not repeat warnings on a daily base or so as some provider
+/// providers report bad values and we would then spam the user.
+pub const QUOTA_ALLCLEAR_PERCENTAGE: u64 = 75;
 
 // if recent quota is older,
 // it is re-fetched on dc_get_connectivity_html()
@@ -63,40 +76,136 @@ async fn get_unique_quota_roots_and_usage(
     Ok(unique_quota_roots)
 }
 
+fn get_highest_usage<'t>(
+    unique_quota_roots: &'t IndexMap<String, Vec<QuotaResource>>,
+) -> Result<(u64, &'t String, &QuotaResource)> {
+    let mut highest: Option<(u64, &'t String, &QuotaResource)> = None;
+    for (name, resources) in unique_quota_roots {
+        for r in resources {
+            let usage_percent = r.get_usage_percentage();
+            match highest {
+                None => {
+                    highest = Some((usage_percent, name, r));
+                }
+                Some((up, ..)) => {
+                    if up <= usage_percent {
+                        highest = Some((usage_percent, name, r));
+                    }
+                }
+            };
+        }
+    }
+
+    highest.ok_or_else(|| anyhow!("no quota_resource found, this is unexpected"))
+}
+
+/// Checks if a quota warning is needed.
+pub fn needs_quota_warning(curr_percentage: u64, warned_at_percentage: u64) -> bool {
+    (curr_percentage >= QUOTA_WARN_THRESHOLD_PERCENTAGE
+        && warned_at_percentage < QUOTA_WARN_THRESHOLD_PERCENTAGE)
+        || (curr_percentage >= QUOTA_ERROR_THRESHOLD_PERCENTAGE
+            && warned_at_percentage < QUOTA_ERROR_THRESHOLD_PERCENTAGE)
+}
+
 impl Context {
     // Adds a job to update `quota.recent`
-    pub(crate) async fn schedule_quota_update(&self) {
-        job::kill_action(self, Action::UpdateRecentQuota).await;
-        job::add(
-            self,
-            job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0),
-        )
-        .await;
+    pub(crate) async fn schedule_quota_update(&self) -> Result<()> {
+        if !job::action_exists(self, Action::UpdateRecentQuota).await? {
+            job::add(
+                self,
+                job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0),
+            )
+            .await?;
+        }
+        Ok(())
     }
 
     /// Updates `quota.recent`, sets `quota.modified` to the current time
     /// and emits an event to let the UIs update connectivity view.
     ///
+    /// Moreover, once each time quota gets larger than `QUOTA_WARN_THRESHOLD_PERCENTAGE`,
+    /// a device message is added.
+    /// As the message is added only once, the user is not spammed
+    /// in case for some providers the quota is always at ~100%
+    /// and new space is allocated as needed.
+    ///
     /// Called in response to `Action::UpdateRecentQuota`.
-    pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Status {
+    pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<Status> {
         if let Err(err) = imap.prepare(self).await {
             warn!(self, "could not connect: {:?}", err);
-            return Status::RetryNow;
+            return Ok(Status::RetryNow);
         }
 
         let quota = if imap.can_check_quota() {
             let folders = get_watched_folders(self).await;
             get_unique_quota_roots_and_usage(folders, imap).await
         } else {
-            Err(anyhow!("Quota not supported by your provider."))
+            Err(anyhow!(stock_str::not_supported_by_provider(self).await))
         };
 
+        if let Ok(quota) = &quota {
+            match get_highest_usage(quota) {
+                Ok((highest, _, _)) => {
+                    if needs_quota_warning(
+                        highest,
+                        self.get_config_int(Config::QuotaExceeding).await? as u64,
+                    ) {
+                        self.set_config(Config::QuotaExceeding, Some(&highest.to_string()))
+                            .await?;
+                        let mut msg = Message::new(Viewtype::Text);
+                        msg.text = Some(stock_str::quota_exceeding(self, highest).await);
+                        add_device_msg_with_importance(self, None, Some(&mut msg), true).await?;
+                    } else if highest <= QUOTA_ALLCLEAR_PERCENTAGE {
+                        self.set_config(Config::QuotaExceeding, None).await?;
+                    }
+                }
+                Err(err) => warn!(self, "cannot get highest quota usage: {:?}", err),
+            }
+        }
+
         *self.quota.write().await = Some(QuotaInfo {
             recent: quota,
             modified: time(),
         });
 
         self.emit_event(EventType::ConnectivityChanged);
-        Status::Finished(Ok(()))
+        Ok(Status::Finished(Ok(())))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::quota::{
+        QUOTA_ALLCLEAR_PERCENTAGE, QUOTA_ERROR_THRESHOLD_PERCENTAGE,
+        QUOTA_WARN_THRESHOLD_PERCENTAGE,
+    };
+
+    #[async_std::test]
+    async fn test_needs_quota_warning() -> Result<()> {
+        assert!(!needs_quota_warning(0, 0));
+        assert!(!needs_quota_warning(10, 0));
+        assert!(!needs_quota_warning(70, 0));
+        assert!(!needs_quota_warning(75, 0));
+        assert!(!needs_quota_warning(79, 0));
+        assert!(needs_quota_warning(80, 0));
+        assert!(needs_quota_warning(81, 0));
+        assert!(!needs_quota_warning(85, 80));
+        assert!(!needs_quota_warning(85, 81));
+        assert!(needs_quota_warning(95, 82));
+        assert!(!needs_quota_warning(97, 95));
+        assert!(!needs_quota_warning(97, 96));
+        assert!(!needs_quota_warning(1000, 96));
+        Ok(())
+    }
+
+    #[allow(clippy::assertions_on_constants)]
+    #[async_std::test]
+    async fn test_quota_thresholds() -> anyhow::Result<()> {
+        assert!(QUOTA_ALLCLEAR_PERCENTAGE > 50);
+        assert!(QUOTA_ALLCLEAR_PERCENTAGE < QUOTA_WARN_THRESHOLD_PERCENTAGE);
+        assert!(QUOTA_WARN_THRESHOLD_PERCENTAGE < QUOTA_ERROR_THRESHOLD_PERCENTAGE);
+        assert!(QUOTA_ERROR_THRESHOLD_PERCENTAGE < 100);
+        Ok(())
     }
 }

src/scheduler.rs

@@ -82,7 +82,11 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         let mut jobs_loaded = 0;
         let mut info = InterruptInfo::default();
         loop {
-            match job::load_next(&ctx, Thread::Imap, &info).await {
+            match job::load_next(&ctx, Thread::Imap, &info)
+                .await
+                .ok()
+                .flatten()
+            {
                 Some(job) if jobs_loaded <= 20 => {
                     jobs_loaded += 1;
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
@@ -289,7 +293,11 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
 
         let mut interrupt_info = Default::default();
         loop {
-            match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info)
+                .await
+                .ok()
+                .flatten()
+            {
                 Some(job) => {
                     info!(ctx, "executing smtp job");
                     job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;

src/scheduler/connectivity.rs

@@ -8,7 +8,7 @@ use crate::events::EventType;
 use crate::quota::{
     QUOTA_ERROR_THRESHOLD_PERCENTAGE, QUOTA_MAX_AGE_SECONDS, QUOTA_WARN_THRESHOLD_PERCENTAGE,
 };
-use crate::{config::Config, dc_tools, scheduler::Scheduler};
+use crate::{config::Config, dc_tools, scheduler::Scheduler, stock_str};
 use crate::{context::Context, log::LogExt};
 use anyhow::{anyhow, Result};
 use humansize::{file_size_opts, FileSize};
@@ -73,33 +73,33 @@ impl DetailedConnectivity {
         }
     }
 
-    fn to_string_imap(&self, _context: &Context) -> String {
+    async fn to_string_imap(&self, context: &Context) -> String {
         match self {
-            DetailedConnectivity::Error(e) => format!("Error: {}", e),
+            DetailedConnectivity::Error(e) => stock_str::error(context, e).await,
             DetailedConnectivity::Uninitialized => "Not started".to_string(),
-            DetailedConnectivity::Connecting => "Connecting…".to_string(),
-            DetailedConnectivity::Working => "Getting new messages…".to_string(),
+            DetailedConnectivity::Connecting => stock_str::connecting(context).await,
+            DetailedConnectivity::Working => stock_str::updating(context).await,
             DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
-                "Connected".to_string()
+                stock_str::connected(context).await
             }
             DetailedConnectivity::NotConfigured => "Not configured".to_string(),
         }
     }
 
-    fn to_string_smtp(&self, _context: &Context) -> String {
+    async fn to_string_smtp(&self, context: &Context) -> String {
         match self {
-            DetailedConnectivity::Error(e) => format!("Error: {}", e),
+            DetailedConnectivity::Error(e) => stock_str::error(context, e).await,
             DetailedConnectivity::Uninitialized => {
-                "(You did not try to send a message recently)".to_string()
+                "You did not try to send a message recently.".to_string()
             }
-            DetailedConnectivity::Connecting => "Connecting…".to_string(),
-            DetailedConnectivity::Working => "Sending…".to_string(),
+            DetailedConnectivity::Connecting => stock_str::connecting(context).await,
+            DetailedConnectivity::Working => stock_str::sending(context).await,
 
             // We don't know any more than that the last message was sent successfully;
             // since sending the last message, connectivity could have changed, which we don't notice
             // until another message is sent
             DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
-                "Your last message was sent successfully".to_string()
+                stock_str::last_msg_sent_successfully(context).await
             }
             DetailedConnectivity::NotConfigured => "Not configured".to_string(),
         }
@@ -251,7 +251,7 @@ impl Context {
     /// One of:
     /// - DC_CONNECTIVITY_NOT_CONNECTED (1000-1999): Show e.g. the string "Not connected" or a red dot
     /// - DC_CONNECTIVITY_CONNECTING (2000-2999): Show e.g. the string "Connecting…" or a yellow dot
-    /// - DC_CONNECTIVITY_WORKING (3000-3999): Show e.g. the string "Getting new messages" or a spinning wheel
+    /// - DC_CONNECTIVITY_WORKING (3000-3999): Show e.g. the string "Updating…" or a spinning wheel
     /// - DC_CONNECTIVITY_CONNECTED (>=4000): Show e.g. the string "Connected" or a green dot
     ///
     /// We don't use exact values but ranges here so that we can split up
@@ -380,7 +380,7 @@ impl Context {
         };
         drop(lock);
 
-        ret += "<h3>Incoming messages</h3><ul>";
+        ret += &format!("<h3>{}</h3><ul>", stock_str::incoming_messages(self).await);
         for (folder, watch, state) in &folders_states {
             let w = self.get_config(*watch).await.ok_or_log(self);
 
@@ -395,7 +395,7 @@ impl Context {
                     ret += " <b>";
                     ret += &*escaper::encode_minimal(&foldername);
                     ret += ":</b> ";
-                    ret += &*escaper::encode_minimal(&*detailed.to_string_imap(self));
+                    ret += &*escaper::encode_minimal(&*detailed.to_string_imap(self).await);
                     ret += "</li>";
 
                     folder_added = true;
@@ -410,18 +410,21 @@ impl Context {
                     ret += "<li>";
                     ret += &*detailed.to_icon();
                     ret += " ";
-                    ret += &*escaper::encode_minimal(&detailed.to_string_imap(self));
+                    ret += &*escaper::encode_minimal(&detailed.to_string_imap(self).await);
                     ret += "</li>";
                 }
             }
         }
         ret += "</ul>";
 
-        ret += "<h3>Outgoing messages</h3><ul><li>";
+        ret += &format!(
+            "<h3>{}</h3><ul><li>",
+            stock_str::outgoing_messages(self).await
+        );
         let detailed = smtp.get_detailed().await;
         ret += &*detailed.to_icon();
         ret += " ";
-        ret += &*escaper::encode_minimal(&detailed.to_string_smtp(self));
+        ret += &*escaper::encode_minimal(&detailed.to_string_smtp(self).await);
         ret += "</li></ul>";
 
         let domain = dc_tools::EmailAddress::new(
@@ -431,7 +434,10 @@ impl Context {
                 .unwrap_or_default(),
         )?
         .domain;
-        ret += &format!("<h3>Storage on {}</h3><ul>", domain);
+        ret += &format!(
+            "<h3>{}</h3><ul>",
+            stock_str::storage_on_domain(self, domain).await
+        );
         let quota = self.quota.read().await;
         if let Some(quota) = &*quota {
             match &quota.recent {
@@ -454,17 +460,26 @@ impl Context {
                             ret += &match &resource.name {
                                 Atom(resource_name) => {
                                     format!(
-                                        "<b>{}:</b> {} of {} used",
+                                        "<b>{}:</b> {}",
                                         &*escaper::encode_minimal(resource_name),
-                                        resource.usage.to_string(),
-                                        resource.limit.to_string(),
+                                        stock_str::part_of_total_used(
+                                            self,
+                                            resource.usage.to_string(),
+                                            resource.limit.to_string()
+                                        )
+                                        .await,
                                     )
                                 }
                                 Message => {
                                     format!(
-                                        "<b>Messages:</b> {} of {} used",
-                                        resource.usage.to_string(),
-                                        resource.limit.to_string(),
+                                        "<b>{}:</b> {}",
+                                        stock_str::messages(self).await,
+                                        stock_str::part_of_total_used(
+                                            self,
+                                            resource.usage.to_string(),
+                                            resource.limit.to_string()
+                                        )
+                                        .await,
                                     )
                                 }
                                 Storage => {
@@ -480,7 +495,7 @@ impl Context {
                                     let limit = (resource.limit * 1024)
                                         .file_size(file_size_opts::BINARY)
                                         .unwrap_or_default();
-                                    format!("{} of {} used", usage, limit)
+                                    stock_str::part_of_total_used(self, usage, limit).await
                                 }
                             };
 
@@ -504,11 +519,11 @@ impl Context {
             }
 
             if quota.modified + QUOTA_MAX_AGE_SECONDS < time() {
-                self.schedule_quota_update().await;
+                self.schedule_quota_update().await?;
             }
         } else {
-            ret += "<li>One moment...</li>";
-            self.schedule_quota_update().await;
+            ret += &format!("<li>{}</li>", stock_str::one_moment(self).await);
+            self.schedule_quota_update().await?;
         }
         ret += "</ul>";
 

src/securejoin/bobstate.rs

@@ -8,11 +8,11 @@
 //! protocol.  Afterwards it must be stored in a mutex and the [`BobStateHandle`] should be
 //! used to work with the state.
 
-use anyhow::{Error, Result};
+use anyhow::{bail, Error, Result};
 use async_std::sync::MutexGuard;
 
 use crate::chat::{self, ChatId};
-use crate::constants::Viewtype;
+use crate::constants::{Blocked, Viewtype};
 use crate::contact::{Contact, Origin};
 use crate::context::Context;
 use crate::events::EventType;
@@ -67,9 +67,18 @@ impl<'a> BobStateHandle<'a> {
         })
     }
 
-    /// Returns the [`ChatId`] of the 1:1 chat with the inviter (Alice).
-    pub fn chat_id(&self) -> ChatId {
-        self.bobstate.chat_id
+    /// Returns the [`ChatId`] of the group chat to join or the 1:1 chat with Alice.
+    pub async fn chat_id(&self, context: &Context) -> Result<ChatId> {
+        match self.bobstate.invite {
+            QrInvite::Group { ref grpid, .. } => {
+                if let Some((chat_id, _, _)) = chat::get_chat_id_by_grpid(context, &grpid).await? {
+                    Ok(chat_id)
+                } else {
+                    bail!("chat not found")
+                }
+            }
+            QrInvite::Contact { .. } => Ok(self.bobstate.chat_id),
+        }
     }
 
     /// Returns a reference to the [`QrInvite`] of the joiner process.
@@ -185,10 +194,11 @@ impl BobState {
         context: &Context,
         invite: QrInvite,
     ) -> Result<(Self, BobHandshakeStage), JoinError> {
-        let chat_id = ChatId::create_for_contact(context, invite.contact_id())
-            .await
-            .map_err(JoinError::UnknownContact)?;
-        if fingerprint_equals_sender(context, invite.fingerprint(), chat_id).await? {
+        let chat_id =
+            ChatId::create_for_contact_with_blocked(context, invite.contact_id(), Blocked::Yes)
+                .await
+                .map_err(JoinError::UnknownContact)?;
+        if fingerprint_equals_sender(context, invite.fingerprint(), invite.contact_id()).await? {
             // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
             info!(context, "Taking securejoin protocol shortcut");
             let state = Self {
@@ -297,7 +307,9 @@ impl BobState {
             self.next = SecureJoinStep::Terminated;
             return Ok(Some(BobHandshakeStage::Terminated(reason)));
         }
-        if !fingerprint_equals_sender(context, self.invite.fingerprint(), self.chat_id).await? {
+        if !fingerprint_equals_sender(context, self.invite.fingerprint(), self.invite.contact_id())
+            .await?
+        {
             self.next = SecureJoinStep::Terminated;
             return Ok(Some(BobHandshakeStage::Terminated("Fingerprint mismatch")));
         }
@@ -357,8 +369,8 @@ impl BobState {
         }
         mark_peer_as_verified(context, self.invite.fingerprint()).await?;
         Contact::scaleup_origin_by_id(context, self.invite.contact_id(), Origin::SecurejoinJoined)
-            .await;
-        emit_event!(context, EventType::ContactsChanged(None));
+            .await?;
+        context.emit_event(EventType::ContactsChanged(None));
 
         if let QrInvite::Group { .. } = self.invite {
             let member_added = mime_message

src/securejoin/qrinvite.rs

@@ -1,16 +1,15 @@
 //! Supporting code for the QR-code invite.
 //!
-//! QR-codes are decoded into a more general-purpose [`Lot`] struct normally, this struct is
-//! so general it is not even specific to QR-codes.  This makes working with it rather hard,
-//! so here we have a wrapper type that specifically deals with Secure-Join QR-codes so
-//! that the Secure-Join code can have many more guarantees when dealing with this.
+//! QR-codes are decoded into a more general-purpose [`Qr`] struct normally.  This makes working
+//! with it rather hard, so here we have a wrapper type that specifically deals with Secure-Join
+//! QR-codes so that the Secure-Join code can have more guarantees when dealing with this.
 
 use std::convert::TryFrom;
 
-use anyhow::Result;
+use anyhow::{bail, Error, Result};
 
 use crate::key::Fingerprint;
-use crate::lot::{Lot, LotState};
+use crate::qr::Qr;
 
 /// Represents the data from a QR-code scan.
 ///
@@ -66,53 +65,38 @@ impl QrInvite {
     }
 }
 
-impl TryFrom<Lot> for QrInvite {
-    type Error = QrError;
+impl TryFrom<Qr> for QrInvite {
+    type Error = Error;
 
-    fn try_from(lot: Lot) -> Result<Self, Self::Error> {
-        if lot.state != LotState::QrAskVerifyContact && lot.state != LotState::QrAskVerifyGroup {
-            return Err(QrError::UnsupportedProtocol);
-        }
-        if lot.id == 0 {
-            return Err(QrError::MissingContactId);
-        }
-        let fingerprint = lot.fingerprint.ok_or(QrError::MissingFingerprint)?;
-        let invitenumber = lot.invitenumber.ok_or(QrError::MissingInviteNumber)?;
-        let authcode = lot.auth.ok_or(QrError::MissingAuthCode)?;
-        match lot.state {
-            LotState::QrAskVerifyContact => Ok(QrInvite::Contact {
-                contact_id: lot.id,
+    fn try_from(qr: Qr) -> Result<Self> {
+        match qr {
+            Qr::AskVerifyContact {
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => Ok(QrInvite::Contact {
+                contact_id,
                 fingerprint,
                 invitenumber,
                 authcode,
             }),
-            LotState::QrAskVerifyGroup => Ok(QrInvite::Group {
-                contact_id: lot.id,
+            Qr::AskVerifyGroup {
+                grpname,
+                grpid,
+                contact_id,
                 fingerprint,
-                name: lot.text1.ok_or(QrError::MissingGroupName)?,
-                grpid: lot.text2.ok_or(QrError::MissingGroupId)?,
+                invitenumber,
+                authcode,
+            } => Ok(QrInvite::Group {
+                contact_id,
+                fingerprint,
+                name: grpname,
+                grpid,
                 invitenumber,
                 authcode,
             }),
-            _ => Err(QrError::UnsupportedProtocol),
+            _ => bail!("Unsupported QR type {:?}", qr),
         }
     }
 }
-
-#[derive(Debug, thiserror::Error)]
-pub enum QrError {
-    #[error("Unsupported protocol in QR-code")]
-    UnsupportedProtocol,
-    #[error("Missing fingerprint")]
-    MissingFingerprint,
-    #[error("Missing invitenumber")]
-    MissingInviteNumber,
-    #[error("Missing auth code")]
-    MissingAuthCode,
-    #[error("Missing group name")]
-    MissingGroupName,
-    #[error("Missing group id")]
-    MissingGroupId,
-    #[error("Missing contact id")]
-    MissingContactId,
-}

src/smtp.rs

@@ -109,7 +109,8 @@ impl Smtp {
             &lp.socks5_config,
             &lp.addr,
             lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
-            lp.provider.map_or(false, |provider| provider.strict_tls),
+            lp.provider
+                .map_or(lp.socks5_config.is_some(), |provider| provider.strict_tls),
         )
         .await
     }

src/sql.rs

@@ -20,7 +20,7 @@ use crate::dc_tools::{dc_delete_file, time};
 use crate::ephemeral::start_ephemeral_timers;
 use crate::message::Message;
 use crate::param::{Param, Params};
-use crate::peerstate::Peerstate;
+use crate::peerstate::{deduplicate_peerstates, Peerstate};
 use crate::stock_str;
 
 #[macro_export]
@@ -598,6 +598,12 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         );
     }
 
+    if let Err(err) = deduplicate_peerstates(&context.sql).await {
+        warn!(context, "Failed to deduplicate peerstates: {}", err)
+    }
+
+    context.schedule_quota_update().await?;
+
     if let Err(e) = context
         .set_config(Config::LastHousekeeping, Some(&time().to_string()))
         .await

src/sql/migrations.rs

@@ -477,6 +477,26 @@ paramsv![]
         sql.execute_migration("UPDATE chats SET archived=1 WHERE blocked=2;", 78)
             .await?;
     }
+    if dbversion < 79 {
+        info!(context, "[migration] v79");
+        sql.execute_migration(
+            r#"
+        ALTER TABLE msgs ADD COLUMN download_state INTEGER DEFAULT 0;
+        "#,
+            79,
+        )
+        .await?;
+    }
+    if dbversion < 80 {
+        info!(context, "[migration] v80");
+        sql.execute_migration(
+            r#"CREATE TABLE multi_device_sync (
+id INTEGER PRIMARY KEY AUTOINCREMENT,
+item TEXT DEFAULT '');"#,
+            80,
+        )
+        .await?;
+    }
 
     Ok((
         recalc_fingerprints,

src/stock_str.rs

@@ -13,8 +13,10 @@ use crate::config::Config;
 use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::contact::{Contact, Origin};
 use crate::context::Context;
+use crate::dc_tools::dc_timestamp_to_str;
 use crate::message::Message;
 use crate::param::Param;
+use humansize::{file_size_opts, FileSize};
 
 /// Stock strings
 ///
@@ -258,6 +260,79 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Forwarded"))]
     Forwarded = 97,
+
+    #[strum(props(
+        fallback = "⚠️ Your provider's storage is about to exceed, already %1$s%% are used.\n\n\
+                    You may not be able to receive message when the storage is 100%% used.\n\n\
+                    👉 Please check if you can delete old data in the provider's webinterface \
+                    and consider to enable \"Settings / Delete Old Messages\". \
+                    You can check your current storage usage anytime at \"Settings / Connectivity\"."
+    ))]
+    QuotaExceedingMsgBody = 98,
+
+    #[strum(props(fallback = "%1$s message"))]
+    PartialDownloadMsgBody = 99,
+
+    #[strum(props(fallback = "Download maximum available until %1$s"))]
+    DownloadAvailability = 100,
+
+    #[strum(props(fallback = "Multi Device Synchronization"))]
+    SyncMsgSubject = 101,
+
+    #[strum(props(
+        fallback = "This message is used to synchronize data between your devices.\n\n\
+                    👉 If you see this message in Delta Chat, please update your Delta Chat apps on all devices."
+    ))]
+    SyncMsgBody = 102,
+
+    #[strum(props(fallback = "Incoming Messages"))]
+    IncomingMessages = 103,
+
+    #[strum(props(fallback = "Outgoing Messages"))]
+    OutgoingMessages = 104,
+
+    #[strum(props(fallback = "Storage on %1$s"))]
+    StorageOnDomain = 105,
+
+    #[strum(props(fallback = "One moment…"))]
+    OneMoment = 106,
+
+    #[strum(props(fallback = "Connected"))]
+    Connected = 107,
+
+    #[strum(props(fallback = "Connecting…"))]
+    Connecting = 108,
+
+    #[strum(props(fallback = "Updating…"))]
+    Updating = 109,
+
+    #[strum(props(fallback = "Sending…"))]
+    Sending = 110,
+
+    #[strum(props(fallback = "Your last message was sent successfully."))]
+    LastMsgSentSuccessfully = 111,
+
+    #[strum(props(fallback = "Error: %1$s"))]
+    Error = 112,
+
+    #[strum(props(fallback = "Not supported by your provider."))]
+    NotSupportedByProvider = 113,
+
+    #[strum(props(fallback = "Messages"))]
+    Messages = 114,
+
+    #[strum(props(fallback = "Broadcast List"))]
+    BroadcastList = 115,
+
+    #[strum(props(fallback = "%1$s of %2$s used"))]
+    PartOfTotallUsed = 116,
+
+    #[strum(props(fallback = "%1$s invited you to join this group.\n\n\
+                             Waiting for the device of %2$s to reply…"))]
+    SecureJoinStarted = 117,
+
+    #[strum(props(fallback = "%1$s replied, waiting for being added to the group…"))]
+    SecureJoinReplies = 118,
 }
 
 impl StockMessage {
@@ -523,6 +598,32 @@ pub(crate) async fn e2e_preferred(context: &Context) -> String {
     translated(context, StockMessage::E2ePreferred).await
 }
 
+/// Stock string: `%1$s invited you to join this group. Waiting for the device of %2$s to reply…`.
+pub(crate) async fn secure_join_started(context: &Context, inviter_contact_id: u32) -> String {
+    if let Ok(contact) = Contact::get_by_id(context, inviter_contact_id).await {
+        translated(context, StockMessage::SecureJoinStarted)
+            .await
+            .replace1(contact.get_name_n_addr())
+            .replace2(contact.get_display_name())
+    } else {
+        format!(
+            "secure_join_started: unknown contact {}",
+            inviter_contact_id
+        )
+    }
+}
+
+/// Stock string: `%1$s replied, waiting for being added to the group…`.
+pub(crate) async fn secure_join_replies(context: &Context, contact_id: u32) -> String {
+    if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
+        translated(context, StockMessage::SecureJoinReplies)
+            .await
+            .replace1(contact.get_display_name())
+    } else {
+        format!("secure_join_replies: unknown contact {}", contact_id)
+    }
+}
+
 /// Stock string: `%1$s verified.`.
 pub(crate) async fn contact_verified(context: &Context, contact_addr: impl AsRef<str>) -> String {
     translated(context, StockMessage::ContactVerified)
@@ -565,6 +666,16 @@ pub(crate) async fn ac_setup_msg_body(context: &Context) -> String {
     translated(context, StockMessage::AcSetupMsgBody).await
 }
 
+/// Stock string: `Multi Device Synchronization`.
+pub(crate) async fn sync_msg_subject(context: &Context) -> String {
+    translated(context, StockMessage::SyncMsgSubject).await
+}
+
+/// Stock string: `This message is used to synchronize data betweeen your devices.`.
+pub(crate) async fn sync_msg_body(context: &Context) -> String {
+    translated(context, StockMessage::SyncMsgBody).await
+}
+
 /// Stock string: `Cannot login as \"%1$s\". Please check...`.
 pub(crate) async fn cannot_login(context: &Context, user: impl AsRef<str>) -> String {
     translated(context, StockMessage::CannotLogin)
@@ -840,6 +951,116 @@ pub(crate) async fn forwarded(context: &Context) -> String {
     translated(context, StockMessage::Forwarded).await
 }
 
+/// Stock string: `⚠️ Your provider's storage is about to exceed...`.
+pub(crate) async fn quota_exceeding(context: &Context, highest_usage: u64) -> String {
+    translated(context, StockMessage::QuotaExceedingMsgBody)
+        .await
+        .replace1(format!("{}", highest_usage))
+        .replace("%%", "%")
+}
+
+/// Stock string: `%1$s message` with placeholder replaced by human-readable size.
+pub(crate) async fn partial_download_msg_body(context: &Context, org_bytes: u32) -> String {
+    let size = org_bytes
+        .file_size(file_size_opts::BINARY)
+        .unwrap_or_default();
+    translated(context, StockMessage::PartialDownloadMsgBody)
+        .await
+        .replace1(size)
+}
+
+/// Stock string: `Download maximum available until %1$s`.
+pub(crate) async fn download_availability(context: &Context, timestamp: i64) -> String {
+    translated(context, StockMessage::DownloadAvailability)
+        .await
+        .replace1(dc_timestamp_to_str(timestamp))
+}
+
+/// Stock string: `Incoming Messages`.
+pub(crate) async fn incoming_messages(context: &Context) -> String {
+    translated(context, StockMessage::IncomingMessages).await
+}
+
+/// Stock string: `Outgoing Messages`.
+pub(crate) async fn outgoing_messages(context: &Context) -> String {
+    translated(context, StockMessage::OutgoingMessages).await
+}
+
+/// Stock string: `Storage on %1$s`.
+/// `%1$s` will be replaced by the domain of the configured email-address.
+pub(crate) async fn storage_on_domain(context: &Context, domain: impl AsRef<str>) -> String {
+    translated(context, StockMessage::StorageOnDomain)
+        .await
+        .replace1(domain)
+}
+
+/// Stock string: `One moment…`.
+pub(crate) async fn one_moment(context: &Context) -> String {
+    translated(context, StockMessage::OneMoment).await
+}
+
+/// Stock string: `Connected`.
+pub(crate) async fn connected(context: &Context) -> String {
+    translated(context, StockMessage::Connected).await
+}
+
+/// Stock string: `Connecting…`.
+pub(crate) async fn connecting(context: &Context) -> String {
+    translated(context, StockMessage::Connecting).await
+}
+
+/// Stock string: `Updating…`.
+pub(crate) async fn updating(context: &Context) -> String {
+    translated(context, StockMessage::Updating).await
+}
+
+/// Stock string: `Sending…`.
+pub(crate) async fn sending(context: &Context) -> String {
+    translated(context, StockMessage::Sending).await
+}
+
+/// Stock string: `Your last message was sent successfully.`.
+pub(crate) async fn last_msg_sent_successfully(context: &Context) -> String {
+    translated(context, StockMessage::LastMsgSentSuccessfully).await
+}
+
+/// Stock string: `Error: %1$s…`.
+/// `%1$s` will be replaced by a possibly more detailed, typically english, error description.
+pub(crate) async fn error(context: &Context, error: impl AsRef<str>) -> String {
+    translated(context, StockMessage::Error)
+        .await
+        .replace1(error)
+}
+
+/// Stock string: `Not supported by your provider.`.
+pub(crate) async fn not_supported_by_provider(context: &Context) -> String {
+    translated(context, StockMessage::NotSupportedByProvider).await
+}
+
+/// Stock string: `Messages`.
+/// Used as a subtitle in quota context; can be plural always.
+pub(crate) async fn messages(context: &Context) -> String {
+    translated(context, StockMessage::Messages).await
+}
+
+/// Stock string: `%1$s of %2$s used`.
+pub(crate) async fn part_of_total_used(
+    context: &Context,
+    part: impl AsRef<str>,
+    total: impl AsRef<str>,
+) -> String {
+    translated(context, StockMessage::PartOfTotallUsed)
+        .await
+        .replace1(part)
+        .replace2(total)
+}
+
+/// Stock string: `Broadcast List`.
+/// Used as the default name for broadcast lists; a number may be added.
+pub(crate) async fn broadcast_list(context: &Context) -> String {
+    translated(context, StockMessage::BroadcastList).await
+}
+
 impl Context {
     /// Set the stock string for the [StockMessage].
     ///
@@ -1023,6 +1244,24 @@ mod tests {
         );
     }
 
+    #[async_std::test]
+    async fn test_quota_exceeding_stock_str() -> anyhow::Result<()> {
+        let t = TestContext::new().await;
+        let str = quota_exceeding(&t, 81).await;
+        assert!(str.contains("81% "));
+        assert!(str.contains("100% "));
+        assert!(!str.contains("%%"));
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_partial_download_msg_body() -> anyhow::Result<()> {
+        let t = TestContext::new().await;
+        let str = partial_download_msg_body(&t, 1024 * 1024).await;
+        assert_eq!(str, "1 MiB message");
+        Ok(())
+    }
+
     #[async_std::test]
     async fn test_update_device_chats() {
         let t = TestContext::new().await;

src/summary.rs

@@ -0,0 +1,317 @@
+//! # Message summary for chatlist.
+
+use crate::chat::Chat;
+use crate::constants::{Chattype, Viewtype, DC_CONTACT_ID_SELF};
+use crate::contact::Contact;
+use crate::context::Context;
+use crate::dc_tools::dc_truncate;
+use crate::message::{Message, MessageState};
+use crate::mimeparser::SystemMessage;
+use crate::param::Param;
+use crate::stock_str;
+use itertools::Itertools;
+use std::borrow::Cow;
+use std::fmt;
+
+/// Prefix displayed before message and separated by ":" in the chatlist.
+#[derive(Debug)]
+pub enum SummaryPrefix {
+    /// Username.
+    Username(String),
+
+    /// Stock string saying "Draft".
+    Draft(String),
+
+    /// Stock string saying "Me".
+    Me(String),
+}
+
+impl fmt::Display for SummaryPrefix {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            SummaryPrefix::Username(username) => write!(f, "{}", username),
+            SummaryPrefix::Draft(text) => write!(f, "{}", text),
+            SummaryPrefix::Me(text) => write!(f, "{}", text),
+        }
+    }
+}
+
+/// Message summary.
+#[derive(Debug, Default)]
+pub struct Summary {
+    /// Part displayed before ":", such as an username or a string "Draft".
+    pub prefix: Option<SummaryPrefix>,
+
+    /// Summary text, always present.
+    pub text: String,
+
+    /// Message timestamp.
+    pub timestamp: i64,
+
+    /// Message state.
+    pub state: MessageState,
+}
+
+impl Summary {
+    pub async fn new(
+        context: &Context,
+        msg: &Message,
+        chat: &Chat,
+        contact: Option<&Contact>,
+    ) -> Self {
+        let prefix = if msg.state == MessageState::OutDraft {
+            Some(SummaryPrefix::Draft(stock_str::draft(context).await))
+        } else if msg.from_id == DC_CONTACT_ID_SELF {
+            if msg.is_info() || chat.is_self_talk() {
+                None
+            } else {
+                Some(SummaryPrefix::Me(stock_str::self_msg(context).await))
+            }
+        } else {
+            match chat.typ {
+                Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
+                    if msg.is_info() || contact.is_none() {
+                        None
+                    } else {
+                        msg.get_override_sender_name()
+                            .or_else(|| contact.map(|contact| msg.get_sender_name(contact)))
+                            .map(SummaryPrefix::Username)
+                    }
+                }
+                Chattype::Single | Chattype::Undefined => None,
+            }
+        };
+
+        let mut text = msg.get_summary_text(context).await;
+
+        if text.is_empty() && msg.quoted_text().is_some() {
+            text = stock_str::reply_noun(context).await
+        }
+
+        Self {
+            prefix,
+            text,
+            timestamp: msg.get_timestamp(),
+            state: msg.state,
+        }
+    }
+
+    /// Returns the [`Summary::text`] attribute truncated to an approximate length.
+    pub fn truncated_text(&self, approx_chars: usize) -> Cow<str> {
+        dc_truncate(&self.text, approx_chars)
+    }
+}
+
+impl Message {
+    /// Returns a summary text.
+    async fn get_summary_text(&self, context: &Context) -> String {
+        let mut append_text = true;
+        let prefix = match self.viewtype {
+            Viewtype::Image => stock_str::image(context).await,
+            Viewtype::Gif => stock_str::gif(context).await,
+            Viewtype::Sticker => stock_str::sticker(context).await,
+            Viewtype::Video => stock_str::video(context).await,
+            Viewtype::Voice => stock_str::voice_message(context).await,
+            Viewtype::Audio | Viewtype::File => {
+                if self.param.get_cmd() == SystemMessage::AutocryptSetupMessage {
+                    append_text = false;
+                    stock_str::ac_setup_msg_subject(context).await
+                } else {
+                    let file_name: String = self
+                        .param
+                        .get_path(Param::File, context)
+                        .unwrap_or(None)
+                        .and_then(|path| {
+                            path.file_name()
+                                .map(|fname| fname.to_string_lossy().into_owned())
+                        })
+                        .unwrap_or_else(|| String::from("ErrFileName"));
+                    let label = if self.viewtype == Viewtype::Audio {
+                        stock_str::audio(context).await
+                    } else {
+                        stock_str::file(context).await
+                    };
+                    format!("{} – {}", label, file_name)
+                }
+            }
+            Viewtype::VideochatInvitation => {
+                append_text = false;
+                stock_str::videochat_invitation(context).await
+            }
+            _ => {
+                if self.param.get_cmd() != SystemMessage::LocationOnly {
+                    "".to_string()
+                } else {
+                    append_text = false;
+                    stock_str::location(context).await
+                }
+            }
+        };
+
+        if !append_text {
+            return prefix;
+        }
+
+        let summary_content = if let Some(text) = &self.text {
+            if text.is_empty() {
+                prefix
+            } else if prefix.is_empty() {
+                text.to_string()
+            } else {
+                format!("{} – {}", prefix, text)
+            }
+        } else {
+            prefix
+        };
+
+        let summary = if self.is_forwarded() {
+            format!(
+                "{}: {}",
+                stock_str::forwarded(context).await,
+                summary_content
+            )
+        } else {
+            summary_content
+        };
+
+        summary.split_whitespace().join(" ")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils as test;
+
+    #[async_std::test]
+    async fn test_get_summary_text() {
+        let d = test::TestContext::new().await;
+        let ctx = &d.ctx;
+
+        let some_text = Some(" bla \t\n\tbla\n\t".to_string());
+        let empty_text = Some("".to_string());
+        let no_text: Option<String> = None;
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(some_text.clone());
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "bla bla" // for simple text, the type is not added to the summary
+        );
+
+        let mut msg = Message::new(Viewtype::Image);
+        msg.set_text(no_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Image" // file names are not added for images
+        );
+
+        let mut msg = Message::new(Viewtype::Video);
+        msg.set_text(no_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Video" // file names are not added for videos
+        );
+
+        let mut msg = Message::new(Viewtype::Gif);
+        msg.set_text(no_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "GIF" // file names are not added for GIFs
+        );
+
+        let mut msg = Message::new(Viewtype::Sticker);
+        msg.set_text(no_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Sticker" // file names are not added for stickers
+        );
+
+        let mut msg = Message::new(Viewtype::Voice);
+        msg.set_text(empty_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Voice message" // file names are not added for voice messages, empty text is skipped
+        );
+
+        let mut msg = Message::new(Viewtype::Voice);
+        msg.set_text(no_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Voice message" // file names are not added for voice messages
+        );
+
+        let mut msg = Message::new(Viewtype::Voice);
+        msg.set_text(some_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Voice message \u{2013} bla bla" // `\u{2013}` explicitly checks for "EN DASH"
+        );
+
+        let mut msg = Message::new(Viewtype::Audio);
+        msg.set_text(no_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Audio \u{2013} foo.bar" // file name is added for audio
+        );
+
+        let mut msg = Message::new(Viewtype::Audio);
+        msg.set_text(empty_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Audio \u{2013} foo.bar" // file name is added for audio, empty text is not added
+        );
+
+        let mut msg = Message::new(Viewtype::Audio);
+        msg.set_text(some_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Audio \u{2013} foo.bar \u{2013} bla bla" // file name and text added for audio
+        );
+
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_text(some_text.clone());
+        msg.set_file("foo.bar", None);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "File \u{2013} foo.bar \u{2013} bla bla" // file name is added for files
+        );
+
+        // Forwarded
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(some_text.clone());
+        msg.param.set_int(Param::Forwarded, 1);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Forwarded: bla bla" // for simple text, the type is not added to the summary
+        );
+
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_text(some_text.clone());
+        msg.set_file("foo.bar", None);
+        msg.param.set_int(Param::Forwarded, 1);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Forwarded: File \u{2013} foo.bar \u{2013} bla bla"
+        );
+
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_text(no_text.clone());
+        msg.param.set(Param::File, "foo.bar");
+        msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
+        assert_eq!(
+            msg.get_summary_text(ctx).await,
+            "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
+        );
+    }
+}

src/sync.rs

@@ -0,0 +1,506 @@
+//! # Synchronize items between devices.
+
+use crate::chat::{Chat, ChatId};
+use crate::config::Config;
+use crate::constants::{Blocked, Viewtype, DC_CONTACT_ID_SELF};
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::message::{Message, MsgId};
+use crate::mimeparser::SystemMessage;
+use crate::param::Param;
+use crate::sync::SyncData::{AddQrToken, DeleteQrToken};
+use crate::token::Namespace;
+use crate::{chat, stock_str, token};
+use anyhow::Result;
+use itertools::Itertools;
+use lettre_email::mime::{self};
+use lettre_email::PartBuilder;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct QrTokenData {
+    pub(crate) invitenumber: String,
+    pub(crate) auth: String,
+    pub(crate) grpid: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) enum SyncData {
+    AddQrToken(QrTokenData),
+    DeleteQrToken(QrTokenData),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct SyncItem {
+    timestamp: i64,
+    data: SyncData,
+}
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct SyncItems {
+    items: Vec<SyncItem>,
+}
+
+impl Context {
+    /// Checks if sync messages shall be sent.
+    /// Receiving sync messages is currently always enabled;
+    /// the messages are force-encrypted anyway.
+    async fn is_sync_sending_enabled(&self) -> Result<bool> {
+        self.get_config_bool(Config::SendSyncMsgs).await
+    }
+
+    /// Adds an item to the list of items that should be synchronized to other devices.
+    pub(crate) async fn add_sync_item(&self, data: SyncData) -> Result<()> {
+        self.add_sync_item_with_timestamp(data, time()).await
+    }
+
+    /// Adds item and timestamp to the list of items that should be synchronized to other devices.
+    /// If device synchronization is disabled, the function does nothing.
+    async fn add_sync_item_with_timestamp(&self, data: SyncData, timestamp: i64) -> Result<()> {
+        if !self.is_sync_sending_enabled().await? {
+            return Ok(());
+        }
+
+        let item = SyncItem { timestamp, data };
+        let item = serde_json::to_string(&item)?;
+        self.sql
+            .execute(
+                "INSERT INTO multi_device_sync (item) VALUES(?);",
+                paramsv![item],
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    /// Adds most recent qr-code tokens for a given chat to the list of items to be synced.
+    /// If device synchronization is disabled,
+    /// no tokens exist or the chat is unpromoted, the function does nothing.
+    pub(crate) async fn sync_qr_code_tokens(&self, chat_id: Option<ChatId>) -> Result<()> {
+        if !self.is_sync_sending_enabled().await? {
+            return Ok(());
+        }
+
+        if let (Some(invitenumber), Some(auth)) = (
+            token::lookup(self, Namespace::InviteNumber, chat_id).await?,
+            token::lookup(self, Namespace::Auth, chat_id).await?,
+        ) {
+            let grpid = if let Some(chat_id) = chat_id {
+                let chat = Chat::load_from_db(self, chat_id).await?;
+                if !chat.is_promoted() {
+                    info!(
+                        self,
+                        "group '{}' not yet promoted, do not sync tokens yet.", chat.grpid
+                    );
+                    return Ok(());
+                }
+                Some(chat.grpid)
+            } else {
+                None
+            };
+            self.add_sync_item(SyncData::AddQrToken(QrTokenData {
+                invitenumber,
+                auth,
+                grpid,
+            }))
+            .await?;
+        }
+        Ok(())
+    }
+
+    // Add deleted qr-code token to the list of items to be synced
+    // so that the token also gets deleted on the other devices.
+    pub(crate) async fn sync_qr_code_token_deletion(
+        &self,
+        invitenumber: String,
+        auth: String,
+    ) -> Result<()> {
+        self.add_sync_item(SyncData::DeleteQrToken(QrTokenData {
+            invitenumber,
+            auth,
+            grpid: None,
+        }))
+        .await
+    }
+
+    /// Sends out a self-sent message with items to be synchronized, if any.
+    pub async fn send_sync_msg(&self) -> Result<Option<MsgId>> {
+        if let Some((json, ids)) = self.build_sync_json().await? {
+            let chat_id =
+                ChatId::create_for_contact_with_blocked(self, DC_CONTACT_ID_SELF, Blocked::Yes)
+                    .await?;
+            let mut msg = Message {
+                chat_id,
+                viewtype: Viewtype::Text,
+                text: Some(stock_str::sync_msg_body(self).await),
+                hidden: true,
+                subject: stock_str::sync_msg_subject(self).await,
+                ..Default::default()
+            };
+            msg.param.set_cmd(SystemMessage::MultiDeviceSync);
+            msg.param.set(Param::Arg, json);
+            msg.param.set(Param::Arg2, ids);
+            msg.param.set_int(Param::GuaranteeE2ee, 1);
+            Ok(Some(chat::send_msg(self, chat_id, &mut msg).await?))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Copies all sync items to a JSON string and clears the sync-table.
+    /// Returns the JSON string and a comma-separated string of the IDs used.
+    pub(crate) async fn build_sync_json(&self) -> Result<Option<(String, String)>> {
+        let (ids, serialized) = self
+            .sql
+            .query_map(
+                "SELECT id, item FROM multi_device_sync ORDER BY id;",
+                paramsv![],
+                |row| Ok((row.get::<_, u32>(0)?, row.get::<_, String>(1)?)),
+                |rows| {
+                    let mut ids = vec![];
+                    let mut serialized = String::default();
+                    for row in rows {
+                        let (id, item) = row?;
+                        ids.push(id);
+                        if !serialized.is_empty() {
+                            serialized.push_str(",\n");
+                        }
+                        serialized.push_str(&item);
+                    }
+                    Ok((ids, serialized))
+                },
+            )
+            .await?;
+
+        if ids.is_empty() {
+            Ok(None)
+        } else {
+            Ok(Some((
+                format!("{{\"items\":[\n{}\n]}}", serialized),
+                ids.iter().map(|x| x.to_string()).join(","),
+            )))
+        }
+    }
+
+    pub(crate) async fn build_sync_part(&self, json: String) -> PartBuilder {
+        PartBuilder::new()
+            .content_type(&"application/json".parse::<mime::Mime>().unwrap())
+            .header((
+                "Content-Disposition",
+                "attachment; filename=\"multi-device-sync.json\"",
+            ))
+            .body(json)
+    }
+
+    /// Deletes IDs as returned by `build_sync_json()`.
+    pub(crate) async fn delete_sync_ids(&self, ids: String) -> Result<()> {
+        self.sql
+            .execute(
+                format!("DELETE FROM multi_device_sync WHERE id IN ({});", ids),
+                paramsv![],
+            )
+            .await?;
+        Ok(())
+    }
+
+    /// Takes a JSON string created by `build_sync_json()`
+    /// and construct `SyncItems` from it.
+    pub(crate) async fn parse_sync_items(&self, serialized: String) -> Result<SyncItems> {
+        let sync_items: SyncItems = serde_json::from_str(&serialized)?;
+        Ok(sync_items)
+    }
+
+    /// Execute sync items.
+    ///
+    /// CAVE: When changing the code to handle other sync items,
+    /// take care that does not result in calls to `add_sync_item()`
+    /// as otherwise we would add in a dead-loop between two devices
+    /// sending message back and forth.
+    ///
+    /// If an error is returned, the caller shall not try over.
+    /// Therefore, errors should only be returned on database errors or so.
+    /// If eg. just an item cannot be deleted,
+    /// that should not hold off the other items to be executed.
+    pub(crate) async fn execute_sync_items(&self, items: &SyncItems) -> Result<()> {
+        info!(self, "executing {} sync item(s)", items.items.len());
+        for item in &items.items {
+            match &item.data {
+                AddQrToken(token) => {
+                    let chat_id = if let Some(grpid) = &token.grpid {
+                        if let Some((chat_id, _, _)) =
+                            chat::get_chat_id_by_grpid(self, grpid).await?
+                        {
+                            Some(chat_id)
+                        } else {
+                            warn!(
+                                self,
+                                "Ignoring token for nonexistent/deleted group '{}'.", grpid
+                            );
+                            continue;
+                        }
+                    } else {
+                        None
+                    };
+                    token::save(self, Namespace::InviteNumber, chat_id, &token.invitenumber)
+                        .await?;
+                    token::save(self, Namespace::Auth, chat_id, &token.auth).await?;
+                }
+                DeleteQrToken(token) => {
+                    token::delete(self, Namespace::InviteNumber, &token.invitenumber).await?;
+                    token::delete(self, Namespace::Auth, &token.auth).await?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chat::Chat;
+    use crate::chatlist::Chatlist;
+    use crate::test_utils::TestContext;
+    use crate::token::Namespace;
+    use anyhow::bail;
+
+    #[async_std::test]
+    async fn test_is_sync_sending_enabled() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        assert!(!t.is_sync_sending_enabled().await?);
+        t.set_config_bool(Config::SendSyncMsgs, true).await?;
+        assert!(t.is_sync_sending_enabled().await?);
+        t.set_config_bool(Config::SendSyncMsgs, false).await?;
+        assert!(!t.is_sync_sending_enabled().await?);
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_build_sync_json() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        t.set_config_bool(Config::SendSyncMsgs, true).await?;
+
+        assert!(t.build_sync_json().await?.is_none());
+
+        t.add_sync_item_with_timestamp(
+            SyncData::AddQrToken(QrTokenData {
+                invitenumber: "testinvite".to_string(),
+                auth: "testauth".to_string(),
+                grpid: Some("group123".to_string()),
+            }),
+            1631781316,
+        )
+        .await?;
+        t.add_sync_item_with_timestamp(
+            SyncData::DeleteQrToken(QrTokenData {
+                invitenumber: "123!?\":.;{}".to_string(),
+                auth: "456".to_string(),
+                grpid: None,
+            }),
+            1631781317,
+        )
+        .await?;
+
+        let (serialized, ids) = t.build_sync_json().await?.unwrap();
+        assert_eq!(
+            serialized,
+            r#"{"items":[
+{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"testinvite","auth":"testauth","grpid":"group123"}}},
+{"timestamp":1631781317,"data":{"DeleteQrToken":{"invitenumber":"123!?\":.;{}","auth":"456","grpid":null}}}
+]}"#
+        );
+
+        assert!(t.build_sync_json().await?.is_some());
+        t.delete_sync_ids(ids).await?;
+        assert!(t.build_sync_json().await?.is_none());
+
+        let sync_items = t.parse_sync_items(serialized).await?;
+        assert_eq!(sync_items.items.len(), 2);
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_build_sync_json_sync_msgs_off() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        t.set_config_bool(Config::SendSyncMsgs, false).await?;
+        t.add_sync_item(SyncData::AddQrToken(QrTokenData {
+            invitenumber: "testinvite".to_string(),
+            auth: "testauth".to_string(),
+            grpid: Some("group123".to_string()),
+        }))
+        .await?;
+        assert!(t.build_sync_json().await?.is_none());
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_parse_sync_items() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        assert!(t
+            .parse_sync_items(r#"{bad json}"#.to_string())
+            .await
+            .is_err());
+
+        assert!(t
+            .parse_sync_items(r#"{"badname":[]}"#.to_string())
+            .await
+            .is_err());
+
+        assert!(t.parse_sync_items(
+                r#"{"items":[{"timestamp":1631781316,"data":{"BadItem":{"invitenumber":"in","auth":"a","grpid":null}}}]}"#
+                    .to_string(),
+            )
+            .await.is_err());
+
+        assert!(t.parse_sync_items(
+                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":123}}}]}"#.to_string(),
+            )
+            .await
+            .is_err()); // `123` is invalid for `String`
+
+        assert!(t.parse_sync_items(
+                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":true}}}]}"#.to_string(),
+            )
+            .await
+            .is_err()); // `true` is invalid for `String`
+
+        assert!(t.parse_sync_items(
+                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":[]}}}]}"#.to_string(),
+            )
+            .await
+            .is_err()); // `[]` is invalid for `String`
+
+        assert!(t.parse_sync_items(
+                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":{}}}}]}"#.to_string(),
+            )
+            .await
+            .is_err()); // `{}` is invalid for `String`
+
+        assert!(t.parse_sync_items(
+                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","grpid":null}}}]}"#.to_string(),
+            )
+            .await
+            .is_err()); // missing field
+
+        // empty item list is okay
+        assert_eq!(
+            t.parse_sync_items(r#"{"items":[]}"#.to_string())
+                .await?
+                .items
+                .len(),
+            0
+        );
+
+        // to allow forward compatibility, additional fields should not break parsing
+        let sync_items = t
+            .parse_sync_items(
+                r#"{"items":[
+{"timestamp":1631781316,"data":{"DeleteQrToken":{"invitenumber":"in","auth":"yip","grpid":null}}},
+{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":"yip","additional":123,"grpid":null}}}
+]}"#
+                .to_string(),
+            )
+            .await?;
+        assert_eq!(sync_items.items.len(), 2);
+
+        let sync_items = t
+            .parse_sync_items(
+                r#"{"items":[
+{"timestamp":1631781318,"data":{"AddQrToken":{"invitenumber":"in","auth":"yip","grpid":null}}}
+],"additional":"field"}"#
+                    .to_string(),
+            )
+            .await?;
+
+        assert_eq!(sync_items.items.len(), 1);
+        if let AddQrToken(token) = &sync_items.items.get(0).unwrap().data {
+            assert_eq!(token.invitenumber, "in");
+            assert_eq!(token.auth, "yip");
+            assert_eq!(token.grpid, None);
+        } else {
+            bail!("bad item");
+        }
+
+        // to allow backward compatibility, missing `Option<>` should not break parsing
+        let sync_items = t.parse_sync_items(
+               r#"{"items":[{"timestamp":1631781319,"data":{"AddQrToken":{"invitenumber":"in","auth":"a"}}}]}"#.to_string(),
+           )
+           .await?;
+        assert_eq!(sync_items.items.len(), 1);
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_execute_sync_items() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        assert!(!token::exists(&t, Namespace::Auth, "yip-auth").await);
+
+        let sync_items = t
+            .parse_sync_items(
+                r#"{"items":[
+{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"yip-in","auth":"a"}}},
+{"timestamp":1631781316,"data":{"DeleteQrToken":{"invitenumber":"in","auth":"delete unexistant, shall continue"}}},
+{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":"yip-auth"}}},
+{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":"foo","grpid":"non-existant"}}},
+{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":"directly deleted"}}},
+{"timestamp":1631781316,"data":{"DeleteQrToken":{"invitenumber":"in","auth":"directly deleted"}}}
+]}"#
+                .to_string(),
+            )
+            .await?;
+        t.execute_sync_items(&sync_items).await?;
+
+        assert!(token::exists(&t, Namespace::InviteNumber, "yip-in").await);
+        assert!(token::exists(&t, Namespace::Auth, "yip-auth").await);
+        assert!(!token::exists(&t, Namespace::Auth, "non-existant").await);
+        assert!(!token::exists(&t, Namespace::Auth, "directly deleted").await);
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_send_sync_msg() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        alice.set_config_bool(Config::SendSyncMsgs, true).await?;
+        alice
+            .add_sync_item(SyncData::AddQrToken(QrTokenData {
+                invitenumber: "in".to_string(),
+                auth: "testtoken".to_string(),
+                grpid: None,
+            }))
+            .await?;
+        let msg_id = alice.send_sync_msg().await?.unwrap();
+        let msg = Message::load_from_db(&alice, msg_id).await?;
+        let chat = Chat::load_from_db(&alice, msg.chat_id).await?;
+        assert!(chat.is_self_talk());
+
+        // check that the used self-talk is not visible to the user
+        // but that creation will still work (in this case, the chat is empty)
+        assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 0);
+        let chat_id = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
+        let chat = Chat::load_from_db(&alice, chat_id).await?;
+        assert!(chat.is_self_talk());
+        assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 1);
+        let msgs = chat::get_chat_msgs(&alice, chat_id, 0, None).await?;
+        assert_eq!(msgs.len(), 0);
+
+        // let alice's other device receive and execute the sync message,
+        // also here, self-talk should stay hidden
+        let sent_msg = alice.pop_sent_msg().await;
+        let alice2 = TestContext::new_alice().await;
+        alice2.recv_msg(&sent_msg).await;
+        assert!(token::exists(&alice2, token::Namespace::Auth, "testtoken").await);
+        assert_eq!(Chatlist::try_load(&alice2, 0, None, None).await?.len(), 0);
+
+        // the same sync message sent to bob must not be executed
+        let bob = TestContext::new_bob().await;
+        bob.recv_msg(&sent_msg).await;
+        assert!(!token::exists(&bob, token::Namespace::Auth, "testtoken").await);
+
+        Ok(())
+    }
+}

src/test_utils.rs

@@ -34,6 +34,9 @@ use crate::message::{update_msg_state, Message, MessageState, MsgId};
 use crate::mimeparser::MimeMessage;
 use crate::param::{Param, Params};
 
+#[allow(non_upper_case_globals)]
+pub const AVATAR_900x900_BYTES: &[u8] = include_bytes!("../test-data/image/avatar900x900.png");
+
 type EventSink =
     dyn Fn(Event) -> Pin<Box<dyn Future<Output = ()> + Send + 'static>> + Send + Sync + 'static;
 
@@ -108,12 +111,15 @@ impl TestContext {
         let (evtracker_sender, evtracker_receiver) = channel::unbounded();
 
         async_std::task::spawn(async move {
-            // Make sure that the test fails if there is a panic on this thread here:
-            let current_id = task::current().id();
+            // Make sure that the test fails if there is a panic on this thread here
+            // (but not if there is a panic on another thread)
+            let looptask_id = task::current().id();
             let orig_hook = panic::take_hook();
             panic::set_hook(Box::new(move |panic_info| {
-                if task::current().id() == current_id {
-                    poison_sender.try_send(panic_info.to_string()).ok();
+                if let Some(panicked_task) = task::try_current() {
+                    if panicked_task.id() == looptask_id {
+                        poison_sender.try_send(panic_info.to_string()).ok();
+                    }
                 }
                 orig_hook(panic_info);
             }));
@@ -587,6 +593,21 @@ impl EvTracker {
             }
         }
     }
+
+    pub async fn get_matching<F: Fn(EventType) -> bool>(&self, event_matcher: F) -> EventType {
+        const TIMEOUT: Duration = Duration::from_secs(20);
+
+        loop {
+            let event = async_std::future::timeout(TIMEOUT, self.recv())
+                .await
+                .unwrap()
+                .unwrap();
+
+            if event_matcher(event.clone()) {
+                return event;
+            }
+        }
+    }
 }
 
 impl Deref for EvTracker {

src/update_helper.rs

@@ -0,0 +1,197 @@
+//! # Functions to update timestamps.
+
+use crate::chat::{Chat, ChatId};
+use crate::contact::Contact;
+use crate::context::Context;
+use crate::param::{Param, Params};
+use anyhow::Result;
+
+impl Context {
+    /// Updates a contact's timestamp, if reasonable.
+    /// Returns true if the caller shall update the settings belonging to the scope.
+    /// (if we have a ContactId type at some point, the function should go there)
+    pub(crate) async fn update_contacts_timestamp(
+        &self,
+        contact_id: u32,
+        scope: Param,
+        new_timestamp: i64,
+    ) -> Result<bool> {
+        let mut contact = Contact::load_from_db(self, contact_id).await?;
+        if contact.param.update_timestamp(scope, new_timestamp)? {
+            contact.update_param(self).await?;
+            return Ok(true);
+        }
+        Ok(false)
+    }
+}
+
+impl ChatId {
+    /// Updates a chat id's timestamp on disk, if reasonable.
+    /// Returns true if the caller shall update the settings belonging to the scope.
+    pub(crate) async fn update_timestamp(
+        &self,
+        context: &Context,
+        scope: Param,
+        new_timestamp: i64,
+    ) -> Result<bool> {
+        let mut chat = Chat::load_from_db(context, *self).await?;
+        if chat.param.update_timestamp(scope, new_timestamp)? {
+            chat.update_param(context).await?;
+            return Ok(true);
+        }
+        Ok(false)
+    }
+}
+
+impl Params {
+    /// Updates a param's timestamp in memory, if reasonable.
+    /// Returns true if the caller shall update the settings belonging to the scope.
+    pub(crate) fn update_timestamp(&mut self, scope: Param, new_timestamp: i64) -> Result<bool> {
+        let old_timestamp = self.get_i64(scope).unwrap_or_default();
+        if new_timestamp >= old_timestamp {
+            self.set_i64(scope, new_timestamp);
+            return Ok(true);
+        }
+        Ok(false)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::dc_tools::time;
+    use crate::test_utils::TestContext;
+
+    #[async_std::test]
+    async fn test_params_update_timestamp() -> Result<()> {
+        let mut params = Params::new();
+        let ts = time();
+
+        assert!(params.update_timestamp(Param::LastSubject, ts)?);
+        assert!(params.update_timestamp(Param::LastSubject, ts)?); // same timestamp -> update
+        assert!(params.update_timestamp(Param::LastSubject, ts + 10)?);
+        assert!(!params.update_timestamp(Param::LastSubject, ts)?); // `ts` is now too old
+        assert!(!params.update_timestamp(Param::LastSubject, 0)?);
+        assert_eq!(params.get_i64(Param::LastSubject).unwrap(), ts + 10);
+
+        assert!(params.update_timestamp(Param::GroupNameTimestamp, 0)?); // stay unset -> update ...
+        assert!(params.update_timestamp(Param::GroupNameTimestamp, 0)?); // ... also on multiple calls
+        assert_eq!(params.get_i64(Param::GroupNameTimestamp).unwrap(), 0);
+
+        assert!(!params.update_timestamp(Param::AvatarTimestamp, -1)?);
+        assert_eq!(params.get_i64(Param::AvatarTimestamp), None);
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_out_of_order_subject() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        dc_receive_imf(
+            &t,
+            b"From: Bob Authname <bob@example.org>\n\
+                 To: alice@example.com\n\
+                 Subject: updated subject\n\
+                 Message-ID: <msg2@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Date: Sun, 22 Mar 2021 23:37:57 +0000\n\
+                 \n\
+                 second message\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await?;
+        dc_receive_imf(
+            &t,
+            b"From: Bob Authname <bob@example.org>\n\
+                 To: alice@example.com\n\
+                 Subject: original subject\n\
+                 Message-ID: <msg1@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
+                 \n\
+                 first message\n",
+            "INBOX",
+            2,
+            false,
+        )
+        .await?;
+
+        let msg = t.get_last_msg().await;
+        let chat = Chat::load_from_db(&t, msg.chat_id).await?;
+        assert_eq!(
+            chat.param.get(Param::LastSubject).unwrap(),
+            "updated subject"
+        );
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_out_of_order_group_name() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        dc_receive_imf(
+            &t,
+            b"From: Bob Authname <bob@example.org>\n\
+                 To: alice@example.com\n\
+                 Message-ID: <msg1@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: abcde\n\
+                 Chat-Group-Name: initial name\n\
+                 Date: Sun, 22 Mar 2021 01:00:00 +0000\n\
+                 \n\
+                 first message\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await?;
+        let msg = t.get_last_msg().await;
+        let chat = Chat::load_from_db(&t, msg.chat_id).await?;
+        assert_eq!(chat.name, "initial name");
+
+        dc_receive_imf(
+            &t,
+            b"From: Bob Authname <bob@example.org>\n\
+                 To: alice@example.com\n\
+                 Message-ID: <msg3@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: abcde\n\
+                 Chat-Group-Name: another name update\n\
+                 Chat-Group-Name-Changed: a name update\n\
+                 Date: Sun, 22 Mar 2021 03:00:00 +0000\n\
+                 \n\
+                 third message\n",
+            "INBOX",
+            2,
+            false,
+        )
+        .await?;
+        dc_receive_imf(
+            &t,
+            b"From: Bob Authname <bob@example.org>\n\
+                 To: alice@example.com\n\
+                 Message-ID: <msg2@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: abcde\n\
+                 Chat-Group-Name: a name update\n\
+                 Chat-Group-Name-Changed: initial name\n\
+                 Date: Sun, 22 Mar 2021 02:00:00 +0000\n\
+                 \n\
+                 second message\n",
+            "INBOX",
+            3,
+            false,
+        )
+        .await?;
+        let msg = t.get_last_msg().await;
+        let chat = Chat::load_from_db(&t, msg.chat_id).await?;
+        assert_eq!(chat.name, "another name update");
+
+        Ok(())
+    }
+}

test-data/message/mailinglist_ttline.eml

@@ -0,0 +1,60 @@
+Return-Path: <return@t.ttline.com>
+X-Original-To: pdetersen@b123.com
+Delivered-To: m123123f@d123123.kasserver.com
+X-policyd-weight:  NOT_IN_SPAMCOP=-1.5 NOT_IN_IX_MANITU=-1.5 CL_IP_EQ_HELO_IP=-2 (check from: .ttline. - helo: .mail18-212.srv2. - helo-domain: .srv2.)  FROM/MX_MATCHES_HELO(DOMAIN)=-2; rate: -7
+Authentication-Results: d123123.kasserver.com;
+	dkim=pass (1024-bit key; unprotected) header.d=ttline.com header.i=newsletter@ttline.com header.b="SEFAAx0a";
+	dkim=pass (1024-bit key; unprotected) header.d=srv2.de header.i=@srv2.de header.b="UqUBlHLF";
+	dkim-atps=neutral
+Received: from mail18-212.srv2.de (mail18-212.srv2.de [193.169.181.212])
+	by d123123.kasserver.com (Postfix) with ESMTPS id 4216753C0228
+	for <pdetersen@b123.com>; Mon, 12 Jul 2021 18:00:55 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed; s=mailing; d=ttline.com;
+ h=Date:From:Reply-To:To:Message-ID:Subject:MIME-Version:Content-Type:X-ulpe:
+ List-Id:X-CSA-Complaints:List-Unsubscribe:List-Unsubscribe-Post:Feedback-ID;
+ i=newsletter@ttline.com;
+ bh=s/Rjfns4gt9JjcDSSsqHZctvWTOtocDJRpEVs80pElM=;
+ b=SEFAAx0aG2fD5fytZ1z0WI2elUpWh5J+ekno+UQE/PDqc4bwz5xEUGRXuBszhV9vh3UJVq9HL0Lz
+   40Bcjzcoob+Iza9KKnl0spLKMPgQNpoCerBCdE/v/DmiWus/gs2MOE+xE5dTM6A8kK0K4ukDoDjr
+   mnkjezkK8iuh5wwjPqA=
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed; s=mailing; d=srv2.de;
+ h=Date:From:Reply-To:To:Message-ID:Subject:MIME-Version:Content-Type:X-ulpe:
+ List-Id:X-CSA-Complaints:List-Unsubscribe:List-Unsubscribe-Post:Feedback-ID;
+ bh=s/Rjfns4gt9JjcDSSsqHZctvWTOtocDJRpEVs80pElM=;
+ b=UqUBlHLFoluhBzwmQDgHdd9OdiyI9Cy8Y5zqJqfyhmdV34Owpvu1Vx7HnlljqxlUTSVSPtL6Ldoe
+   bjWHA8yBc0lFKnF7Kt8a2Wd2ac0aHsLgQvwVmoM0T9Av8Hgx4qyRhaTQIho2IbcKcP0IEwEUKVou
+   KwU4tfT8MLuZHX4rkWc=
+Date: Mon, 12 Jul 2021 18:00:54 +0200 (CEST)
+From: =?UTF-8?Q?TT-Line_-_Die_Schwedenf=C3=A4hren?= <newsletter@ttline.com>
+Reply-To: 
+	=?UTF-8?Q?TT-Line_-_Die_Schwedenf=C3=A4hren?= <newsletter@ttline.com>
+To: pdetersen@b123.com
+Message-ID: <re-p123123123123123123123123-41231231-4J123123-1123123@t.ttline.com>
+Subject: =?UTF-8?Q?Unsere_Sommerangebote_an_Bord_=E2=9A=93?=
+MIME-Version: 1.0
+Content-Type: multipart/alternative; 
+	boundary="----=_Part_15756440_1404911700.1626105654088"
+X-ulpe: re-p123123123123123123123123-41231231-4J123123-1123123@t.ttline.com
+List-Id: <39123123-1BBQXPY.t.ttline.com>
+X-Report-Spam: complaints@episerver.com
+X-CSA-Complaints: csa-complaints@eco.de
+X-sender: =?UTF-8?Q?TT-Line_-_Die_Schwedenf=C3=A4hren?= <newsletter@ttline.com>
+List-Unsubscribe: <mailto:listoff-41231231-4J123123-5M123@t.ttline.com?subject=unsubscribe>,<https://t.ttline.com/go/0/41231231-4J123123-39123123-A9E123-UL.html>
+List-Unsubscribe-Post: List-Unsubscribe=One-Click
+Feedback-ID: 39123123:4J123123:episerver
+X-KasLoop: m123123f
+
+------=_Part_15756440_1404911700.1626105654088
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+Content-Disposition: inline
+
+plain...
+------=_Part_15756440_1404911700.1626105654088
+Content-Type: text/html; charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+Content-Disposition: inline
+
+html...
+------=_Part_15756440_1404911700.1626105654088--
+

test-data/message/mailinglist_xing.eml

@@ -0,0 +1,75 @@
+Return-Path: <mailrobot@mail.xing.com>
+X-Original-To: pbetersen@b123.com
+Delivered-To: m123123f@dd12312.kasserver.com
+X-policyd-weight: using cached result; rate: -7
+Authentication-Results: dd12312.kasserver.com;
+	dkim=pass (1024-bit key; unprotected) header.d=mail.xing.com header.i=@mail.xing.com header.b="o123123j";
+	dkim-atps=neutral
+Received: from mailout1-107.xing.com (mailout1-107.xing.com [109.233.158.107])
+	by dd12312.kasserver.com (Postfix) with ESMTPS id DCB9D53C055C
+	for <pbetersen@b123.com>; Tue, 14 Sep 2021 12:11:17 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; d=mail.xing.com; s=main; c=relaxed/simple;
+	q=dns/txt; i=@mail.xing.com; t=1631614277;
+	h=From:Subject:Date:To:Mime-Version:Content-Type:X-CSA-Complaints:List-Help;
+	bh=bc0sqvXhKYO4cLpVn9ZYqLQQYPgPysrjt9/f6mx2BQI=;
+	b=o123123jtfoTCEiqaRZ3ax7h17rGfXP1ZQ3sjbdaBguPy5q1k+EpuXYdCwFq7S7z
+	yRjbK+VvTSKDn4Dxqk/wA9hFyGrO6XuYdJt3NEZ3Yye7W222dNR58ww3XCavjSpY
+	pzhJdAEo9Zw1sG3fhtm2eI60Oe1hLOr6G657sAo2ubQ=;
+X-MSFBL: SwzhdcKRUcljkfTYaEiSU28Q3a0pxO+Z9dY1NMgWyq4=|eyJnIjoibWFpbG91dDE
+	iLCJiIjoibWFpbG91dDEtMTA3IiwidSI6ImNvbnRhY3RzL215bWs7RzhMWjI0RlR
+	SSUItdVkxaXRaa1EiLCJyIjoiYnBldGVyc2VuQGI0NHQuY29tIn0=
+Received: from [10.12.225.241] ([10.12.225.241:27696])
+	by mta-6.mail.ams1.xing.com (envelope-from <mailrobot@mail.xing.com>)
+	(ecelerity 4.2.1.51128 r(Core:4.2.1.5)) with REST
+	id 82/69-03094-54570416; Tue, 14 Sep 2021 12:11:17 +0200
+Date: Tue, 14 Sep 2021 12:11:17 +0200
+From: =?UTF-8?B?WElORyBLb250YWt0dm9yc2NobMOkZ2U=?= <mailrobot@mail.xing.com>
+Reply-To: no-reply@mail.xing.com
+To: =?UTF-8?B?QmrDtnJuIFBldGVyc2Vu?= <pbetersen@b123.com>
+Message-ID: <6123123123123_5123123123123@hermes-worker-5123123123-7lcmp.mail>
+Subject: Kennst Du Dr. Mabuse?
+Mime-Version: 1.0
+Content-Type: multipart/alternative;
+ boundary="--==_mimepart_614075452ac33_5ed3cc13386fc";
+ charset=UTF-8
+Content-Transfer-Encoding: 7bit
+List-Help: <https://www.xing.com/settings/notifications>,
+ <mailto:fbl@xing.com>
+List-Id: <51231231231231231231231232869f58.xing.com>
+X-CSA-Complaints: csa-complaints@eco.de
+X-KasLoop: m123123f
+
+
+----==_mimepart_614075452ac33_5ed3cc13386fc
+Content-Type: text/plain;
+ charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+
+**********************************************************
+* unfortunately:
+*
+* - xing mailinglists do not have a name in `List-Id:`
+*   so we cannot get the name from there
+*
+* - different senders may use the same `List-Id:`,
+*   at least i found that two times,
+*   maybe it is a bug on xing's side as most times `List-Id:` differs,
+*   however, so we cannot get the name from `From:`.
+*
+* to avoid chat names as `51231231231231231231231232869f58.xing.com`,
+* we detect the hash prefix and strip that;
+* as the sender, we have "xing.com" then, which is fine.
+*
+* that approach should also work for other mailinglist.
+**********************************************************
+
+
+----==_mimepart_614075452ac33_5ed3cc13386fc
+Content-Type: text/html;
+ charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+
+html ...
+
+----==_mimepart_614075452ac33_5ed3cc13386fc--
+

test-data/message/mailinglist_xt_local_microsoft.eml

@@ -0,0 +1,164 @@
+Return-Path: <bounce-889884_HTML-1111111111-2222222-33333333-4444@bounce.microsoftstoreemail.com>
+X-Original-To: me@foobar.com
+Delivered-To: m111111f@dd22222.kasserver.com
+X-policyd-weight:  NOT_IN_SPAMCOP=-1.5 NOT_IN_IX_MANITU=-1.5 CL_IP_EQ_HELO_MX=-3.1 (check from: .microsoftstoreemail. - helo: .merlinux. - helo-domain: .merlinux.)  FROM/MX_MATCHES_NOT_HELO(DOMAIN)=1; rate: -5.1
+Authentication-Results: dd22222.kasserver.com;
+	dkim=pass (1024-bit key; unprotected) header.d=microsoftstoreemail.com header.i=microsoftstore@microsoftstoreemail.com header.b="XV8jLprF";
+	dkim-atps=neutral
+Received: from merlin.eu (hq6.merlin.eu [95.217.159.152])
+	by dd22222.kasserver.com (Postfix) with ESMTPS id BAD2E53C0541
+	for <me@foobar.com>; Mon, 30 Aug 2021 20:29:57 +0200 (CEST)
+Received: from [127.0.0.1] (localhost [127.0.0.1])
+	(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
+	(No client certificate requested)
+	by merlin.eu (Postfix) with ESMTPS id 40DA541C7E
+	for <me@merlin.eu>; Mon, 30 Aug 2021 20:29:55 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; s=200608; d=microsoftstoreemail.com;
+ h=From:To:Subject:Date:List-Help:MIME-Version:List-ID:X-CSA-Complaints:
+ Message-ID:Content-Type; i=microsoftstore@microsoftstoreemail.com;
+ bh=dvrsbCk+3USZNtvsQRvPSo2qpqsG0det56Snu0/Vz7I=;
+ b=XV8jLprFcPv/OmruNBYNRrau26cDZYl4EchN88fJa3q49VpWwom5Pakcw2fkj1i63acBRpyVOBEr
+   M3rZ/p/S3c+n5wkqcQJO/ruWPR16GacnfwYq3zGFIEs5HVjFLbMF+26YuCD7u6GEJC559yD7kWje
+   1RCh9UZqYxBOsdYhkyk=
+Received: by mta16.microsoftstoreemail.com id h1111111111v for <me@merlin.eu>; Mon, 30 Aug 2021 18:14:50 +0000 (envelope-from <bounce-889884_HTML-1111111111-2222222-33333333-4444@bounce.microsoftstoreemail.com>)
+From: "Microsoft Store" <microsoftstore@microsoftstoreemail.com>
+To: <me@merlin.eu>
+Subject: Notice of Update to Microsoft Store Marketing Disclosure Document
+Date: Mon, 30 Aug 2021 12:14:50 -0600
+List-Help: <https://click.microsoftstoreemail.com/subscription_center.aspx?jwt=123.123.123>
+x-CSA-Compliance-Source: SFMC
+MIME-Version: 1.0
+List-ID: <96540.xt.local>
+X-CSA-Complaints: csa-complaints@eco.de
+X-SFMC-Stack: 1
+x-job: 10359607_7641603
+Message-ID: <9b806dd3-1234-1234-1234-49603f588b2c@ind1s01mta720.xt.local>
+Content-Type: multipart/alternative;
+	boundary="x28u2yBkExvV=_?:"
+X-Spamd-Bar: ++++
+X-Spam-Level: ****
+X-Rspamd-Server: hq6
+Authentication-Results: merlin.eu;
+	dkim=pass header.d=microsoftstoreemail.com;
+	dmarc=pass (policy=none) header.from=microsoftstoreemail.com;
+	spf=pass smtp.mailfrom=bounce-889884_HTML-1111111111-2222222-33333333-4444@bounce.microsoftstoreemail.com
+X-Rspamd-Queue-Id: 40DA541C7E
+X-Spamd-Result: default: False [4.17 / 15.00];
+	 ARC_NA(0.00)[];
+	 R_DKIM_ALLOW(-0.20)[microsoftstoreemail.com];
+	 FROM_HAS_DN(0.00)[];
+	 R_SPF_ALLOW(-0.20)[+ip4:64.132.88.0/23];
+	 TO_MATCH_ENVRCPT_ALL(0.00)[];
+	 R_BAD_CTE_7BIT(1.05)[7bit,utf8];
+	 PREVIOUSLY_DELIVERED(0.00)[me@merlin.eu];
+	 TO_DN_NONE(0.00)[];
+	 MIME_GOOD(-0.10)[multipart/alternative,text/plain];
+	 URI_COUNT_ODD(1.00)[63];
+	 RCPT_COUNT_ONE(0.00)[1];
+	 MANY_INVISIBLE_PARTS(0.10)[2];
+	 IP_SCORE(-0.47)[asn: 22606(-2.23), country: US(-0.10)];
+	 DKIM_TRACE(0.00)[microsoftstoreemail.com:+];
+	 HTML_SHORT_LINK_IMG_2(1.00)[];
+	 DMARC_POLICY_ALLOW(-0.50)[microsoftstoreemail.com,none];
+	 MX_GOOD(-0.01)[cached: inbound.s1.exacttarget.com];
+	 FORGED_SENDER(0.30)[microsoftstore@microsoftstoreemail.com,bounce-889884_HTML-1111111111-2222222-33333333-4444@bounce.microsoftstoreemail.com];
+	 RWL_MAILSPIKE_POSSIBLE(0.00)[177.89.132.64.rep.mailspike.net : 127.0.0.17];
+	 RCVD_TLS_LAST(0.00)[];
+	 HFILTER_URL_ONLY(2.20)[1];
+	 ASN(0.00)[asn:22606, ipnet:64.132.89.0/24, country:US];
+	 FROM_NEQ_ENVFROM(0.00)[microsoftstore@microsoftstoreemail.com,bounce-889884_HTML-1111111111-2222222-33333333-4444@bounce.microsoftstoreemail.com];
+	 GREYLIST(0.00)[pass,body];
+	 RCVD_COUNT_TWO(0.00)[2]
+X-KasLoop: m111111f
+
+This is a multi-part message in MIME format.
+
+--x28u2yBkExvV=_?:
+Content-Type: text/plain;
+	charset="utf-8"
+Content-Transfer-Encoding: 7bit
+
+
+
+
+<!--
+    en_us
+
+    @templateVersion = "Templates_v01"
+-->
+<span style="display: none"></span>
+<html lang="en">
+<head>
+    <title>Notice of Update to Microsoft Store Marketing Disclosure Document</title>
+    <style media="all" type="text/css">
+
+        div.preheader {
+            display: none !important;
+        }
+
+        ...
+
+    </style>
+</head>
+<body>
+    <!-- Wrapper -->
+    <div style="display: none; max-height: 0px; overflow: hidden;">Please read these important updates&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+
+... 100 or so more of these lines - and yes, this is really in a `text/plain` part ...
+
+</body>
+</html>
+
+
+--x28u2yBkExvV=_?:
+Content-Type: text/html;
+	charset="utf-8"
+Content-Transfer-Encoding: 7bit
+
+
+
+<!-- Template Top -->
+<!--
+    en_us
+
+    @templateVersion = "Templates_v01"
+-->
+<span style="display: none"></span>
+<html lang="en">
+<head>
+    <title>Notice of Update to Microsoft Store Marketing Disclosure Document</title>
+    <style media="all" type="text/css">
+
+        div.preheader {
+            display: none !important;
+        }
+
+        ...
+
+    </style>
+</head>
+<body>
+    <!-- Wrapper -->
+    <div style="display: none; max-height: 0px; overflow: hidden;">Please read these important updates&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;&nbsp;&zwnj;
+
+
+... 100 or so more of these lines ...
+
+</body>
+</html>
+
+
+--x28u2yBkExvV=_?:--
+

test-data/message/mailinglist_xt_local_spiegel.eml

@@ -0,0 +1,70 @@
+Return-Path: <bounce-155_HTML-111111111-222222-333333333-1234@bounce.angebote.spiegel.de>
+X-Original-To: me@foobar.com
+Delivered-To: m000002f@dd22222.kasserver.com
+X-policyd-weight:  NOT_IN_SPAMCOP=-1.5 NOT_IN_IX_MANITU=-1.5 CL_IP_EQ_HELO_IP=-2 (check from: .spiegel. - helo: .mta.angebote.spiegel. - helo-domain: .spiegel.)  FROM/MX_MATCHES_HELO(DOMAIN)=-2; rate: -7
+Authentication-Results: dd22222.kasserver.com;
+	dkim=pass (2048-bit key; unprotected) header.d=angebote.spiegel.de header.i=service@angebote.spiegel.de header.b="EO7/nr7w";
+	dkim-atps=neutral
+Received: from mta.angebote.spiegel.de (mta.angebote.spiegel.de [13.111.60.76])
+	by dd22222.kasserver.com (Postfix) with ESMTPS id 5667B53C0576
+	for <me@foobar.com>; Mon,  6 Sep 2021 13:27:43 +0200 (CEST)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; s=10dkim1; d=angebote.spiegel.de;
+ h=From:To:Subject:Date:List-Unsubscribe:List-Unsubscribe-Post:MIME-Version:
+ Reply-To:List-ID:X-CSA-Complaints:Message-ID:Content-Type;
+ i=service@angebote.spiegel.de;
+ bh=OgZoChKe0M5AaHNMVEyCAZ9jK0q01BajqQhtcqP22/Q=;
+ b=EO7/nr7w54xd68XV1+qUteycqAN63r7HH4QAw60wImD4rE76J4+vWAcf8TNYayM/vPefcM7zcfFk
+   ERwrlR/aT4BoRAWghyQHKgAZ0lwVKpqWGuYe9cF8wjLuYJOwPCoYIiry/GgOSXIVwazlXE2FRN9V
+   Y8m8OStmH3KbVZC55j1Ta6OXMMHEyvwZ+/OHbJ2CLW3jinw84NevP2aMDoI60TidBS5HYVclUT9W
+   IT4bs1o6a659LeVw/ViitQULL7c2P/UWPv0gm5w5IRci6jmdCOzfa+rvFmxSGIlfalTJ/VVG4V+V
+   PlbJl49RrPBuXl62ub6f1EPjFGtbJD8Gy3BliQ==
+Received: by mta.angebote.spiegel.de id h6ntj42fmd4o for <me@foobar.com>; Mon, 6 Sep 2021 11:27:41 +0000 (envelope-from <bounce-155_HTML-111111111-222222-333333333-1234@bounce.angebote.spiegel.de>)
+From: "DER SPIEGEL Kundenservice" <service@angebote.spiegel.de>
+To: <me@foobar.com>
+Subject: subject here
+Date: Mon, 06 Sep 2021 05:27:41 -0600
+List-Unsubscribe: <https://click.angebote.spiegel.de/subscription_center.aspx?jwt=123.123.123-123-123>, <mailto:leave-123-123-123-123-123@leave.angebote.spiegel.de>
+List-Unsubscribe-Post: List-Unsubscribe=One-Click
+x-CSA-Compliance-Source: SFMC
+MIME-Version: 1.0
+Reply-To: "DER SPIEGEL Kundenservice" <reply-123-123-123-123-123@angebote.spiegel.de>
+List-ID: <121231234.xt.local>
+X-CSA-Complaints: csa-complaints@eco.de
+X-SFMC-Stack: 10
+x-job: 100006074_884883
+Message-ID: <123-123-123-123-123@dfw123123a96.xt.local>
+Content-Type: multipart/alternative;
+	boundary="NwnF3UJtimpn=_?:"
+X-KasLoop: m000002f
+
+This is a multi-part message in MIME format.
+
+--NwnF3UJtimpn=_?:
+Content-Type: text/plain;
+	charset="utf-8"
+Content-Transfer-Encoding: 8bit
+
+
+  
+ 
+
+ 
+plain text here
+
+ 
+ 
+
+
+  
+ 
+
+
+--NwnF3UJtimpn=_?:
+Content-Type: text/html;
+	charset="utf-8"
+Content-Transfer-Encoding: 8bit
+
+<p>html text here</p>
+
+--NwnF3UJtimpn=_?:--
+

test-data/message/testrun_ndn_2.eml

@@ -0,0 +1,97 @@
+Return-Path: <>
+Delivered-To: alice@example.org
+Received: from hq5.merlinux.eu
+	by hq5.merlinux.eu with LMTP
+	id IF/+LHrTFGEQBQAAPzvFDg
+	(envelope-from <>)
+	for <alice@example.org>; Thu, 12 Aug 2021 09:53:30 +0200
+Received: by hq5.merlinux.eu (Postfix)
+	id 9C87727A0006; Thu, 12 Aug 2021 09:53:30 +0200 (CEST)
+Date: Thu, 12 Aug 2021 09:53:30 +0200 (CEST)
+From: MAILER-DAEMON@hq5.merlinux.eu (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: alice@example.org
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="A82D727A0003.1628754810/hq5.merlinux.eu"
+Content-Transfer-Encoding: 8bit
+Message-Id: <20210812075330.9C87727A0006@hq5.merlinux.eu>
+
+This is a MIME-encapsulated message.
+
+--A82D727A0003.1628754810/hq5.merlinux.eu
+Content-Description: Notification
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+This is the mail system at host hq5.merlinux.eu.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<bob@example.org>: Host or domain name not found. Name service error for
+    name=echedelyr.tk type=AAAA: Host not found
+
+--A82D727A0003.1628754810/hq5.merlinux.eu
+Content-Description: Delivery report
+Content-Type: message/global-delivery-status
+Content-Transfer-Encoding: 8bit
+
+Reporting-MTA: dns; hq5.merlinux.eu
+X-Postfix-Queue-ID: A82D727A0003
+X-Postfix-Sender: rfc822; alice@example.org
+Arrival-Date: Thu, 12 Aug 2021 09:53:29 +0200 (CEST)
+
+Final-Recipient: rfc822; bob@example.org
+Original-Recipient: rfc822;bob@example.org
+Action: failed
+Status: 5.4.4
+Diagnostic-Code: X-Postfix; Host or domain name not found. Name service error
+    for name=echedelyr.tk type=AAAA: Host not found
+
+--A82D727A0003.1628754810/hq5.merlinux.eu
+Content-Description: Undelivered Message Headers
+Content-Type: message/global-headers
+Content-Transfer-Encoding: 8bit
+
+Return-Path: <alice@example.org>
+Chat-Disposition-Notification-To: alice@example.org
+DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=testrun.org;
+	s=testrun; t=1628754810;
+	bh=exDToKrRerWMZ62UQVK/RgNiwDDKe+GqF6zGdG64jl8=;
+	h=Subject:References:Date:To:From:From;
+	b=SNTTDdhkppXqimCSPP+cqDvdzmryYwzurtZdN2XkTVQEeqMMdnGvEA9TOgeZpHqi0
+	 oHSjJ5oD+eUK1ECnfRuxDF9DmFnK7sbw1MaHxIcAVTLPgHrMxv+2Fjq1nmrerzmr1t
+	 z9jOYY8e6gfEBw1uDAfHMmIl4OGuoDSll8haqNF3C2JqdwTtcdtE/w6ERJwSeHhCsR
+	 bknan9Rh75Tr46Zh8WVi9YYRVDGFj7+OlL/67Va+Jxl3c4v4EJ5vF6ncxyJupP9eU2
+	 qndV+g1BbxsARo663codYZRiGh217AI8DG2HUr0rVOPdvWm1kw/NTkp3BxoHkv5q2a
+	 Uak5Jiieur6Hg==
+Subject: Message from Hocuri
+Chat-User-Avatar: 0
+MIME-Version: 1.0
+References: <Mr.5xqflwt0YFv.IXDFfHauvWx@testrun.org>
+Date: Thu, 12 Aug 2021 07:53:28 +0000
+Chat-Version: 1.0
+Autocrypt: addr=alice@example.org;
+	keydata=xjMEX3tmZxYJKwYBBAHaRw8BAQdAl4LKVKPRqxG1ZXEO8e9s1DZWt6f38wSuJnY0mLSOuf
+	7NFTxob2N1cmkxQHRlc3RydW4ub3JnPsKLBBAWCAAzAhkBBQJfe2ZnAhsDBAsJCAcGFQgJCgsCAxYC
+	ARYhBBmltZLxgqC0SHJPEL7qvlxmQUTNAAoJEL7qvlxmQUTNIucBAJkRclHRG7cWpFbMYW+rspEFIQ
+	j1GTKwriiBpk5ffnroAQC3h/scScpG/EeIPL0y80GRS5BoR1Ium3zrlR92EaijDc44BF97ZmcSCisG
+	AQQBl1UBBQEBB0CmxhyX/NuXIlrl0/fdeEseAv6KCbZ4tV3tIvSvnH1KHgMBCAfCeAQYFggAIAUCX3
+	tmZwIbDBYhBBmltZLxgqC0SHJPEL7qvlxmQUTNAAoJEL7qvlxmQUTNnkYA/3qY+e6PrtR1WT7PiVeZ
+	RIQBkkJjWWSx+lBQ5fNb3e92AQCBEG3OnGy4RrxOqWW2ry7ETP33CJeiwAwCvv4LQlwCCw==
+Message-ID: <Mr.5xqflwt0YFv.IXDFfHauvWx@testrun.org>
+To: <bob@example.org>
+From: Hocuri <alice@example.org>
+Content-Type: multipart/mixed; boundary="qtFe0wPDNHWVlvqV0B8ymdWmE6ZmKD"
+
+--A82D727A0003.1628754810/hq5.merlinux.eu--
+