Prototype dc_blob_t API

Previously default of `auto_vacuum=NONE` was used.

`PRAGMA auto_vacuum=INCERMENTAL` allows running `PRAGMA
incremental_vacuum`. Unlike `VACUUM`, `PRAGMA incremental_vacuum`
frees unused pages without rebuilding the whole database, like the
`VACUUM` does, so it does not require additional disk space to build a
vacuumed copy of the database. This may even free enough space on the
disk to run `VACUUM` afterwards.

New setting will only be enabled for new databases or if the `VACUUM`
command runs successfully. Currently `VACUUM` is only executed on
backup export, but may fail nevertheless if there is not enough space
on the disk.

Also try to run `PRAGMA incremental_vacuum` during housekeeping. It
may not be the best strategy, but likely does not make any difference
under normal usage when the database only grows and there are no free
pages. Free pages are created only if enough data is deleted to free
at least one database page of 4096 bytes, for example when automatic
deletion of messages is deleted for the first time. In the future if
more data is placed into the database, like avatars and other blobs,
it may be necessary to revise this strategy, for example to keep some
free pages instead of removing all of them each time by querying
`PRAGMA freelist_pages` and running `PRAGMA incremental_vacuum(N)`.

.github/workflows/ci.yml

@@ -8,6 +8,9 @@ on:
       - staging
       - trying
 
+env:
+  RUSTFLAGS: -Dwarnings
+
 jobs:
 
   fmt:
@@ -21,6 +24,8 @@ jobs:
           toolchain: stable
           override: true
       - run: rustup component add rustfmt
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v1
       - uses: actions-rs/cargo@v1
         with:
           command: fmt
@@ -35,6 +40,8 @@ jobs:
             toolchain: stable
             components: clippy
             override: true
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v1
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -76,14 +83,13 @@ jobs:
             rust: 1.54.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.48.0
-          # This is the Debian "bullseye" release version of Rust.
+          # Minimum Supported Rust Version = 1.51.0
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.48.0
+            rust: 1.51.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:
@@ -95,31 +101,14 @@ jobs:
         toolchain: ${{ matrix.rust }}
         override: true
 
-    - name: Cache cargo registry
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/registry
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo index
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/git
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo build
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+    - name: Cache rust cargo artifacts
+      uses: swatinem/rust-cache@v1
 
     - name: check
       uses: actions-rs/cargo@v1
-      env:
-        RUSTFLAGS: -D warnings
       with:
         command: check
-        args: --all --bins --examples --tests --features repl
+        args: --all --bins --examples --tests --features repl --benches
 
     - name: tests
       uses: actions-rs/cargo@v1
@@ -151,4 +140,4 @@ jobs:
         DCC_RS_TARGET: debug
         DCC_RS_DEV: ${{ github.workspace }}
       working-directory: python
-      run: tox -e lint,doc,py3
+      run: tox -e lint,mypy,doc,py3

.gitignore

@@ -28,3 +28,4 @@ deltachat-ffi/xml
 .rsynclist
 
 coverage/
+.DS_Store

CHANGELOG.md

@@ -1,4 +1,163 @@
-# Changelog 
+# Changelog
+
+## Unreleased
+
+### API changes
+
+- Removed `mvbox_watch` option. #2906
+  It is automatically enabled whenever `mvbox_move` is enabled.
+
+## 1.70.0
+
+### Fixes
+- fix: do not abort Param parsing on unknown keys #2856
+- fix: execute `Chat-Group-Member-Removed:` even when arriving disordered #2857
+
+
+## 1.69.0
+
+### Fixes
+- fix group-related system messages in multi-device setups #2848
+- fix "Google Workspace" (former "G Suite") issues related to bad resolvers #2852
+
+
+## 1.68.0
+
+### Fixes
+- fix chat assignment when forwarding #2843
+- fix layout issues with the generated QR code svg #2842
+
+
+## 1.67.0
+
+### API changes
+- `dc_get_securejoin_qr_svg(chat_id)` added #2815
+- added stock-strings `DC_STR_SETUP_CONTACT_QR_DESC` and `DC_STR_SECURE_JOIN_GROUP_QR_DESC`
+
+
+## 1.66.0
+
+### API changes
+- `dc_contact_get_last_seen()` added #2823
+- python: `Contact.last_seen` added #2823
+- removed `DC_STR_NEWGROUPDRAFT`, we don't set draft after creating group anymore #2805
+
+### Changes
+- python: add cutil.from_optional_dc_charpointer() #2824
+- refactorings #2807 #2822 #2825
+
+
+## 1.65.0
+
+### Changes
+- python: add mypy support and some type hints #2809
+
+### Fixes
+- do not disable ephemeral timer when downloading a message partially #2811
+- apply existing ephemeral timer also to partially downloaded messages;
+  after full download, the ephemeral timer starts over #2811
+- replace user-visible error on verification failure with warning;
+  the error is logged to the corresponding chat anyway #2808
+
+
+## 1.64.0
+
+### Fixes
+- add 'waiting for being added to the group' only for group-joins,
+  not for setup-contact #2797
+- prioritize In-Reply-To: and References: headers over group IDs when assigning
+  messages to chats to fix incorrect assignment of Delta Chat replies to
+  classic email threads #2795
+
+
+## 1.63.0
+
+### API changes
+- `dc_get_last_error()` added #2788
+
+### Changes
+- Optimize Autocrypt gossip #2743
+
+### Fixes
+- fix permanently hiding of one-to-one chats after secure-join #2791
+
+
+## 1.62.0
+
+### API Changes
+- `dc_join_securejoin()` now always returns immediately;
+  the returned chat may not allow sending (`dc_chat_can_send()` returns false)
+  which may change as usual on `DC_EVENT_CHAT_MODIFIED` #2508 #2767
+- introduce multi-device-sync-messages;
+  as older cores display them as files in self-chat,
+  they are currently only sent if config option `send_sync_msgs` is set #2669
+- add `DC_EVENT_SELFAVATAR_CHANGED` #2742
+
+### Changes
+- use system DNS instead of google for MX queries #2780
+- improve error logging #2758
+- improve tests #2764 #2781
+- improve ci #2770
+- refactorings #2677 #2728 #2740 #2729 #2766 #2778
+
+### Fixes
+- add Let's Encrypt certificate to core as it may be missing older devices #2752
+- prioritize certificate setting from user over the one from provider-db #2749
+- fix "QR process failed" error #2725
+- do not update quota in endless loop #2726
+
+
+## 1.61.0
+
+### API Changes
+- download-on-demand added: `dc_msg_get_download_state()`, `dc_download_full_msg()`
+  and `download_limit` config option #2631 #2696
+- `dc_create_broadcast_list()` and chat type `DC_CHAT_TYPE_BROADCAST` added #2707 #2722
+- allow ui-specific configs using `ui.`-prefix in key (`dc_set_config(context, "ui.*", value)`) #2672
+- new strings from `DC_STR_PARTIAL_DOWNLOAD_MSG_BODY`
+  to `DC_STR_PART_OF_TOTAL_USED` #2631 #2694 #2707 #2723
+- emit warnings and errors from account manager with account-id 0 #2712
+
+### Changes
+- notify about incoming contact requests #2690
+- messages are marked as read on first read receipt #2699
+- quota warning reappears after import, rewarning at 95% #2702
+- lock strict TLS if certificate checks are automatic #2711
+- always check certificates strictly when connecting over SOCKS5 in Automatic mode #2657
+- `Accounts` is not cloneable anymore #2654 #2658
+- update chat/contact data only when there was no newer update #2642
+- better detection of mailing list names #2665 #2685
+- log all decisions when applying ephemeral timer to chats #2679
+- connectivity view now translatable #2694 #2723
+- improve Doxygen documentation #2647 #2668 #2684 #2688 #2705
+- refactorings #2656 #2659 #2677 #2673 #2678 #2675 #2663 #2692 #2706
+- update provider database #2618
+
+### Fixes
+- ephemeral timer rollback protection #2693 #2709
+- recreate configured folders if they are deleted #2691
+- ignore MDNs sent to self #2674
+- recognize NDNs that put headers into "message/global-headers" part #2598
+- avoid `dc_get_contacts()` returning duplicate contact ids #2591
+- do not leak group names on forwarding messages #2719
+- in case of smtp-errors, iterate over all addresses to fix ipv6/v4 problems #2720
+- fix pkg-config file #2660
+- fix "QR process failed" error #2725
+
+
+## 1.60.0
+
+### Added
+- add device message to warn about QUOTA #2621
+- add SOCKS5 support #2474 #2620
+
+### Changes
+- don't emit multiple events with the same import/export progress number #2639
+- reduce message length limit to 5000 chars #2615
+
+### Fixes
+- keep event emitter from closing when there are no accounts #2636
+
 
 ## 1.59.0
 
@@ -24,7 +183,6 @@
 ## 1.58.0
 
 ### Fixes
-
 - move WAL file together with database
   and avoid using data if the database was not closed correctly before #2583
 

CMakeLists.txt

@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.16)
 project(deltachat LANGUAGES C)
+include(GNUInstallDirs)
 
 find_program(CARGO cargo)
 
@@ -35,7 +36,6 @@ add_custom_target(
 	"target/release/pkgconfig/deltachat.pc"
 )
 
-include(GNUInstallDirs)
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})

Cargo.toml

@@ -1,9 +1,10 @@
 [package]
 name = "deltachat"
-version = "1.59.0"
+version = "1.70.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
+resolver = "2"
 
 [profile.dev]
 debug = 0
@@ -15,75 +16,76 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1.0.42"
+anyhow = "1"
 async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", rev="c8800625f7cf29f437143ac7e720ac2730a0962f" }
-async-std-resolver = "0.20.3"
-async-std = { version = "~1.9.0", features = ["unstable"] }
-async-tar = "0.3.0"
-async-trait = "0.1.50"
-backtrace = "0.3.59"
+async-native-tls = { version = "0.3" }
+async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
+async-std-resolver = "0.20"
+async-std = { version = "1", features = ["unstable"] }
+async-tar = { version = "0.4", default-features=false }
+async-trait = "0.1"
+backtrace = "0.3"
 base64 = "0.13"
-bitflags = "1.3.1"
-byteorder = "1.3.1"
-chrono = "0.4.6"
-dirs = { version = "3.0.2", optional=true }
+bitflags = "1.3"
+byteorder = "1.3"
+chrono = "0.4"
+dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1.1"
-futures = "0.3.16"
+escaper = "0.1"
+futures = "0.3"
 hex = "0.4.0"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-indexmap = "1.7.0"
-itertools = "0.10.1"
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2.98"
+libc = "0.2"
 log = {version = "0.4.8", optional = true }
-mailparse = "0.13.5"
-native-tls = "0.2.3"
-num_cpus = "1.13.0"
-num-derive = "0.3.0"
-num-traits = "0.2.6"
-once_cell = "1.8.0"
+mailparse = "0.13"
+native-tls = "0.2"
+num_cpus = "1.13"
+num-derive = "0.3"
+num-traits = "0.2"
+once_cell = "1.9.0"
 percent-encoding = "2.0"
-pgp = { version = "0.7.0", default-features = false }
-pretty_env_logger = { version = "0.4.0", optional = true }
-quick-xml = "0.22.0"
-r2d2 = "0.8.9"
-r2d2_sqlite = "0.18.0"
-rand = "0.7.0"
-regex = "1.4.6"
-rusqlite = "0.25"
-rust-hsluv = "0.1.4"
-rustyline = { version = "8.2.0", optional = true }
-sanitize-filename = "0.3.0"
+pgp = { version = "0.7", default-features = false }
+pretty_env_logger = { version = "0.4", optional = true }
+quick-xml = "0.22"
+r2d2 = "0.8"
+r2d2_sqlite = "0.19"
+rand = "0.7"
+regex = "1.5"
+rusqlite = { version = "0.26", features = ["sqlcipher"] }
+rust-hsluv = "0.1"
+rustyline = { version = "9", optional = true }
+sanitize-filename = "0.3"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9.7"
-sha2 = "0.9.5"
-smallvec = "1.0.0"
-stop-token = "0.2.0"
-strum = "0.21.0"
-strum_macros = "0.21.1"
-surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
-thiserror = "1.0.26"
-toml = "0.5.6"
-url = "2.2.2"
+sha-1 = "0.10"
+sha2 = "0.10"
+smallvec = "1"
+stop-token = "0.6"
+strum = "0.23"
+strum_macros = "0.23"
+surf = { version = "2.3", default-features = false, features = ["h1-client"] }
+thiserror = "1"
+toml = "0.5"
+url = "2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
-humansize = "1.1.1"
+fast-socks5 = "0.4"
+humansize = "1"
+qrcodegen = "1.7.0"
+tagger = "3.3.0"
+textwrap = "0.14.2"
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1.9.0", features = ["unstable", "attributes"] }
+async-std = { version = "1", features = ["unstable", "attributes"] }
 criterion = "0.3"
-futures-lite = "1.12.0"
-log = "0.4.11"
-pretty_assertions = "0.7.2"
-pretty_env_logger = "0.4.0"
-proptest = "1.0"
-tempfile = "3.0"
+futures-lite = "1.12"
+log = "0.4"
+pretty_env_logger = "0.4"
+proptest = { version = "1", default-features = false, features = ["std"] }
+tempfile = "3"
 
 [workspace]
 members = [
@@ -118,5 +120,5 @@ harness = false
 default = ["vendored"]
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl"]
 nightly = ["pgp/nightly"]

assets/icon-broadcast.svg

@@ -0,0 +1,149 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   enable-background="new 0 0 128 128"
+   viewBox="0 0 60 60"
+   version="1.1"
+   id="svg878"
+   sodipodi:docname="icon-broadcast.svg"
+   width="60"
+   height="60"
+   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
+   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-broadcast.png"
+   inkscape:export-xdpi="409.60001"
+   inkscape:export-ydpi="409.60001">
+  <metadata
+     id="metadata884">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs882" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1329"
+     inkscape:window-height="847"
+     id="namedview880"
+     showgrid="false"
+     inkscape:zoom="5.21875"
+     inkscape:cx="36.598802"
+     inkscape:cy="32.191617"
+     inkscape:window-x="111"
+     inkscape:window-y="205"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg878"
+     inkscape:document-rotation="0" />
+  <radialGradient
+     id="c"
+     cx="65.25"
+     cy="89"
+     r="26.440001"
+     gradientTransform="matrix(0.77611266,0.11996647,-0.18999676,1.2286617,-11.305867,-60.065999)"
+     gradientUnits="userSpaceOnUse">
+    <stop
+       stop-color="#FFC107"
+       offset="0"
+       id="stop833" />
+    <stop
+       stop-color="#FFBD06"
+       offset=".3502"
+       id="stop835" />
+    <stop
+       stop-color="#FFB104"
+       offset=".6938"
+       id="stop837" />
+    <stop
+       stop-color="#FFA000"
+       offset="1"
+       id="stop839" />
+  </radialGradient>
+  <radialGradient
+     id="b"
+     cx="52.5"
+     cy="19.75"
+     r="92.975998"
+     gradientUnits="userSpaceOnUse"
+     gradientTransform="rotate(45.323856,68.997115,75.979538)">
+    <stop
+       stop-color="#EF5350"
+       offset="0"
+       id="stop848" />
+    <stop
+       stop-color="#EB4F4C"
+       offset=".246"
+       id="stop850" />
+    <stop
+       stop-color="#E04341"
+       offset=".4878"
+       id="stop852" />
+    <stop
+       stop-color="#CD302F"
+       offset=".7272"
+       id="stop854" />
+    <stop
+       stop-color="#C62828"
+       offset=".8004"
+       id="stop856" />
+    <stop
+       stop-color="#C62828"
+       offset="1"
+       id="stop858" />
+  </radialGradient>
+  <radialGradient
+     id="a"
+     cx="16.979"
+     cy="92"
+     r="24.165001"
+     gradientUnits="userSpaceOnUse"
+     gradientTransform="rotate(45.323856,68.997115,75.979538)"
+     xlink:href="#b">
+    <stop
+       stop-color="#E0E0E0"
+       offset="0"
+       id="stop863" />
+    <stop
+       stop-color="#CFCFCF"
+       offset=".3112"
+       id="stop865" />
+    <stop
+       stop-color="#A4A4A4"
+       offset=".9228"
+       id="stop867" />
+    <stop
+       stop-color="#9E9E9E"
+       offset="1"
+       id="stop869" />
+  </radialGradient>
+  <rect
+     y="0"
+     x="0"
+     height="60"
+     width="60"
+     id="rect1420"
+     style="fill:#7cc0bc;fill-opacity:1;stroke:none;stroke-width:1.29077" />
+  <path
+     id="path872"
+     style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.336872;stroke-opacity:1"
+     d="m 8.6780027,35.573064 0.032831,-11.910176 c 0.00138,-0.476406 0.4881282,-0.794259 0.9235226,-0.604877 l 4.1144877,2.345752 -0.02386,8.656315 -4.1268029,2.122946 C 9.1617452,36.370003 8.6766889,36.049472 8.6780027,35.573064 Z m 5.0469633,-1.508222 0.02386,-8.656314 31.145424,-9.537653 c 0.841472,-0.219211 1.65915,0.41667 1.656755,1.283728 l -0.06929,25.139995 c -0.0024,0.867062 -0.825942,1.500799 -1.663803,1.274581 z m 3.8042,6.892234 C 16.681121,40.104348 16.315444,38.819414 16.69043,37.591308 l 2.252234,-7.347193 c 0.2644,-0.861571 0.845185,-1.567441 1.641953,-1.989251 0.796769,-0.421808 1.706956,-0.509819 2.568531,-0.245419 l 7.263888,2.225804 c 1.775518,0.543235 2.780299,2.432591 2.232297,4.208094 L 30.3971,41.790532 c -0.545627,1.777887 -2.432591,2.780297 -4.208095,2.232298 l -7.263891,-2.225804 c -0.545033,-0.165864 -1.01825,-0.460162 -1.395948,-0.83995 z m 12.377693,-7.976728 c -0.07601,-0.07642 -0.17114,-0.133864 -0.280621,-0.167516 l -7.263891,-2.225803 c -0.233244,-0.07209 -0.421626,0.0013 -0.512275,0.04861 -0.09064,0.0474 -0.25772,0.166033 -0.327435,0.396899 l -2.252234,7.347191 c -0.108166,0.354628 0.09088,0.731541 0.447888,0.842099 l 7.263891,2.225802 c 0.354626,0.108174 0.731539,-0.09088 0.842099,-0.447888 l 2.249845,-7.344814 c 0.07453,-0.245145 0.0014,-0.504991 -0.167267,-0.67458 z" />
+</svg>

assets/qrcode_logo_footer.svg

@@ -0,0 +1,10 @@
+<text
+     xml:space="preserve"
+     style="font-weight:bold;font-size:24.4118px;line-height:1.25;font-family:sans-serif;fill:#aaaaaa;fill-opacity:1;stroke:none;stroke-width:0.915439"
+     x="42.325161"
+     y="23.32255"
+     id="text72398">get.delta.chat</text>
+  <path
+     id="path84310"
+     style="opacity:0.25;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.915439"
+     d="M 17.13769,0.00129321 C 7.6753075,0.11650893 0,7.8915283 0,17.362467 c 0,9.47094 7.6753075,17.059745 17.13769,16.944599 8.99669,-0.03598 6.880074,-5.025654 16.824785,-0.405885 -5.447648,-8.510047 0.184241,-9.642482 0.311117,-16.955289 0,-9.4709395 -7.673512,-17.0597453 -17.135895,-16.94459879 z M 17.0769,4.9986797 c 1.84214,0 3.447355,0.253959 4.815003,0.7616693 1.381603,0.5076411 2.072253,1.207862 2.072253,2.0990711 0,0.4286855 -0.167495,0.7836052 -0.50242,1.0656242 -0.334921,0.2819844 -0.724544,0.4237724 -1.171121,0.4237724 -0.641952,0 -1.396532,-0.3909376 -2.261778,-1.169353 C 19.14963,7.3898036 18.402555,6.83791 17.788507,6.5220182 17.188416,6.1950547 16.484552,6.0321266 15.675129,6.0321266 c -1.032717,0 -1.883352,0.1854523 -2.553215,0.5578447 -0.655913,0.372254 -0.98517,0.8460916 -0.98517,1.4214436 0,0.5414792 0.272815,1.0495355 0.817093,1.5233385 0.544275,0.4738026 1.946291,1.3367446 4.207097,2.5889976 2.414319,1.342419 4.117377,2.390985 5.108232,3.146807 1.004795,0.755857 1.821505,1.675853 2.449514,2.758846 0.628002,1.082993 0.942253,2.227607 0.942253,3.434674 0,2.120834 -0.929555,3.993314 -2.785656,5.617786 -1.84214,1.613228 -3.99694,2.41915 -6.467082,2.41915 -2.246845,0 -4.145607,-0.647976 -5.694677,-1.945312 -1.5490699,-1.297336 -2.3225722,-3.028063 -2.3225722,-5.194049 0,-2.087031 0.8506345,-3.83094 2.5532182,-5.229825 1.716541,-1.398884 3.824203,-2.245599 6.322256,-2.538897 -0.697774,-0.631749 -1.668763,-1.387225 -2.910816,-2.267155 -1.367648,-0.970199 -2.287914,-1.73045 -2.762402,-2.283243 -0.474491,-0.5640381 -0.711618,-1.1795944 -0.711618,-1.8451814 0,-0.9927581 0.572093,-1.7710351 1.716451,-2.3351077 1.144362,-0.5753173 2.636724,-0.8635642 4.478865,-0.8635642 z m 1.110327,10.3738083 c -4.005262,0.5302 -6.007576,2.75279 -6.007576,6.667322 0,2.01932 0.49495,3.587291 1.485805,4.704157 1.004806,1.116832 2.169696,1.675299 3.495479,1.675299 1.381602,0 2.520072,-0.535632 3.413229,-1.60738 0.893168,-1.082959 1.339187,-2.545264 1.339187,-4.384079 0,-2.662348 -1.242022,-5.013441 -3.726124,-7.055319 z" />

benches/contacts.rs

@@ -8,9 +8,7 @@ async fn address_book_benchmark(n: u32, read_count: u32) {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
+    let context = Context::new(dbfile.into(), id).await.unwrap();
 
     let book = (0..n)
         .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))

benches/create_account.rs

@@ -8,7 +8,7 @@ async fn create_accounts(n: u32) {
     let dir = tempdir().unwrap();
     let p: PathBuf = dir.path().join("accounts").into();
 
-    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+    let mut accounts = Accounts::new(p.clone()).await.unwrap();
 
     for expected_id in 2..n {
         let id = accounts.add_account().await.unwrap();

benches/search_msgs.rs

@@ -6,9 +6,7 @@ use std::path::Path;
 async fn search_benchmark(path: impl AsRef<Path>) {
     let dbfile = path.as_ref();
     let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
+    let context = Context::new(dbfile.into(), id).await.unwrap();
 
     for _ in 0..10u32 {
         context.search_msgs(None, "hello").await.unwrap();

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.59.0"
+version = "1.70.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -17,13 +17,13 @@ crate-type = ["cdylib", "staticlib"]
 [dependencies]
 deltachat = { path = "../", default-features = false }
 libc = "0.2"
-human-panic = "1.0.1"
-num-traits = "0.2.6"
+human-panic = "1"
+num-traits = "0.2"
 serde_json = "1.0"
-async-std = "1.9.0"
-anyhow = "1.0.42"
-thiserror = "1.0.26"
-rand = "0.7.3"
+async-std = "1"
+anyhow = "1"
+thiserror = "1"
+rand = "0.7"
 
 [features]
 default = ["vendored"]

deltachat-ffi/Doxyfile

@@ -583,7 +583,7 @@ SORT_MEMBERS_CTORS_1ST = NO
 # appear in their defined order.
 # The default value is: NO.
 
-SORT_GROUP_NAMES       = NO
+SORT_GROUP_NAMES       = YES
 
 # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
 # fully-qualified names, including namespaces. If set to NO, the class list will

deltachat-ffi/Doxyfile.css

@@ -4,4 +4,16 @@ div.fragment {
 	background-color: #e0e0e0;
 	border: 0;
 	padding: 1em;
+	border-radius: 6px;
 }
+
+code {
+	background-color: #e0e0e0;
+	padding-left: .5em;
+	padding-right: .5em;
+	border-radius: 6px;
+}
+
+li {
+	margin-bottom: .5em;
+}

deltachat-ffi/deltachat.h

@@ -14,6 +14,7 @@ extern "C" {
 typedef struct _dc_context  dc_context_t;
 typedef struct _dc_accounts dc_accounts_t;
 typedef struct _dc_array    dc_array_t;
+typedef struct _dc_blob     dc_blob_t;
 typedef struct _dc_chatlist dc_chatlist_t;
 typedef struct _dc_chat     dc_chat_t;
 typedef struct _dc_msg      dc_msg_t;
@@ -271,6 +272,11 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `send_port`    = SMTP-port, guessed if left out
  * - `send_security`= SMTP-socket, one of @ref DC_SOCKET, defaults to #DC_SOCKET_AUTO
  * - `server_flags` = IMAP-/SMTP-flags as a combination of @ref DC_LP flags, guessed if left out
+ * - `socks5_enabled` = SOCKS5 enabled
+ * - `socks5_host` = SOCKS5 proxy server host
+ * - `socks5_port` = SOCKS5 proxy server port
+ * - `socks5_user` = SOCKS5 proxy username
+ * - `socks5_password` = SOCKS5 proxy password
  * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `displayname`  = Own name to use when sending messages.  MUAs are allowed to spread this way e.g. using CC, defaults to empty
@@ -288,18 +294,14 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    1=send a copy of outgoing messages to self.
  *                    Sending messages to self is needed for a proper multi-account setup,
  *                    however, on the other hand, may lead to unwanted notifications in non-delta clients.
- * - `inbox_watch`  = 1=watch `INBOX`-folder for changes (default),
- *                    0=do not watch the `INBOX`-folder,
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `sentbox_watch`= 1=watch `Sent`-folder for changes (default),
  *                    0=do not watch the `Sent`-folder,
  *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
- * - `mvbox_watch`  = 1=watch `DeltaChat`-folder for changes (default),
- *                    0=do not watch the `DeltaChat`-folder,
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
- * - `mvbox_move`   = 1=heuristically detect chat-messages
- *                    and move them to the `DeltaChat`-folder,
+ * - `mvbox_move`   = 1=detect chat messages,
+ *                    move them to the `DeltaChat` folder,
+ *                    and watch the `DeltaChat` folder for updates (default),
  *                    0=do not move chat-messages
+ *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `show_emails`  = DC_SHOW_EMAILS_OFF (0)=
  *                    show direct replies to chats only (default),
  *                    DC_SHOW_EMAILS_ACCEPTED_CONTACTS (1)=
@@ -346,6 +348,19 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
  *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
+ * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
+ *                    For larger messages, only the header is downloaded and a placeholder is shown.
+ *                    These messages can be downloaded fully using dc_download_full_msg() later.
+ *                    The limit is compared against raw message sizes, including headers.
+ *                    The actually used limit may be corrected
+ *                    to not mess up with non-delivery-reports or read-receipts.
+ *                    0=no limit (default).
+ *                    Changes affect future messages only.
+ * - `ui.*`         = All keys prefixed by `ui.` can be used by the user-interfaces for system-specific purposes.
+ *                    The prefix should be followed by the system and maybe subsystem,
+ *                    eg. `ui.desktop.foo`, `ui.desktop.linux.bar`, `ui.android.foo`, `ui.dc40.bar`, `ui.bot.simplebot.baz`.
+ *                    These keys go to backups and allow easy per-account settings when using @ref dc_accounts_t,
+ *                    however, are not handled by the core otherwise.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -372,6 +387,9 @@ int             dc_set_config                (dc_context_t* context, const char*
  *                    an error (no warning as it should be shown to the user) is logged but the attachment is sent anyway.
  * - `sys.config_keys` = get a space-separated list of all config-keys available.
  *                    The config-keys are the keys that can be passed to the parameter `key` of this function.
+ * - `quota_exceeding` = 0: quota is unknown or in normal range;
+ *                    >=80: quota is about to exceed, the value is the concrete percentage,
+ *                    a device message is added when that happens, however, that value may still be interesting for bots.
  *
  * @memberof dc_context_t
  * @param context The context object. For querying system values, this can be NULL.
@@ -578,6 +596,14 @@ void            dc_configure                 (dc_context_t* context);
  * Typically, for unconfigured accounts, the user is prompted
  * to enter some settings and dc_configure() is called in a thread then.
  *
+ * A once successfully configured context cannot become unconfigured again;
+ * if a subsequent call to dc_configure() fails,
+ * the prior configuration is used.
+ *
+ * However, of course, also a configuration may stop working,
+ * as eg. the password was changed on the server.
+ * To check that use eg. dc_get_connectivity().
+ *
  * @memberof dc_context_t
  * @param context The context object.
  * @return 1=context is configured and can be used;
@@ -654,8 +680,8 @@ void            dc_maybe_network             (dc_context_t* context);
  * @param context The context as created by dc_context_new().
  * @param addr The email address of the user.  This must match the
  *    configured_addr setting of the context as well as the UID of the key.
- * @param public_data The public key as base64.
- * @param secret_data The secret key as base64.
+ * @param public_data ASCII armored public key.
+ * @param secret_data ASCII armored secret key.
  * @return 1 on success, 0 on failure.
  */
 int             dc_preconfigure_keypair        (dc_context_t* context, const char *addr, const char *public_data, const char *secret_data);
@@ -899,7 +925,7 @@ uint32_t        dc_send_text_msg             (dc_context_t* context, uint32_t ch
  * dc_send_videochat_invitation() is blocking and may take a while,
  * so the UIs will typically call the function from within a thread.
  * Moreover, UIs will typically enter the room directly without an additional click on the message,
- * for this purpose, the function returns the message-id directly.
+ * for this purpose, the function returns the message id directly.
  *
  * As for other messages sent, this function
  * sends the event #DC_EVENT_MSGS_CHANGED on success, the message has a delivery state, and so on.
@@ -937,8 +963,6 @@ uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
  * @param msg The message to save as a draft.
  *     Existing draft will be overwritten.
  *     NULL deletes the existing draft, if any, without sending it.
- *     Currently, also non-text-messages
- *     will delete the existing drafts.
  */
 void            dc_set_draft                 (dc_context_t* context, uint32_t chat_id, dc_msg_t* msg);
 
@@ -1270,6 +1294,8 @@ void            dc_accept_chat               (dc_context_t* context, uint32_t ch
  *   explicitly as it may happen that oneself gets removed from a still existing
  *   group
  *
+ * - for broadcasts, all recipients are returned, DC_CONTACT_ID_SELF is not included
+ *
  * - for mailing lists, the behavior is not documented currently, we will decide on that later.
  *   for now, the UI should not show the list for mailing lists.
  *   (we do not know all members and there is not always a global mailing list address,
@@ -1352,7 +1378,6 @@ dc_chat_t*      dc_get_chat                  (dc_context_t* context, uint32_t ch
  * Create a new group chat.
  *
  * After creation,
- * the draft of the chat is set to a default text,
  * the group has one member with the ID DC_CONTACT_ID_SELF
  * and is in _unpromoted_ state.
  * This means, you can add or remove members, change the name,
@@ -1379,6 +1404,36 @@ dc_chat_t*      dc_get_chat                  (dc_context_t* context, uint32_t ch
 uint32_t        dc_create_group_chat         (dc_context_t* context, int protect, const char* name);
 
 
+/**
+ * Create a new broadcast list.
+ *
+ * Broadcast lists are similar to groups on the sending device,
+ * however, recipients get the messages in normal one-to-one chats
+ * and will not be aware of other members.
+ *
+ * Replies to broadcasts go only to the sender
+ * and not to all broadcast recipients.
+ * Moreover, replies will not appear in the broadcast list
+ * but in the one-to-one chat with the person answering.
+ *
+ * The name and the image of the broadcast list is set automatically
+ * and is visible to the sender only.
+ * Not asking for these data allows more focused creation
+ * and we bypass the question who will get which data.
+ * Also, many users will have at most one broadcast list
+ * so, a generic name and image is sufficient at the first place.
+ *
+ * Later on, however, the name can be changed using dc_set_chat_name().
+ * The image cannot be changed to have a unique, recognizable icon in the chat lists.
+ * All in all, this is also what other messengers are doing here.
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @return The chat ID of the new broadcast list, 0 on errors.
+ */
+uint32_t        dc_create_broadcast_list     (dc_context_t* context);
+
+
 /**
  * Check if a given contact ID is a member of a group chat.
  *
@@ -1576,6 +1631,28 @@ char*           dc_get_msg_info              (dc_context_t* context, uint32_t ms
 char*           dc_get_msg_html              (dc_context_t* context, uint32_t msg_id);
 
 
+/**
+  * Asks the core to start downloading a message fully.
+  * This function is typically called when the user hits the "Download" button
+  * that is shown by the UI in case dc_msg_get_download_state()
+  * returns @ref DC_DOWNLOAD_AVAILABLE or @ref DC_DOWNLOAD_FAILURE.
+  *
+  * On success, the @ref DC_MSG "view type of the message" may change
+  * or the message may be replaced completely by one or more messages with other message ids.
+  * That may happen eg. in cases where the message was encrypted
+  * and the type could not be determined without fully downloading.
+  * Downloaded content can be accessed as usual after download,
+  * eg. using dc_msg_get_file().
+  *
+  * To reflect these changes a @ref DC_EVENT_MSGS_CHANGED event will be emitted.
+  *
+  * @memberof dc_context_t
+  * @param context The context object.
+  * @param msg_id Message ID to download the content for.
+  */
+void dc_download_full_msg (dc_context_t* context, int msg_id);
+
+
 /**
  * Get the raw mime-headers of the given message.
  * Raw headers are saved for incoming messages
@@ -1626,7 +1703,7 @@ void            dc_forward_msgs              (dc_context_t* context, const uint3
  *
  * - For normal chats, the IMAP state is updated, MDN is sent
  *   (if dc_set_config()-options `mdns_enabled` is set)
- *   and the internal state is changed to DC_STATE_IN_SEEN to reflect these actions.
+ *   and the internal state is changed to @ref DC_STATE_IN_SEEN to reflect these actions.
  *
  * - For contact requests, no IMAP or MDNs is done
  *   and the internal state is not changed therefore.
@@ -2047,7 +2124,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_FPR_MISMATCH           220 // id=contact
 #define         DC_QR_FPR_WITHOUT_ADDR       230 // test1=formatted fingerprint
 #define         DC_QR_ACCOUNT                250 // text1=domain
-#define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain
+#define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain, text2=instance pattern
 #define         DC_QR_ADDR                   320 // id=contact
 #define         DC_QR_TEXT                   330 // text1=text
 #define         DC_QR_URL                    332 // text1=URL
@@ -2160,27 +2237,32 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
 char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t chat_id);
 
 
+/**
+ * Get QR code image from the QR code text generated by dc_get_securejoin_qr().
+ * See dc_get_securejoin_qr() for details about the contained QR code.
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @param chat_id group-chat-id for secure-join or 0 for setup-contact,
+ *     see dc_get_securejoin_qr() for details.
+ * @return SVG-Image with the QR code.
+ *     On errors, an empty string is returned.
+ *     The returned string must be released using dc_str_unref() after usage.
+ */
+char*           dc_get_securejoin_qr_svg         (dc_context_t* context, uint32_t chat_id);
+
 /**
  * Continue a Setup-Contact or Verified-Group-Invite protocol
  * started on another device with dc_get_securejoin_qr().
  * This function is typically called when dc_check_qr() returns
  * lot.state=DC_QR_ASK_VERIFYCONTACT or lot.state=DC_QR_ASK_VERIFYGROUP.
  *
- * Depending on the given QR code,
- * this function may takes some time and sends and receives several messages.
- * Therefore, you should call it always in a separate thread;
- * if you want to abort it, you should call dc_stop_ongoing_process().
+ * The function returns immediately and the handshake runs in background,
+ * sending and receiving several messages.
+ * During the handshake, info messages are added to the chat,
+ * showing progress, success or errors.
  *
- * - If the given QR code starts the Setup-Contact protocol,
- *   the function typically returns immediately
- *   and the handshake runs in background.
- *   Subsequent calls of dc_join_securejoin() will abort unfinished tasks.
- *   The returned chat is the one-to-one opportunistic chat.
- *   When the protocol has finished, an info-message is added to that chat.
- * - If the given QR code starts the Verified-Group-Invite protocol,
- *   the function waits until the protocol has finished.
- *   This is because the protected group is not opportunistic
- *   and can be created only when the contacts have verified each other.
+ * Subsequent calls of dc_join_securejoin() will abort previous, unfinished handshakes.
  *
  * See https://countermitm.readthedocs.io/en/latest/new.html
  * for details about both protocols.
@@ -2190,10 +2272,8 @@ char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t ch
  * @param qr The text of the scanned QR code. Typically, the same string as given
  *     to dc_check_qr().
  * @return Chat-id of the joined chat, the UI may redirect to the this chat.
- *     If the out-of-band verification failed or was aborted, 0 is returned.
+ *     On errors, 0 is returned, however, most errors will happen during handshake later on.
  *     A returned chat-id does not guarantee that the chat is protected or the belonging contact is verified.
- *     If needed, this be checked with dc_chat_is_protected() and dc_contact_is_verified(),
- *     however, in practise, the UI will just listen to #DC_EVENT_CONTACTS_CHANGED unconditionally.
  */
 uint32_t        dc_join_securejoin           (dc_context_t* context, const char* qr);
 
@@ -2342,6 +2422,22 @@ dc_array_t* dc_get_locations                (dc_context_t* context, uint32_t cha
 void        dc_delete_all_locations         (dc_context_t* context);
 
 
+/**
+ * Get last error string.
+ *
+ * This is the same error string as logged via #DC_EVENT_ERROR,
+ * however, using this function avoids race conditions
+ * if the failing function is called in another thread than dc_get_next_event().
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @return Last error or an empty string if there is no last error.
+ *     NULL is never returned.
+ *     The returned value must be released using dc_str_unref() after usage.
+ */
+char* dc_get_last_error (dc_context_t* context);
+
+
 /**
  * Release a string returned by another deltachat-core function.
  * - Strings returned by any deltachat-core-function
@@ -2723,13 +2819,13 @@ uint32_t         dc_array_get_contact_id     (const dc_array_t* array, size_t in
 
 
 /**
- * Return the message-id of the item at the given index.
+ * Return the message id of the item at the given index.
  *
  * @memberof dc_array_t
  * @param array The array object.
  * @param index Index of the item. Must be between 0 and dc_array_get_cnt()-1.
- * @return Message-id of the item at the given index.
- *     0 if there is no message-id bound to the given item,
+ * @return Message id of the item at the given index.
+ *     0 if there is no message id bound to the given item,
  */
 uint32_t         dc_array_get_msg_id         (const dc_array_t* array, size_t index);
 
@@ -2776,6 +2872,45 @@ int              dc_array_is_independent     (const dc_array_t* array, size_t in
 int              dc_array_search_id          (const dc_array_t* array, uint32_t needle, size_t* ret_index);
 
 
+/**
+ * @class dc_blob_t
+ *
+ * An object representing a file blob.
+ */
+
+
+/**
+ * Free a blob object.
+ *
+ * @memberof dc_blob_t
+ * @param blob The blob object.
+ */
+void             dc_blob_unref               (dc_blob_t* blob);
+
+/**
+ * Returns blob size in bytes.
+ *
+ * @memberof dc_blob_t
+ * @param blob The blob object.
+ * @return Returns blob size. Never fails.
+ */
+
+size_t           dc_blob_bytes               (dc_blob_t* blob);
+
+/**
+ * Reads data from a blob.
+ * Reads exactly the number of bytes requested and
+ * returns an error if it's not possible.
+ *
+ * @memberof dc_blob_t
+ * @param blob The blob object.
+ * @param count Number of bytes to read from the blob.
+ * @param offset Offset of the first byte to read inside the blob.
+ * @return 1 on success, 0 on error.
+ */
+int              dc_blob_read                (dc_blob_t* blob, void* buf, size_t count, off_t offset);
+
+
 /**
  * @class dc_chatlist_t
  *
@@ -2875,7 +3010,7 @@ uint32_t         dc_chatlist_get_msg_id      (const dc_chatlist_t* chatlist, siz
  *
  * - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
  *
- * - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).  0 if not applicable.
+ * - dc_lot_t::state: The state of the message as one of the @ref DC_STATE constants.  0 if not applicable.
  *
  * @memberof dc_chatlist_t
  * @param chatlist The chatlist to query as returned e.g. from dc_get_chatlist().
@@ -2891,7 +3026,7 @@ dc_lot_t*        dc_chatlist_get_summary     (const dc_chatlist_t* chatlist, siz
  * Create a chatlist summary item when the chatlist object is already unref()'d.
  *
  * This function is similar to dc_chatlist_get_summary(), however,
- * takes the chat-id and message-id as returned by dc_chatlist_get_chat_id() and dc_chatlist_get_msg_id()
+ * takes the chat-id and message id as returned by dc_chatlist_get_chat_id() and dc_chatlist_get_msg_id()
  * as arguments. The chatlist object itself is not needed directly.
  *
  * This maybe useful if you convert the complete object into a different represenation
@@ -2929,7 +3064,7 @@ dc_context_t*    dc_chatlist_get_context     (dc_chatlist_t* chatlist);
  * color: color of this chat
  * last-message-from: who sent the last message
  * last-message-text: message (truncated)
- * last-message-state: DC_STATE* constant
+ * last-message-state: @ref DC_STATE constant
  * last-message-date:
  * avatar-path: path-to-blobfile
  * is_verified: yes/no
@@ -2953,12 +3088,6 @@ char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat
 #define         DC_CHAT_ID_LAST_SPECIAL      9 // larger chat IDs are "real" chats, their messages are "real" messages.
 
 
-#define         DC_CHAT_TYPE_UNDEFINED       0
-#define         DC_CHAT_TYPE_SINGLE          100
-#define         DC_CHAT_TYPE_GROUP           120
-#define         DC_CHAT_TYPE_MAILINGLIST     140
-
-
 /**
  * Free a chat object.
  *
@@ -2985,21 +3114,30 @@ uint32_t        dc_chat_get_id               (const dc_chat_t* chat);
 
 
 /**
- * Get chat type.
+ * Get chat type as one of the @ref DC_CHAT_TYPE constants:
  *
- * Currently, there are two chat types:
- *
- * - DC_CHAT_TYPE_SINGLE (100) - a normal chat is a chat with a single contact,
+ * - @ref DC_CHAT_TYPE_SINGLE - a normal chat is a chat with a single contact,
  *   chats_contacts contains one record for the user.  DC_CONTACT_ID_SELF
  *   (see dc_contact_t::id) is added _only_ for a self talk.
+ *   These chats are created by dc_create_chat_by_contact_id().
  *
- * - DC_CHAT_TYPE_GROUP  (120) - a group chat, chats_contacts contain all group
- *   members, incl. DC_CONTACT_ID_SELF
+ * - @ref DC_CHAT_TYPE_GROUP - a group chat, chats_contacts contain all group
+ *   members, incl. DC_CONTACT_ID_SELF.
+ *   Groups are created by dc_create_group_chat().
  *
- * - DC_CHAT_TYPE_MAILINGLIST (140) - a mailing list, this is similar to groups,
+ * - @ref DC_CHAT_TYPE_MAILINGLIST - a mailing list, this is similar to groups,
  *   however, the member list cannot be retrieved completely
  *   and cannot be changed using this api.
- *   moreover, for now, mailist lists are read-only.
+ *   Mailing lists are created as needed by incoming messages
+ *   and usually require some special server;
+ *   they cannot be created by a function call as the other chat types.
+ *   Moreover, for now, mailing lists are read-only.
+ *
+ * - @ref DC_CHAT_TYPE_BROADCAST - a broadcast list,
+ *   the recipients will get messages in a one-to-one chats and
+ *   the sender will get answers in a one-to-one as well.
+ *   chats_contacts contain all recipients but DC_CONTACT_ID_SELF.
+ *   Broadcasts are created by dc_create_broadcast_list().
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -3208,22 +3346,6 @@ int64_t          dc_chat_get_remaining_mute_duration (const dc_chat_t* chat);
 #define         DC_MSG_ID_LAST_SPECIAL       9
 
 
-#define         DC_STATE_UNDEFINED           0
-#define         DC_STATE_IN_FRESH            10
-#define         DC_STATE_IN_NOTICED          13
-#define         DC_STATE_IN_SEEN             16
-#define         DC_STATE_OUT_PREPARING       18
-#define         DC_STATE_OUT_DRAFT           19
-#define         DC_STATE_OUT_PENDING         20
-#define         DC_STATE_OUT_FAILED          24
-#define         DC_STATE_OUT_DELIVERED       26 // to check if a mail was sent, use dc_msg_is_sent()
-#define         DC_STATE_OUT_MDN_RCVD        28
-
-
-#define         DC_MAX_GET_TEXT_LEN          30000 // approx. max. length returned by dc_msg_get_text()
-#define         DC_MAX_GET_INFO_LEN          100000 // approx. max. length returned by dc_get_msg_info()
-
-
 /**
  * Create new message object. Message objects are needed e.g. for sending messages using
  * dc_send_msg().  Moreover, they are returned e.g. from dc_get_msg(),
@@ -3303,28 +3425,37 @@ int             dc_msg_get_viewtype           (const dc_msg_t* msg);
  * Get the state of a message.
  *
  * Incoming message states:
- * - DC_STATE_IN_FRESH (10) - Incoming _fresh_ message. Fresh messages are neither noticed nor seen and are typically shown in notifications. Use dc_get_fresh_msgs() to get all fresh messages.
- * - DC_STATE_IN_NOTICED (13) - Incoming _noticed_ message. E.g. chat opened but message not yet read - noticed messages are not counted as unread but were not marked as read nor resulted in MDNs. Use dc_marknoticed_chat() to mark messages as being noticed.
- * - DC_STATE_IN_SEEN (16) - Incoming message, really _seen_ by the user. Marked as read on IMAP and MDN may be sent. Use dc_markseen_msgs() to mark messages as being seen.
+ * - @ref DC_STATE_IN_FRESH - Incoming _fresh_ message.
+ *   Fresh messages are neither noticed nor seen and are typically shown in notifications.
+ *   Use dc_get_fresh_msgs() to get all fresh messages.
+ * - @ref DC_STATE_IN_NOTICED - Incoming _noticed_ message.
+ *   E.g. chat opened but message not yet read.
+ *   Noticed messages are not counted as unread but were not marked as read nor resulted in MDNs.
+ *   Use dc_marknoticed_chat() to mark messages as being noticed.
+ * - @ref DC_STATE_IN_SEEN - Incoming message, really _seen_ by the user.
+ *   Marked as read on IMAP and MDN may be sent. Use dc_markseen_msgs() to mark messages as being seen.
  *
  * Outgoing message states:
- * - DC_STATE_OUT_PREPARING (18) - For files which need time to be prepared before they can be sent,
- *   the message enters this state before DC_STATE_OUT_PENDING.
- * - DC_STATE_OUT_DRAFT (19) - Message saved as draft using dc_set_draft()
- * - DC_STATE_OUT_PENDING (20) - The user has pressed the "send" button but the
+ * - @ref DC_STATE_OUT_PREPARING - For files which need time to be prepared before they can be sent,
+ *   the message enters this state before @ref DC_STATE_OUT_PENDING.
+ * - @ref DC_STATE_OUT_DRAFT - Message saved as draft using dc_set_draft()
+ * - @ref DC_STATE_OUT_PENDING - The user has pressed the "send" button but the
  *   message is not yet sent and is pending in some way. Maybe we're offline (no checkmark).
- * - DC_STATE_OUT_FAILED (24) - _Unrecoverable_ error (_recoverable_ errors result in pending messages), you'll receive the event #DC_EVENT_MSG_FAILED.
- * - DC_STATE_OUT_DELIVERED (26) - Outgoing message successfully delivered to server (one checkmark). Note, that already delivered messages may get into the state DC_STATE_OUT_FAILED if we get such a hint from the server.
+ * - @ref DC_STATE_OUT_FAILED - _Unrecoverable_ error (_recoverable_ errors result in pending messages),
+ *   you'll receive the event #DC_EVENT_MSG_FAILED.
+ * - @ref DC_STATE_OUT_DELIVERED - Outgoing message successfully delivered to server (one checkmark).
+ *   Note, that already delivered messages may get into the state @ref DC_STATE_OUT_FAILED if we get such a hint from the server.
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_DELIVERED.
- * - DC_STATE_OUT_MDN_RCVD (28) - Outgoing message read by the recipient (two checkmarks; this requires goodwill on the receiver's side)
+ * - @ref DC_STATE_OUT_MDN_RCVD - Outgoing message read by the recipient
+ *   (two checkmarks; this requires goodwill on the receiver's side)
  *   If a sent message changes to this state, you'll receive the event #DC_EVENT_MSG_READ.
  *   Also messages already read by some recipients
- *   may get into the state DC_STATE_OUT_FAILED at a later point,
+ *   may get into the state @ref DC_STATE_OUT_FAILED at a later point,
  *   e.g. when in a group, delivery fails for some recipients.
  *
  * If you just want to check if a message is sent or not, please use dc_msg_is_sent() which regards all states accordingly.
  *
- * The state of just created message objects is DC_STATE_UNDEFINED (0).
+ * The state of just created message objects is @ref DC_STATE_UNDEFINED.
  * The state is always set by the core-library, users of the library cannot set the state directly, but it is changed implicitly e.g.
  * when calling  dc_marknoticed_chat() or dc_markseen_msgs().
  *
@@ -3480,6 +3611,16 @@ char*           dc_msg_get_filemime           (const dc_msg_t* msg);
 uint64_t        dc_msg_get_filebytes          (const dc_msg_t* msg);
 
 
+/**
+ * Get the file blob.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return File blob, NULL if not applicable or on errors.
+ */
+dc_blob_t      *dc_msg_get_fileblob           (const dc_msg_t* msg);
+
+
 /**
  * Get width of image or video.  The width is returned in pixels.
  * If the width is unknown or if the associated file is no image or video file,
@@ -3588,7 +3729,7 @@ int64_t          dc_msg_get_ephemeral_timestamp (const dc_msg_t* msg);
  *   Typically used to show dc_lot_t::text1 with different colors. 0 if not applicable.
  * - dc_lot_t::text2: contains an excerpt of the message text.
  * - dc_lot_t::timestamp: the timestamp of the message.
- * - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
+ * - dc_lot_t::state: The state of the message as one of the @ref DC_STATE constants.
  *
  * Typically used to display a search result. See also dc_chatlist_get_summary() to display a list of chats.
  *
@@ -3902,6 +4043,31 @@ int dc_msg_get_videochat_type (const dc_msg_t* msg);
 int dc_msg_has_html (dc_msg_t* msg);
 
 
+/**
+  * Check if the message is completely downloaded
+  * or if some further action is needed.
+  *
+  * Messages may be not fully downloaded
+  * if they are larger than the limit set by the dc_set_config()-option `download_limit`.
+  *
+  * The function returns one of:
+  * - @ref DC_DOWNLOAD_DONE        - The message does not need any further download action
+  *                                  and should be rendered as usual.
+  * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
+  *                                  In addition to the usual message rendering,
+  *                                  the UI shall show a download button that calls dc_download_full_msg()
+  * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_download_full_msg() and is still in progress.
+  *                                  If the download fails or succeeds,
+  *                                  the event @ref DC_EVENT_MSGS_CHANGED is emitted.
+  * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_download_full_msg() again.
+  *
+  * @memberof dc_msg_t
+  * @param msg The message object.
+  * @return One of the @ref DC_DOWNLOAD values
+  */
+int dc_msg_get_download_state (const dc_msg_t* msg);
+
+
 /**
  * Set the text of a message object.
  * This does not alter any information in the database; this may be done by dc_send_msg() later.
@@ -4094,6 +4260,15 @@ char*           dc_msg_get_quoted_text        (const dc_msg_t* msg);
  */
 dc_msg_t*       dc_msg_get_quoted_msg         (const dc_msg_t* msg);
 
+/**
+ * Force the message to be sent in plain text.
+ *
+ * This API is for bots, there is no need to expose it in the UI.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ */
+void            dc_msg_force_plaintext        (dc_msg_t* msg);
 
 /**
  * @class dc_contact_t
@@ -4273,6 +4448,16 @@ uint32_t        dc_contact_get_color         (const dc_contact_t* contact);
  */
 char*           dc_contact_get_status        (const dc_contact_t* contact);
 
+/**
+ * Get the contact's last seen timestamp.
+ *
+ * @memberof dc_contact_t
+ * @param contact The contact object.
+ * @return Last seen timestamp.
+ *     0 on error or if the contact was never seen.
+ */
+int64_t         dc_contact_get_last_seen     (const dc_contact_t* contact);
+
 /**
  * Check if a contact is blocked.
  *
@@ -4584,6 +4769,115 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 
+/**
+  * @defgroup DC_STATE DC_STATE
+  *
+  * These constants describe the state of a message.
+  * The state can be retrieved using dc_msg_get_state()
+  * and may change by various actions reported by various events
+  *
+  * @addtogroup DC_STATE
+  * @{
+  */
+
+/**
+ * Message just created. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_UNDEFINED           0
+
+/**
+ * Incoming fresh message. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_IN_FRESH            10
+
+/**
+ * Incoming noticed message. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_IN_NOTICED          13
+
+/**
+ * Incoming seen message. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_IN_SEEN             16
+
+/**
+ * Outgoing message being prepared. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_PREPARING       18
+
+/**
+ * Outgoing message drafted. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_DRAFT           19
+
+/**
+ * Outgoing message waiting to be sent. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_PENDING         20
+
+/**
+ * Outgoing message failed sending. See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_FAILED          24
+
+/**
+ * Outgoing message sent. To check if a mail was actually sent, use dc_msg_is_sent().
+ * See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_DELIVERED       26
+
+/**
+ * Outgoing message sent and seen by recipients(s). See dc_msg_get_state() for details.
+ */
+#define         DC_STATE_OUT_MDN_RCVD        28
+
+/**
+ * @}
+ */
+
+
+/**
+ * @defgroup DC_CHAT_TYPE DC_CHAT_TYPE
+ *
+ * These constants describe the type of a chat.
+ * The chat type can be retrieved using dc_chat_get_type()
+ * and the type does not change during the chat's lifetime.
+ *
+ * @addtogroup DC_CHAT_TYPE
+ * @{
+ */
+
+/**
+ * Undefined chat type.
+ * Normally, this type is not returned.
+ */
+#define         DC_CHAT_TYPE_UNDEFINED       0
+
+/**
+ * A one-to-one chat with a single contact. See dc_chat_get_type() for details.
+ */
+#define         DC_CHAT_TYPE_SINGLE          100
+
+/**
+ * A group chat. See dc_chat_get_type() for details.
+ */
+#define         DC_CHAT_TYPE_GROUP           120
+
+/**
+ * A mailing list. See dc_chat_get_type() for details.
+ */
+#define         DC_CHAT_TYPE_MAILINGLIST     140
+
+/**
+ * A broadcast list. See dc_chat_get_type() for details.
+ */
+#define         DC_CHAT_TYPE_BROADCAST       160
+
+/**
+ * @}
+ */
+
+
 /**
  * @defgroup DC_SOCKET DC_SOCKET
  *
@@ -4828,7 +5122,7 @@ char* dc_event_get_data2_str(dc_event_t* event);
  *
  * @memberof dc_event_t
  * @param event Event object as returned from dc_accounts_get_next_event().
- * @return account-id belonging to the event or 0 for errors.
+ * @return account-id belonging to the event, 0 for account manager errors.
  */
 uint32_t dc_event_get_account_id(dc_event_t* event);
 
@@ -4976,8 +5270,8 @@ void dc_event_unref(dc_event_t* event);
  * - Chats created, deleted or archived
  * - A draft has been set
  *
- * @param data1 (int) chat_id for single added messages
- * @param data2 (int) msg_id for single added messages
+ * @param data1 (int) chat_id if only a single chat is affected by the changes, otherwise 0
+ * @param data2 (int) msg_id if only a single message is affected by the changes, otherwise 0
  */
 #define DC_EVENT_MSGS_CHANGED             2000
 
@@ -5010,8 +5304,8 @@ void dc_event_unref(dc_event_t* event);
 
 
 /**
- * A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
- * DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
+ * A single message is sent successfully. State changed from @ref DC_STATE_OUT_PENDING to
+ * @ref DC_STATE_OUT_DELIVERED.
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
@@ -5021,8 +5315,8 @@ void dc_event_unref(dc_event_t* event);
 
 /**
  * A single message could not be sent.
- * State changed from DC_STATE_OUT_PENDING, DC_STATE_OUT_DELIVERED or DC_STATE_OUT_MDN_RCVD
- * to DC_STATE_OUT_FAILED, see dc_msg_get_state().
+ * State changed from @ref DC_STATE_OUT_PENDING, @ref DC_STATE_OUT_DELIVERED or @ref DC_STATE_OUT_MDN_RCVD
+ * to @ref DC_STATE_OUT_FAILED.
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
@@ -5031,8 +5325,8 @@ void dc_event_unref(dc_event_t* event);
 
 
 /**
- * A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
- * DC_STATE_OUT_MDN_RCVD, see dc_msg_get_state().
+ * A single message is read by the receiver. State changed from @ref DC_STATE_OUT_DELIVERED to
+ * @ref DC_STATE_OUT_MDN_RCVD.
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
@@ -5053,6 +5347,9 @@ void dc_event_unref(dc_event_t* event);
 
 /**
  * Chat ephemeral timer changed.
+ *
+ * @param data1 (int) chat_id
+ * @param data2 (int) Timer value in seconds or 0 for disabled timer
  */
 #define DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED 2021
 
@@ -5152,6 +5449,14 @@ void dc_event_unref(dc_event_t* event);
  */
 #define DC_EVENT_CONNECTIVITY_CHANGED             2100
 
+
+/**
+ * The user's avatar changed.
+ * You can get the new avatar file with `dc_get_config(context, "selfavatar")`.
+ */
+#define DC_EVENT_SELFAVATAR_CHANGED               2110
+
+
 /**
  * @}
  */
@@ -5276,6 +5581,44 @@ void dc_event_unref(dc_event_t* event);
  */
 #define         DC_CHAT_VISIBILITY_PINNED      2
 
+/**
+ * @}
+ */
+
+
+/**
+  * @defgroup DC_DOWNLOAD DC_DOWNLOAD
+  *
+  * These constants describe the download state of a message.
+  * The download state can be retrieved using dc_msg_get_download_state()
+  * and usually changes after calling dc_download_full_msg().
+  *
+  * @addtogroup DC_DOWNLOAD
+  * @{
+  */
+
+/**
+ * Download not needed, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_DONE          0
+
+/**
+ * Download available, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_AVAILABLE    10
+
+/**
+ * Download failed, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_FAILURE      20
+
+/**
+ * Download in progress, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_IN_PROGRESS  1000
+
+
+
 /**
  * @}
  */
@@ -5346,12 +5689,6 @@ void dc_event_unref(dc_event_t* event);
 /// if nothing else is set by the dc_set_config()-option `selfstatus`.
 #define DC_STR_STATUSLINE                 13
 
-/// "Hi, i've created the group %1$s for us."
-///
-/// Used as a draft text after group creation.
-/// - %1$s will be replaced by the group name
-#define DC_STR_NEWGROUPDRAFT              14
-
 /// "Group name changed from %1$s to %2$s."
 ///
 /// Used in status messages for group name changes.
@@ -5674,6 +6011,152 @@ void dc_event_unref(dc_event_t* event);
 /// Used in message summary text for notifications and chatlist.
 #define DC_STR_FORWARDED                  97
 
+/// "Quota exceeding, already %1$s%% used."
+///
+/// Used as device message text.
+///
+/// `%1$s` will be replaced by the percentage used
+#define DC_STR_QUOTA_EXCEEDING_MSG_BODY   98
+
+/// "%1$s message"
+///
+/// Used as the message body when a message
+/// was not yet downloaded completely
+/// (dc_msg_get_download_state() is eg. @ref DC_DOWNLOAD_AVAILABLE).
+///
+/// `%1$s` will be replaced by human-readable size (eg. "1.2 MiB").
+#define DC_STR_PARTIAL_DOWNLOAD_MSG_BODY  99
+
+/// "Download maximum available until %1$s"
+///
+/// Appended after some separator to @ref DC_STR_PARTIAL_DOWNLOAD_MSG_BODY.
+///
+/// `%1$s` will be replaced by human-readable date and time.
+#define DC_STR_DOWNLOAD_AVAILABILITY      100
+
+/// "Multi Device Synchronization"
+///
+/// Used in subjects of outgoing sync messages.
+#define DC_STR_SYNC_MSG_SUBJECT           101
+
+/// "This message is used to synchronize data between your devices."
+///
+///
+/// Used as message text of outgoing sync messages.
+/// The text is visible in non-dc-muas or in outdated Delta Chat versions,
+/// the default text therefore adds the following hint:
+/// "If you see this message in Delta Chat,
+/// please update your Delta Chat apps on all devices."
+#define DC_STR_SYNC_MSG_BODY              102
+
+/// "Incoming Messages"
+///
+/// Used as a headline in the connectivity view.
+#define DC_STR_INCOMING_MESSAGES          103
+
+/// "Outgoing Messages"
+///
+/// Used as a headline in the connectivity view.
+#define DC_STR_OUTGOING_MESSAGES          104
+
+/// "Storage on %1$s"
+///
+/// Used as a headline in the connectivity view.
+///
+/// `%1$s` will be replaced by the domain of the configured email-address.
+#define DC_STR_STORAGE_ON_DOMAIN          105
+
+/// "One moment…"
+///
+/// Used in the connectivity view when some information are not yet there.
+#define DC_STR_ONE_MOMENT                 106
+
+/// "Connected"
+///
+/// Used as status in the connectivity view.
+#define DC_STR_CONNECTED                  107
+
+/// "Connecting…"
+///
+/// Used as status in the connectivity view.
+#define DC_STR_CONNTECTING                108
+
+/// "Updating…"
+///
+/// Used as status in the connectivity view.
+#define DC_STR_UPDATING                   109
+
+/// "Sending…"
+///
+/// Used as status in the connectivity view.
+#define DC_STR_SENDING                    110
+
+/// "Your last message was sent successfully."
+///
+/// Used as status in the connectivity view.
+#define DC_STR_LAST_MSG_SENT_SUCCESSFULLY 111
+
+/// "Error: %1$s"
+///
+/// Used as status in the connectivity view.
+///
+/// `%1$s` will be replaced by a possibly more detailed, typically english, error description.
+#define DC_STR_ERROR                      112
+
+/// "Not supported by your provider."
+///
+/// Used in the connectivity view.
+#define DC_STR_NOT_SUPPORTED_BY_PROVIDER  113
+
+/// "Messages"
+///
+/// Used as a subtitle in quota context; can be plural always.
+#define DC_STR_MESSAGES                   114
+
+/// "Broadcast List"
+///
+/// Used as the default name for broadcast lists; a number may be added.
+#define DC_STR_BROADCAST_LIST             115
+
+/// "%1$s of %2$s used"
+///
+/// Used for describing resource usage, resulting string will be eg. "1.2 GiB of 3 GiB used".
+#define DC_STR_PART_OF_TOTAL_USED         116
+
+/// "%1$s invited you to join this group. Waiting for the device of %2$s to reply…"
+///
+/// Added as an info-message directly after scanning a QR code for joining a group.
+/// May be followed by the info-messages
+/// #DC_STR_SECURE_JOIN_REPLIES, #DC_STR_CONTACT_VERIFIED and #DC_STR_MSGADDMEMBER.
+///
+/// `%1$s` will be replaced by name and address of the inviter,
+/// `%2$s` will be replaced by the name of the inviter.
+#define DC_STR_SECURE_JOIN_STARTED        117
+
+/// "%1$s replied, waiting for being added to the group…"
+///
+/// Info-message on scanning a QR code for joining a group.
+/// Added after #DC_STR_SECURE_JOIN_STARTED.
+/// If the handshake allows to skip a step and go for #DC_STR_CONTACT_VERIFIED directly,
+/// this info-message is skipped.
+///
+/// `%1$s` will be replaced by the name of the inviter.
+#define DC_STR_SECURE_JOIN_REPLIES        118
+
+/// "Scan to chat with %1$s"
+///
+/// Subtitle for verification qrcode svg image generated by the core.
+///
+/// `%1$s` will be replaced by name and address of the inviter.
+#define DC_STR_SETUP_CONTACT_QR_DESC      119
+
+/// "Scan to join %1$s"
+///
+/// Subtitle for group join qrcode svg image generated by the core.
+///
+/// `%1$s` will be replaced with the group name.
+#define DC_STR_SECURE_JOIN_GROUP_QR_DESC  120
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -15,14 +15,17 @@ extern crate num_traits;
 extern crate serde_json;
 
 use std::collections::BTreeMap;
-use std::convert::TryInto;
+use std::convert::TryFrom;
 use std::fmt::Write;
+use std::ops::Deref;
 use std::ptr;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use anyhow::Context as _;
+use async_std::sync::RwLock;
 use async_std::task::{block_on, spawn};
+use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use num_traits::{FromPrimitive, ToPrimitive};
 
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
@@ -37,6 +40,7 @@ use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
 
 mod dc_array;
+mod lot;
 
 mod string;
 use self::string::*;
@@ -59,7 +63,7 @@ pub type dc_context_t = Context;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_context_new(
-    os_name: *const libc::c_char,
+    _os_name: *const libc::c_char,
     dbfile: *const libc::c_char,
     blobdir: *const libc::c_char,
 ) -> *mut dc_context_t {
@@ -70,21 +74,11 @@ pub unsafe extern "C" fn dc_context_new(
         return ptr::null_mut();
     }
 
-    let os_name = if os_name.is_null() {
-        String::from("DcFFI")
-    } else {
-        to_string_lossy(os_name)
-    };
-
     let ctx = if blobdir.is_null() || *blobdir == 0 {
         use rand::Rng;
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
-        block_on(Context::new(
-            os_name,
-            as_path(dbfile).to_path_buf().into(),
-            id,
-        ))
+        block_on(Context::new(as_path(dbfile).to_path_buf().into(), id))
     } else {
         eprintln!("blobdir can not be defined explicitly anymore");
         return ptr::null_mut();
@@ -132,20 +126,30 @@ pub unsafe extern "C" fn dc_set_config(
     }
     let ctx = &*context;
     let key = to_string_lossy(key);
-    match config::Config::from_str(&key) {
-        Ok(key) => block_on(async move {
-            let value = to_opt_string_lossy(value);
-            ctx.set_config(key, value.as_deref())
+    let value = to_opt_string_lossy(value);
+
+    block_on(async move {
+        if key.starts_with("ui.") {
+            ctx.set_ui_config(&key, value.as_deref())
                 .await
                 .with_context(|| format!("Can't set {} to {:?}", key, value))
                 .log_err(ctx, "dc_set_config() failed")
                 .is_ok() as libc::c_int
-        }),
-        Err(_) => {
-            warn!(ctx, "dc_set_config(): invalid key");
-            0
+        } else {
+            match config::Config::from_str(&key) {
+                Ok(key) => ctx
+                    .set_config(key, value.as_deref())
+                    .await
+                    .with_context(|| format!("Can't set {} to {:?}", key, value))
+                    .log_err(ctx, "dc_set_config() failed")
+                    .is_ok() as libc::c_int,
+                Err(_) => {
+                    warn!(ctx, "dc_set_config(): invalid key");
+                    0
+                }
+            }
         }
-    }
+    })
 }
 
 #[no_mangle]
@@ -158,20 +162,33 @@ pub unsafe extern "C" fn dc_get_config(
         return "".strdup();
     }
     let ctx = &*context;
-    match config::Config::from_str(&to_string_lossy(key)) {
-        Ok(key) => block_on(async move {
-            ctx.get_config(key)
+
+    let key = to_string_lossy(key);
+
+    block_on(async move {
+        if key.starts_with("ui.") {
+            ctx.get_ui_config(&key)
                 .await
-                .log_err(ctx, "Can't get config")
+                .log_err(ctx, "Can't get ui-config")
                 .unwrap_or_default()
                 .unwrap_or_default()
                 .strdup()
-        }),
-        Err(_) => {
-            warn!(ctx, "dc_get_config(): invalid key");
-            "".strdup()
+        } else {
+            match config::Config::from_str(&key) {
+                Ok(key) => ctx
+                    .get_config(key)
+                    .await
+                    .log_err(ctx, "Can't get config")
+                    .unwrap_or_default()
+                    .unwrap_or_default()
+                    .strdup(),
+                Err(_) => {
+                    warn!(ctx, "dc_get_config(): invalid key '{}'", &key);
+                    "".strdup()
+                }
+            }
         }
-    }
+    })
 }
 
 #[no_mangle]
@@ -275,7 +292,15 @@ pub unsafe extern "C" fn dc_get_connectivity_html(
         return "".strdup();
     }
     let ctx = &*context;
-    block_on(async move { ctx.get_connectivity_html().await.strdup() })
+    block_on(async move {
+        match ctx.get_connectivity_html().await {
+            Ok(html) => html.strdup(),
+            Err(err) => {
+                error!(ctx, "Failed to get connectivity html: {}", err);
+                "".strdup()
+            }
+        }
+    })
 }
 
 #[no_mangle]
@@ -303,9 +328,12 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
     let redirect = to_string_lossy(redirect);
 
     block_on(async move {
-        match oauth2::dc_get_oauth2_url(ctx, &addr, &redirect).await {
-            Some(res) => res.strdup(),
-            None => ptr::null_mut(),
+        match oauth2::dc_get_oauth2_url(ctx, &addr, &redirect)
+            .await
+            .log_err(ctx, "dc_get_oauth2_url failed")
+        {
+            Ok(Some(res)) => res.strdup(),
+            Ok(None) | Err(_) => ptr::null_mut(),
         }
     })
 }
@@ -401,6 +429,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::Warning(_)
         | EventType::Error(_)
         | EventType::ConnectivityChanged
+        | EventType::SelfavatarChanged
         | EventType::ErrorSelfNotInGroup(_) => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -451,6 +480,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ImexFileWritten(_)
         | EventType::MsgsNoticed(_)
         | EventType::ConnectivityChanged
+        | EventType::SelfavatarChanged
         | EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
@@ -500,6 +530,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SecurejoinInviterProgress { .. }
         | EventType::SecurejoinJoinerProgress { .. }
         | EventType::ConnectivityChanged
+        | EventType::SelfavatarChanged
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -601,8 +632,8 @@ pub unsafe extern "C" fn dc_preconfigure_keypair(
     let ctx = &*context;
     block_on(async move {
         let addr = dc_tools::EmailAddress::new(&to_string_lossy(addr))?;
-        let public = key::SignedPublicKey::from_base64(&to_string_lossy(public_data))?;
-        let secret = key::SignedSecretKey::from_base64(&to_string_lossy(secret_data))?;
+        let public = key::SignedPublicKey::from_asc(&to_string_lossy(public_data))?.0;
+        let secret = key::SignedSecretKey::from_asc(&to_string_lossy(secret_data))?.0;
         let keypair = key::KeyPair {
             addr,
             public,
@@ -632,7 +663,10 @@ pub unsafe extern "C" fn dc_get_chatlist(
     let qi = if query_id == 0 { None } else { Some(query_id) };
 
     block_on(async move {
-        match chatlist::Chatlist::try_load(ctx, flags as usize, qs.as_deref(), qi).await {
+        match chatlist::Chatlist::try_load(ctx, flags as usize, qs.as_deref(), qi)
+            .await
+            .log_err(ctx, "Failed to get chatlist")
+        {
             Ok(list) => {
                 let ffi_list = ChatlistWrapper { context, list };
                 Box::into_raw(Box::new(ffi_list))
@@ -1291,6 +1325,19 @@ pub unsafe extern "C" fn dc_create_group_chat(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_create_broadcast_list(context: *mut dc_context_t) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_create_broadcast_list()");
+        return 0;
+    }
+    let ctx = &*context;
+    block_on(chat::create_broadcast_list(ctx))
+        .log_err(ctx, "Failed to create broadcast list")
+        .map(|id| id.to_u32())
+        .unwrap_or(0)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_is_contact_in_chat(
     context: *mut dc_context_t,
@@ -1303,8 +1350,13 @@ pub unsafe extern "C" fn dc_is_contact_in_chat(
     }
     let ctx = &*context;
 
-    block_on(async move { chat::is_contact_in_chat(ctx, ChatId::new(chat_id), contact_id).await })
-        .into()
+    block_on(chat::is_contact_in_chat(
+        ctx,
+        ChatId::new(chat_id),
+        contact_id,
+    ))
+    .log_err(ctx, "is_contact_in_chat failed")
+    .unwrap_or_default() as libc::c_int
 }
 
 #[no_mangle]
@@ -1319,9 +1371,13 @@ pub unsafe extern "C" fn dc_add_contact_to_chat(
     }
     let ctx = &*context;
 
-    block_on(async move {
-        chat::add_contact_to_chat(ctx, ChatId::new(chat_id), contact_id).await as libc::c_int
-    })
+    block_on(chat::add_contact_to_chat(
+        ctx,
+        ChatId::new(chat_id),
+        contact_id,
+    ))
+    .log_err(ctx, "Failed to add contact")
+    .is_ok() as libc::c_int
 }
 
 #[no_mangle]
@@ -1336,12 +1392,13 @@ pub unsafe extern "C" fn dc_remove_contact_from_chat(
     }
     let ctx = &*context;
 
-    block_on(async move {
-        chat::remove_contact_from_chat(ctx, ChatId::new(chat_id), contact_id)
-            .await
-            .map(|_| 1)
-            .unwrap_or_log_default(ctx, "Failed to remove contact")
-    })
+    block_on(chat::remove_contact_from_chat(
+        ctx,
+        ChatId::new(chat_id),
+        contact_id,
+    ))
+    .log_err(ctx, "Failed to remove contact")
+    .is_ok() as libc::c_int
 }
 
 #[no_mangle]
@@ -1556,6 +1613,8 @@ pub unsafe extern "C" fn dc_delete_msgs(
     let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
 
     block_on(message::delete_msgs(ctx, &msg_ids))
+        .log_err(ctx, "failed dc_delete_msgs() call")
+        .ok();
 }
 
 #[no_mangle]
@@ -1634,6 +1693,18 @@ pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) ->
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_download_full_msg(context: *mut dc_context_t, msg_id: u32) {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_download_full_msg()");
+        return;
+    }
+    let ctx = &*context;
+    block_on(MsgId::new(msg_id).download_full(ctx))
+        .log_err(ctx, "Failed to download message fully.")
+        .ok();
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_may_be_valid_addr(addr: *const libc::c_char) -> libc::c_int {
     if addr.is_null() {
@@ -1656,7 +1727,7 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     let ctx = &*context;
 
     block_on(async move {
-        Contact::lookup_id_by_addr(ctx, to_string_lossy(addr), Origin::IncomingReplyTo)
+        Contact::lookup_id_by_addr(ctx, &to_string_lossy(addr), Origin::IncomingReplyTo)
             .await
             .unwrap_or_log_default(ctx, "failed to lookup id")
             .unwrap_or(0)
@@ -1967,10 +2038,11 @@ pub unsafe extern "C" fn dc_check_qr(
     }
     let ctx = &*context;
 
-    block_on(async move {
-        let lot = qr::check_qr(ctx, &to_string_lossy(qr)).await;
-        Box::into_raw(Box::new(lot))
-    })
+    let lot = match block_on(qr::check_qr(ctx, &to_string_lossy(qr))) {
+        Ok(qr) => qr.into(),
+        Err(err) => err.into(),
+    };
+    Box::into_raw(Box::new(lot))
 }
 
 #[no_mangle]
@@ -1989,12 +2061,30 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
         Some(ChatId::new(chat_id))
     };
 
-    block_on(async move {
-        securejoin::dc_get_securejoin_qr(ctx, chat_id)
-            .await
-            .unwrap_or_else(|| "".to_string())
-            .strdup()
-    })
+    block_on(securejoin::dc_get_securejoin_qr(ctx, chat_id))
+        .unwrap_or_else(|_| "".to_string())
+        .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_securejoin_qr_svg(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to generate_verification_qr()");
+        return "".strdup();
+    }
+    let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
+
+    block_on(get_securejoin_qr_svg(ctx, chat_id))
+        .unwrap_or_else(|_| "".to_string())
+        .strdup()
 }
 
 #[no_mangle]
@@ -2033,7 +2123,9 @@ pub unsafe extern "C" fn dc_send_locations_to_chat(
         ctx,
         ChatId::new(chat_id),
         seconds as i64,
-    ));
+    ))
+    .log_err(ctx, "Failed dc_send_locations_to_chat()")
+    .ok();
 }
 
 #[no_mangle]
@@ -2052,7 +2144,8 @@ pub unsafe extern "C" fn dc_is_sending_locations_to_chat(
         Some(ChatId::new(chat_id))
     };
 
-    block_on(location::is_sending_locations_to_chat(ctx, chat_id)) as libc::c_int
+    block_on(location::is_sending_locations_to_chat(ctx, chat_id))
+        .unwrap_or_log_default(ctx, "Failed dc_is_sending_locations_to_chat()") as libc::c_int
 }
 
 #[no_mangle]
@@ -2125,6 +2218,16 @@ pub unsafe extern "C" fn dc_delete_all_locations(context: *mut dc_context_t) {
     });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_last_error(context: *mut dc_context_t) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_last_error()");
+        return "".strdup();
+    }
+    let ctx = &*context;
+    block_on(ctx.get_last_error()).strdup()
+}
+
 // dc_array_t
 
 pub type dc_array_t = dc_array::dc_array_t;
@@ -2383,13 +2486,13 @@ pub unsafe extern "C" fn dc_chatlist_get_summary(
     let ctx = &*ffi_list.context;
 
     block_on(async move {
-        let lot = ffi_list
+        let summary = ffi_list
             .list
             .get_summary(ctx, index as usize, maybe_chat)
             .await
             .log_err(ctx, "get_summary failed")
             .unwrap_or_default();
-        Box::into_raw(Box::new(lot))
+        Box::into_raw(Box::new(summary.into()))
     })
 }
 
@@ -2409,13 +2512,15 @@ pub unsafe extern "C" fn dc_chatlist_get_summary2(
     } else {
         Some(MsgId::new(msg_id))
     };
-    block_on(async move {
-        let lot = Chatlist::get_summary2(ctx, ChatId::new(chat_id), msg_id, None)
-            .await
-            .log_err(ctx, "get_summary2 failed")
-            .unwrap_or_default();
-        Box::into_raw(Box::new(lot))
-    })
+    let summary = block_on(Chatlist::get_summary2(
+        ctx,
+        ChatId::new(chat_id),
+        msg_id,
+        None,
+    ))
+    .log_err(ctx, "get_summary2 failed")
+    .unwrap_or_default();
+    Box::into_raw(Box::new(summary.into()))
 }
 
 #[no_mangle]
@@ -2580,8 +2685,10 @@ pub unsafe extern "C" fn dc_chat_can_send(chat: *mut dc_chat_t) -> libc::c_int {
         return 0;
     }
     let ffi_chat = &*chat;
-    let cxt = &*ffi_chat.context;
-    block_on(ffi_chat.chat.can_send(cxt)) as libc::c_int
+    let ctx = &*ffi_chat.context;
+    block_on(ffi_chat.chat.can_send(ctx))
+        .log_err(ctx, "can_send failed")
+        .unwrap_or_default() as libc::c_int
 }
 
 #[no_mangle]
@@ -2768,6 +2875,16 @@ pub unsafe extern "C" fn dc_msg_get_state(msg: *mut dc_msg_t) -> libc::c_int {
     ffi_msg.message.get_state() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_download_state(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_download_state()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.download_state() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_timestamp(msg: *mut dc_msg_t) -> i64 {
     if msg.is_null() {
@@ -2957,10 +3074,10 @@ pub unsafe extern "C" fn dc_msg_get_summary(
     let ffi_msg = &mut *msg;
     let ctx = &*ffi_msg.context;
 
-    block_on(async move {
-        let lot = ffi_msg.message.get_summary(ctx, maybe_chat).await;
-        Box::into_raw(Box::new(lot))
-    })
+    let summary = block_on(ffi_msg.message.get_summary(ctx, maybe_chat))
+        .log_err(ctx, "dc_msg_get_summary failed")
+        .unwrap_or_default();
+    Box::into_raw(Box::new(summary.into()))
 }
 
 #[no_mangle]
@@ -2975,12 +3092,13 @@ pub unsafe extern "C" fn dc_msg_get_summarytext(
     let ffi_msg = &mut *msg;
     let ctx = &*ffi_msg.context;
 
-    block_on({
-        ffi_msg
-            .message
-            .get_summarytext(ctx, approx_characters.try_into().unwrap_or_default())
-    })
-    .strdup()
+    let summary = block_on(ffi_msg.message.get_summary(ctx, None))
+        .log_err(ctx, "dc_msg_get_summarytext failed")
+        .unwrap_or_default();
+    match usize::try_from(approx_characters) {
+        Ok(chars) => summary.truncated_text(chars).strdup(),
+        Err(_) => summary.text.strdup(),
+    }
 }
 
 #[no_mangle]
@@ -3307,6 +3425,16 @@ pub unsafe extern "C" fn dc_msg_get_quoted_msg(msg: *const dc_msg_t) -> *mut dc_
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_force_plaintext(msg: *mut dc_msg_t) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_force_plaintext()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg.message.force_plaintext();
+}
+
 // dc_contact_t
 
 /// FFI struct for [dc_contact_t]
@@ -3438,6 +3566,16 @@ pub unsafe extern "C" fn dc_contact_get_status(contact: *mut dc_contact_t) -> *m
     ffi_contact.contact.get_status().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_last_seen(contact: *mut dc_contact_t) -> i64 {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_last_seen()");
+        return 0;
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.last_seen()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_is_blocked(contact: *mut dc_contact_t) -> libc::c_int {
     if contact.is_null() {
@@ -3457,7 +3595,9 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
     let ffi_contact = &*contact;
     let ctx = &*ffi_contact.context;
 
-    block_on(async move { ffi_contact.contact.is_verified(ctx).await as libc::c_int })
+    block_on(ffi_contact.contact.is_verified(ctx))
+        .log_err(ctx, "is_verified failed")
+        .unwrap_or_default() as libc::c_int
 }
 
 // dc_lot_t
@@ -3515,7 +3655,7 @@ pub unsafe extern "C" fn dc_lot_get_state(lot: *mut dc_lot_t) -> libc::c_int {
     }
 
     let lot = &*lot;
-    lot.get_state().to_i64().expect("impossible") as libc::c_int
+    lot.get_state() as libc::c_int
 }
 
 #[no_mangle]
@@ -3603,9 +3743,26 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
         return ptr::null();
     }
     let addr = to_string_lossy(addr);
-    match block_on(provider::get_provider_info(addr.as_str())) {
-        Some(provider) => provider,
-        None => ptr::null_mut(),
+
+    let ctx = &*context;
+    let socks5_enabled = block_on(async move {
+        ctx.get_config_bool(config::Config::Socks5Enabled)
+            .await
+            .log_err(ctx, "Can't get config")
+    });
+
+    match socks5_enabled {
+        Ok(socks5_enabled) => {
+            match block_on(provider::get_provider_info(
+                ctx,
+                addr.as_str(),
+                socks5_enabled,
+            )) {
+                Some(provider) => provider,
+                None => ptr::null_mut(),
+            }
+        }
+        Err(_) => ptr::null_mut(),
     }
 }
 
@@ -3656,12 +3813,33 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
 
 // -- Accounts
 
+/// Reader-writer lock wrapper for accounts manager to guarantee thread safety when using
+/// `dc_accounts_t` in multiple threads at once.
+pub struct AccountsWrapper {
+    inner: RwLock<Accounts>,
+}
+
+impl Deref for AccountsWrapper {
+    type Target = RwLock<Accounts>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl AccountsWrapper {
+    fn new(accounts: Accounts) -> Self {
+        let inner = RwLock::new(accounts);
+        Self { inner }
+    }
+}
+
 /// Struct representing a list of deltachat accounts.
-pub type dc_accounts_t = Accounts;
+pub type dc_accounts_t = AccountsWrapper;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_new(
-    os_name: *const libc::c_char,
+    _os_name: *const libc::c_char,
     dbfile: *const libc::c_char,
 ) -> *mut dc_accounts_t {
     setup_panic!();
@@ -3671,16 +3849,10 @@ pub unsafe extern "C" fn dc_accounts_new(
         return ptr::null_mut();
     }
 
-    let os_name = if os_name.is_null() {
-        String::from("DcFFI")
-    } else {
-        to_string_lossy(os_name)
-    };
-
-    let accs = block_on(Accounts::new(os_name, as_path(dbfile).to_path_buf().into()));
+    let accs = block_on(Accounts::new(as_path(dbfile).to_path_buf().into()));
 
     match accs {
-        Ok(accs) => Box::into_raw(Box::new(accs)),
+        Ok(accs) => Box::into_raw(Box::new(AccountsWrapper::new(accs))),
         Err(err) => {
             // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
             eprintln!("failed to create accounts: {:#}", err);
@@ -3712,7 +3884,7 @@ pub unsafe extern "C" fn dc_accounts_get_account(
     }
 
     let accounts = &*accounts;
-    block_on(accounts.get_account(id))
+    block_on(async move { accounts.read().await.get_account(id).await })
         .map(|ctx| Box::into_raw(Box::new(ctx)))
         .unwrap_or_else(std::ptr::null_mut)
 }
@@ -3727,7 +3899,7 @@ pub unsafe extern "C" fn dc_accounts_get_selected_account(
     }
 
     let accounts = &*accounts;
-    block_on(accounts.get_selected_account())
+    block_on(async move { accounts.read().await.get_selected_account().await })
         .map(|ctx| Box::into_raw(Box::new(ctx)))
         .unwrap_or_else(std::ptr::null_mut)
 }
@@ -3743,9 +3915,19 @@ pub unsafe extern "C" fn dc_accounts_select_account(
     }
 
     let accounts = &*accounts;
-    block_on(accounts.select_account(id))
-        .map(|_| 1)
-        .unwrap_or(0)
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        match accounts.select_account(id).await {
+            Ok(()) => 1,
+            Err(err) => {
+                accounts.emit_event(EventType::Error(format!(
+                    "Failed to select account: {:#}",
+                    err
+                )));
+                0
+            }
+        }
+    })
 }
 
 #[no_mangle]
@@ -3755,9 +3937,21 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -
         return 0;
     }
 
-    let accounts = &*accounts;
+    let accounts = &mut *accounts;
 
-    block_on(accounts.add_account()).unwrap_or(0)
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        match accounts.add_account().await {
+            Ok(id) => id,
+            Err(err) => {
+                accounts.emit_event(EventType::Error(format!(
+                    "Failed to add account: {:#}",
+                    err
+                )));
+                0
+            }
+        }
+    })
 }
 
 #[no_mangle]
@@ -3770,11 +3964,21 @@ pub unsafe extern "C" fn dc_accounts_remove_account(
         return 0;
     }
 
-    let accounts = &*accounts;
+    let accounts = &mut *accounts;
 
-    block_on(accounts.remove_account(id))
-        .map(|_| 1)
-        .unwrap_or_else(|_| 0)
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        match accounts.remove_account(id).await {
+            Ok(()) => 1,
+            Err(err) => {
+                accounts.emit_event(EventType::Error(format!(
+                    "Failed to remove account: {:#}",
+                    err
+                )));
+                0
+            }
+        }
+    })
 }
 
 #[no_mangle]
@@ -3787,12 +3991,25 @@ pub unsafe extern "C" fn dc_accounts_migrate_account(
         return 0;
     }
 
-    let accounts = &*accounts;
+    let accounts = &mut *accounts;
     let dbfile = to_string_lossy(dbfile);
 
-    block_on(accounts.migrate_account(async_std::path::PathBuf::from(dbfile)))
-        .map(|_| 1)
-        .unwrap_or_else(|_| 0)
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        match accounts
+            .migrate_account(async_std::path::PathBuf::from(dbfile))
+            .await
+        {
+            Ok(id) => id,
+            Err(err) => {
+                accounts.emit_event(EventType::Error(format!(
+                    "Failed to migrate account: {:#}",
+                    err
+                )));
+                0
+            }
+        }
+    })
 }
 
 #[no_mangle]
@@ -3803,7 +4020,7 @@ pub unsafe extern "C" fn dc_accounts_get_all(accounts: *mut dc_accounts_t) -> *m
     }
 
     let accounts = &*accounts;
-    let list = block_on(accounts.get_all());
+    let list = block_on(async move { accounts.read().await.get_all().await });
     let array: dc_array_t = list.into();
 
     Box::into_raw(Box::new(array))
@@ -3816,7 +4033,7 @@ pub unsafe extern "C" fn dc_accounts_all_work_done(accounts: *mut dc_accounts_t)
         return 0;
     }
     let accounts = &*accounts;
-    block_on(async move { accounts.all_work_done().await as libc::c_int })
+    block_on(async move { accounts.read().await.all_work_done().await as libc::c_int })
 }
 
 #[no_mangle]
@@ -3827,7 +4044,7 @@ pub unsafe extern "C" fn dc_accounts_start_io(accounts: *mut dc_accounts_t) {
     }
 
     let accounts = &*accounts;
-    block_on(accounts.start_io());
+    block_on(async move { accounts.read().await.start_io().await });
 }
 
 #[no_mangle]
@@ -3838,7 +4055,7 @@ pub unsafe extern "C" fn dc_accounts_stop_io(accounts: *mut dc_accounts_t) {
     }
 
     let accounts = &*accounts;
-    block_on(accounts.stop_io());
+    block_on(async move { accounts.read().await.stop_io().await });
 }
 
 #[no_mangle]
@@ -3849,7 +4066,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network(accounts: *mut dc_accounts_t)
     }
 
     let accounts = &*accounts;
-    block_on(accounts.maybe_network());
+    block_on(async move { accounts.read().await.maybe_network().await });
 }
 
 #[no_mangle]
@@ -3860,7 +4077,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     }
 
     let accounts = &*accounts;
-    block_on(accounts.maybe_network_lost());
+    block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
 pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
@@ -3875,7 +4092,7 @@ pub unsafe extern "C" fn dc_accounts_get_event_emitter(
     }
 
     let accounts = &*accounts;
-    let emitter = block_on(accounts.get_event_emitter());
+    let emitter = block_on(async move { accounts.read().await.get_event_emitter().await });
 
     Box::into_raw(Box::new(emitter))
 }

deltachat-ffi/src/lot.rs

@@ -0,0 +1,245 @@
+//! # Legacy generic return values for C API.
+
+use crate::message::MessageState;
+use crate::qr::Qr;
+use crate::summary::{Summary, SummaryPrefix};
+use anyhow::Error;
+use std::borrow::Cow;
+
+/// An object containing a set of values.
+/// The meaning of the values is defined by the function returning the object.
+/// Lot objects are created
+/// eg. by chatlist.get_summary() or dc_msg_get_summary().
+///
+/// *Lot* is used in the meaning *heap* here.
+#[derive(Debug)]
+pub enum Lot {
+    Summary(Summary),
+    Qr(Qr),
+    Error(String),
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Meaning {
+    None = 0,
+    Text1Draft = 1,
+    Text1Username = 2,
+    Text1Self = 3,
+}
+
+impl Default for Meaning {
+    fn default() -> Self {
+        Meaning::None
+    }
+}
+
+impl Lot {
+    pub fn get_text1(&self) -> Option<&str> {
+        match self {
+            Self::Summary(summary) => match &summary.prefix {
+                None => None,
+                Some(SummaryPrefix::Draft(text)) => Some(text),
+                Some(SummaryPrefix::Username(username)) => Some(username),
+                Some(SummaryPrefix::Me(text)) => Some(text),
+            },
+            Self::Qr(qr) => match qr {
+                Qr::AskVerifyContact { .. } => None,
+                Qr::AskVerifyGroup { grpname, .. } => Some(grpname),
+                Qr::FprOk { .. } => None,
+                Qr::FprMismatch { .. } => None,
+                Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
+                Qr::Account { domain } => Some(domain),
+                Qr::WebrtcInstance { domain, .. } => Some(domain),
+                Qr::Addr { .. } => None,
+                Qr::Url { url } => Some(url),
+                Qr::Text { text } => Some(text),
+                Qr::WithdrawVerifyContact { .. } => None,
+                Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
+                Qr::ReviveVerifyContact { .. } => None,
+                Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
+            },
+            Self::Error(err) => Some(err),
+        }
+    }
+
+    pub fn get_text2(&self) -> Option<Cow<str>> {
+        match self {
+            Self::Summary(summary) => Some(summary.truncated_text(160)),
+            Self::Qr(_) => None,
+            Self::Error(_) => None,
+        }
+    }
+
+    pub fn get_text1_meaning(&self) -> Meaning {
+        match self {
+            Self::Summary(summary) => match &summary.prefix {
+                None => Meaning::None,
+                Some(SummaryPrefix::Draft(_text)) => Meaning::Text1Draft,
+                Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
+                Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
+            },
+            Self::Qr(_qr) => Meaning::None,
+            Self::Error(_err) => Meaning::None,
+        }
+    }
+
+    pub fn get_state(&self) -> LotState {
+        match self {
+            Self::Summary(summary) => summary.state.into(),
+            Self::Qr(qr) => match qr {
+                Qr::AskVerifyContact { .. } => LotState::QrAskVerifyContact,
+                Qr::AskVerifyGroup { .. } => LotState::QrAskVerifyGroup,
+                Qr::FprOk { .. } => LotState::QrFprOk,
+                Qr::FprMismatch { .. } => LotState::QrFprMismatch,
+                Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
+                Qr::Account { .. } => LotState::QrAccount,
+                Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
+                Qr::Addr { .. } => LotState::QrAddr,
+                Qr::Url { .. } => LotState::QrUrl,
+                Qr::Text { .. } => LotState::QrText,
+                Qr::WithdrawVerifyContact { .. } => LotState::QrWithdrawVerifyContact,
+                Qr::WithdrawVerifyGroup { .. } => LotState::QrWithdrawVerifyGroup,
+                Qr::ReviveVerifyContact { .. } => LotState::QrReviveVerifyContact,
+                Qr::ReviveVerifyGroup { .. } => LotState::QrReviveVerifyGroup,
+            },
+            Self::Error(_err) => LotState::QrError,
+        }
+    }
+
+    pub fn get_id(&self) -> u32 {
+        match self {
+            Self::Summary(_) => Default::default(),
+            Self::Qr(qr) => match qr {
+                Qr::AskVerifyContact { contact_id, .. } => *contact_id,
+                Qr::AskVerifyGroup { .. } => Default::default(),
+                Qr::FprOk { contact_id } => *contact_id,
+                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default(),
+                Qr::FprWithoutAddr { .. } => Default::default(),
+                Qr::Account { .. } => Default::default(),
+                Qr::WebrtcInstance { .. } => Default::default(),
+                Qr::Addr { contact_id } => *contact_id,
+                Qr::Url { .. } => Default::default(),
+                Qr::Text { .. } => Default::default(),
+                Qr::WithdrawVerifyContact { contact_id, .. } => *contact_id,
+                Qr::WithdrawVerifyGroup { .. } => Default::default(),
+                Qr::ReviveVerifyContact { contact_id, .. } => *contact_id,
+                Qr::ReviveVerifyGroup { .. } => Default::default(),
+            },
+            Self::Error(_) => Default::default(),
+        }
+    }
+
+    pub fn get_timestamp(&self) -> i64 {
+        match self {
+            Self::Summary(summary) => summary.timestamp,
+            Self::Qr(_) => Default::default(),
+            Self::Error(_) => Default::default(),
+        }
+    }
+}
+
+#[repr(u32)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LotState {
+    // Default
+    Undefined = 0,
+
+    // Qr States
+    /// id=contact
+    QrAskVerifyContact = 200,
+
+    /// text1=groupname
+    QrAskVerifyGroup = 202,
+
+    /// id=contact
+    QrFprOk = 210,
+
+    /// id=contact
+    QrFprMismatch = 220,
+
+    /// text1=formatted fingerprint
+    QrFprWithoutAddr = 230,
+
+    /// text1=domain
+    QrAccount = 250,
+
+    /// text1=domain, text2=instance pattern
+    QrWebrtcInstance = 260,
+
+    /// id=contact
+    QrAddr = 320,
+
+    /// text1=text
+    QrText = 330,
+
+    /// text1=URL
+    QrUrl = 332,
+
+    /// text1=error string
+    QrError = 400,
+
+    QrWithdrawVerifyContact = 500,
+
+    /// text1=groupname
+    QrWithdrawVerifyGroup = 502,
+
+    QrReviveVerifyContact = 510,
+
+    /// text1=groupname
+    QrReviveVerifyGroup = 512,
+
+    // Message States
+    MsgInFresh = 10,
+    MsgInNoticed = 13,
+    MsgInSeen = 16,
+    MsgOutPreparing = 18,
+    MsgOutDraft = 19,
+    MsgOutPending = 20,
+    MsgOutFailed = 24,
+    MsgOutDelivered = 26,
+    MsgOutMdnRcvd = 28,
+}
+
+impl Default for LotState {
+    fn default() -> Self {
+        LotState::Undefined
+    }
+}
+
+impl From<MessageState> for LotState {
+    fn from(s: MessageState) -> Self {
+        use MessageState::*;
+        match s {
+            Undefined => LotState::Undefined,
+            InFresh => LotState::MsgInFresh,
+            InNoticed => LotState::MsgInNoticed,
+            InSeen => LotState::MsgInSeen,
+            OutPreparing => LotState::MsgOutPreparing,
+            OutDraft => LotState::MsgOutDraft,
+            OutPending => LotState::MsgOutPending,
+            OutFailed => LotState::MsgOutFailed,
+            OutDelivered => LotState::MsgOutDelivered,
+            OutMdnRcvd => LotState::MsgOutMdnRcvd,
+        }
+    }
+}
+
+impl From<Summary> for Lot {
+    fn from(summary: Summary) -> Self {
+        Lot::Summary(summary)
+    }
+}
+
+impl From<Qr> for Lot {
+    fn from(qr: Qr) -> Self {
+        Lot::Qr(qr)
+    }
+}
+
+// Make it easy to convert errors into the final `Lot`.
+impl From<Error> for Lot {
+    fn from(error: Error) -> Self {
+        Lot::Error(error.to_string())
+    }
+}

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1.0.74"
-quote = "1.0.2"
+syn = "1"
+quote = "1"

draft/aeap-mvp.rst

@@ -0,0 +1,33 @@
+AEAP MVP
+========
+
+Changes to the UIs
+------------------
+
+- The secondary self addresses (see below) are shown in the UI, but not editable.
+
+- When the user changed the email address in the configure screen, show a dialog to the user, either directly explaining things or with a link to the FAQ (see "Other" below)
+
+Changes in the core
+-------------------
+
+- We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
+
+- If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
+
+  - don't forget to deduplicate secondary self addresses in case the user switches back and forth between addresses).
+
+  - The key stays the same.
+
+- No changes for 1:1 chats, there simply is a new one
+
+- When we send a message to a group, and the primary address is not a member of a group, but a secondary address is:
+  
+  Add Chat-Group-Member-Removed=<old address> and Chat-Group-Member-Added=<new address> headers to this message
+
+  - On the receiving side, make sure that we accept this (even in verified groups) if the message is signed and the key stayed the same
+
+Other
+-----
+
+- The user is responsible that messages to the old address arrive at the new address, for example by configuring the old provider to forward all emails to the new one.

examples/repl/cmdline.rs

@@ -2,7 +2,7 @@ extern crate dirs;
 
 use std::str::FromStr;
 
-use anyhow::{bail, ensure, Error};
+use anyhow::{bail, ensure, Result};
 use async_std::path::Path;
 use deltachat::chat::{
     self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
@@ -13,10 +13,10 @@ use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
+use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
-use deltachat::lot::LotState;
 use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
@@ -98,10 +98,10 @@ async fn reset_tables(context: &Context, bits: i32) {
     });
 }
 
-async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
+async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
     let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false).await {
+    if let Err(err) = dc_receive_imf(context, &data, "import", false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -189,10 +189,18 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         MessageState::OutFailed => " !!",
         _ => "",
     };
+
+    let downloadstate = match msg.download_state() {
+        DownloadState::Done => "",
+        DownloadState::Available => " [⬇ Download available]",
+        DownloadState::InProgress => " [⬇ Download in progress...]️",
+        DownloadState::Failure => " [⬇ Download failed]",
+    };
+
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -226,11 +234,12 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             ""
         },
         statestr,
+        downloadstate,
         &temp2,
     );
 }
 
-async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
+async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
     let mut lines_out = 0;
     for &msg_id in msglist {
         if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
@@ -258,59 +267,59 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error>
     Ok(())
 }
 
-async fn log_contactlist(context: &Context, contacts: &[u32]) {
+async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        if let Ok(contact) = Contact::get_by_id(context, *contact_id).await {
-            let name = contact.get_display_name();
-            let addr = contact.get_addr();
-            let verified_state = contact.is_verified(context).await;
-            let verified_str = if VerifiedStatus::Unverified != verified_state {
-                if verified_state == VerifiedStatus::BidirectVerified {
-                    " √√"
-                } else {
-                    " √"
-                }
+        let contact = Contact::get_by_id(context, *contact_id).await?;
+        let name = contact.get_display_name();
+        let addr = contact.get_addr();
+        let verified_state = contact.is_verified(context).await?;
+        let verified_str = if VerifiedStatus::Unverified != verified_state {
+            if verified_state == VerifiedStatus::BidirectVerified {
+                " √√"
             } else {
-                ""
-            };
-            line = format!(
-                "{}{} <{}>",
-                if !name.is_empty() {
-                    &name
-                } else {
-                    "<name unset>"
-                },
-                verified_str,
-                if !addr.is_empty() {
-                    &addr
-                } else {
-                    "addr unset"
-                }
-            );
-            let peerstate = Peerstate::from_addr(context, &addr)
-                .await
-                .expect("peerstate error");
-            if peerstate.is_some() && *contact_id != 1 {
-                line2 = format!(
-                    ", prefer-encrypt={}",
-                    peerstate.as_ref().unwrap().prefer_encrypt
-                );
+                " √"
             }
-
-            println!("Contact#{}: {}{}", *contact_id, line, line2);
+        } else {
+            ""
+        };
+        line = format!(
+            "{}{} <{}>",
+            if !name.is_empty() {
+                &name
+            } else {
+                "<name unset>"
+            },
+            verified_str,
+            if !addr.is_empty() {
+                &addr
+            } else {
+                "addr unset"
+            }
+        );
+        let peerstate = Peerstate::from_addr(context, &addr)
+            .await
+            .expect("peerstate error");
+        if peerstate.is_some() && *contact_id != 1 {
+            line2 = format!(
+                ", prefer-encrypt={}",
+                peerstate.as_ref().unwrap().prefer_encrypt
+            );
         }
+
+        println!("Contact#{}: {}{}", *contact_id, line, line2);
     }
+    Ok(())
 }
 
 fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<(), Error> {
+pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<()> {
     let mut sel_chat = if !chat_id.is_unset() {
-        Chat::load_from_db(&context, *chat_id).await.ok()
+        Some(Chat::load_from_db(&context, *chat_id).await?)
     } else {
         None
     };
@@ -361,6 +370,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  chat [<chat-id>|0]\n\
                  createchat <contact-id>\n\
                  creategroup <name>\n\
+                 createbroadcast\n\
                  createprotected <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
@@ -376,6 +386,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  sendsticker <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
                  sendhtml <file for html-part> [<text for plain-part>]\n\
+                 sendsyncmsg\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -394,6 +405,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
+                 download <msg-id>\n\
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
@@ -411,8 +423,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listblocked\n\
                  ======================================Misc.==\n\
                  getqr [<chat-id>]\n\
+                 getqrsvg [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
+                 joinqr <qr-content>\n\
                  setqr <qr-content>\n\
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
@@ -450,7 +464,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            continue_key_transfer(&context, MsgId::new(arg1.parse()?), &arg2).await?;
+            continue_key_transfer(&context, MsgId::new(arg1.parse()?), arg2).await?;
         }
         "has-backup" => {
             has_backup(&context, blobdir).await?;
@@ -497,13 +511,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "set" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(&arg1)?;
+            let key = config::Config::from_str(arg1)?;
             let value = if arg2.is_empty() { None } else { Some(arg2) };
             context.set_config(key, value).await?;
         }
         "get" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(&arg1)?;
+            let key = config::Config::from_str(arg1)?;
             let val = context.get_config(key).await;
             println!("{}={:?}", key, val);
         }
@@ -514,9 +528,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let file = dirs::home_dir()
                 .unwrap_or_default()
                 .join("connectivity.html");
-            let html = context.get_connectivity_html().await;
-            fs::write(&file, html)?;
-            println!("Report written to: {:#?}", file);
+            match context.get_connectivity_html().await {
+                Ok(html) => {
+                    fs::write(&file, html)?;
+                    println!("Report written to: {:#?}", file);
+                }
+                Err(err) => {
+                    bail!("Failed to get connectivity html: {}", err);
+                }
+            }
         }
         "maybenetwork" => {
             context.maybe_network().await;
@@ -563,26 +583,25 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             ""
                         },
                     );
-                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await?;
+                    let summary = chatlist.get_summary(&context, i, Some(&chat)).await?;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
-                        match lot.get_state() {
-                            LotState::MsgOutPending => " o",
-                            LotState::MsgOutDelivered => " √",
-                            LotState::MsgOutMdnRcvd => " √√",
-                            LotState::MsgOutFailed => " !!",
+                        match summary.state {
+                            MessageState::OutPending => " o",
+                            MessageState::OutDelivered => " √",
+                            MessageState::OutMdnRcvd => " √√",
+                            MessageState::OutFailed => " !!",
                             _ => "",
                         }
                     };
-                    let timestr = dc_timestamp_to_str(lot.get_timestamp());
-                    let text1 = lot.get_text1();
-                    let text2 = lot.get_text2();
+                    let timestr = dc_timestamp_to_str(summary.timestamp);
                     println!(
-                        "{}{}{}{} [{}]{}",
-                        text1.unwrap_or(""),
-                        if text1.is_some() { ": " } else { "" },
-                        text2.unwrap_or(""),
+                        "{}{}{} [{}]{}",
+                        summary
+                            .prefix
+                            .map_or_else(String::new, |prefix| format!("{}: ", prefix)),
+                        summary.text,
                         statestr,
                         &timestr,
                         if chat.is_sending_locations() {
@@ -596,7 +615,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, None).await {
+            if location::is_sending_locations_to_chat(&context, None).await? {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
@@ -699,6 +718,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!("Group#{} created successfully.", chat_id);
         }
+        "createbroadcast" => {
+            let chat_id = chat::create_broadcast_list(&context).await?;
+
+            println!("Broadcast#{} created successfully.", chat_id);
+        }
         "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
@@ -711,17 +735,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
             let contact_id_0: u32 = arg1.parse()?;
-            if chat::add_contact_to_chat(
-                &context,
-                sel_chat.as_ref().unwrap().get_id(),
-                contact_id_0,
-            )
-            .await
-            {
-                println!("Contact added to chat.");
-            } else {
-                bail!("Cannot add contact to chat.");
-            }
+            chat::add_contact_to_chat(&context, sel_chat.as_ref().unwrap().get_id(), contact_id_0)
+                .await?;
+            println!("Contact added to chat.");
         }
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -739,7 +755,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "groupname" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            chat::set_chat_name(&context, sel_chat.as_ref().unwrap().get_id(), arg1).await?;
+            chat::set_chat_name(
+                &context,
+                sel_chat.as_ref().unwrap().get_id(),
+                &format!("{} {}", arg1, arg2).trim(),
+            )
+            .await?;
 
             println!("Chat name set");
         }
@@ -759,7 +780,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await?;
             println!("Memberlist:");
 
-            log_contactlist(&context, &contacts).await;
+            log_contactlist(&context, &contacts).await?;
             println!(
                 "{} contacts\nLocation streaming: {}",
                 contacts.len(),
@@ -767,7 +788,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     &context,
                     Some(sel_chat.as_ref().unwrap().get_id())
                 )
-                .await,
+                .await?,
             );
         }
         "getlocations" => {
@@ -812,7 +833,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 sel_chat.as_ref().unwrap().get_id(),
                 seconds,
             )
-            .await;
+            .await?;
             println!(
                 "Locations will be sent to Chat#{} for {} seconds. Use 'setlocation <lat> <lng>' to play around.",
                 sel_chat.as_ref().unwrap().get_id(),
@@ -882,6 +903,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }));
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "sendsyncmsg" => match context.send_sync_msg().await? {
+            Some(msg_id) => println!("sync message sent as {}.", msg_id),
+            None => println!("sync message not needed."),
+        },
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -889,12 +914,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
-            let chat = if let Some(ref sel_chat) = sel_chat {
-                Some(sel_chat.get_id())
-            } else {
-                None
-            };
-
+            let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
             let time_start = std::time::SystemTime::now();
             let msglist = context.search_msgs(chat, arg1).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
@@ -1024,6 +1044,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let res = message::get_msg_info(&context, id).await?;
             println!("{}", res);
         }
+        "download" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            println!("Scheduling download for {:?}", id);
+            id.download_full(&context).await?;
+        }
         "html" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
@@ -1061,7 +1087,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
             ids[0] = MsgId::new(arg1.parse()?);
-            message::delete_msgs(&context, &ids).await;
+            message::delete_msgs(&context, &ids).await?;
         }
         "listcontacts" | "contacts" | "listverified" => {
             let contacts = Contact::get_all(
@@ -1074,7 +1100,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 Some(arg1),
             )
             .await?;
-            log_contactlist(&context, &contacts).await;
+            log_contactlist(&context, &contacts).await?;
             println!("{} contacts.", contacts.len());
         }
         "addcontact" => {
@@ -1139,19 +1165,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "listblocked" => {
             let contacts = Contact::get_all_blocked(&context).await?;
-            log_contactlist(&context, &contacts).await;
+            log_contactlist(&context, &contacts).await?;
             println!("{} blocked contacts.", contacts.len());
         }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            let res = check_qr(&context, arg1).await;
-            println!(
-                "state={}, id={}, text1={:?}, text2={:?}",
-                res.get_state(),
-                res.get_id(),
-                res.get_text1(),
-                res.get_text2()
-            );
+            let qr = check_qr(&context, arg1).await?;
+            println!("qr={:?}", qr);
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
@@ -1162,7 +1182,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            match provider::get_provider_info(arg1).await {
+            let socks5_enabled = context
+                .get_config_bool(config::Config::Socks5Enabled)
+                .await?;
+            match provider::get_provider_info(&context, arg1, socks5_enabled).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);

examples/repl/main.rs

@@ -33,6 +33,8 @@ use rustyline::{
 
 mod cmdline;
 use self::cmdline::*;
+use deltachat::qr_code_generator::get_securejoin_qr_svg;
+use std::fs;
 
 /// Event Handler
 fn receive_event(event: EventType) {
@@ -167,13 +169,14 @@ const DB_COMMANDS: [&str; 10] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 33] = [
+const CHAT_COMMANDS: [&str; 35] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
     "creategroup",
-    "createverified",
+    "createbroadcast",
+    "createprotected",
     "addmember",
     "removemember",
     "groupname",
@@ -187,6 +190,7 @@ const CHAT_COMMANDS: [&str; 33] = [
     "sendimage",
     "sendfile",
     "sendhtml",
+    "sendsyncmsg",
     "videochat",
     "draft",
     "listmedia",
@@ -202,13 +206,14 @@ const CHAT_COMMANDS: [&str; 33] = [
     "accept",
     "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 6] = [
+const MESSAGE_COMMANDS: [&str; 7] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
     "delmsg",
+    "download",
 ];
 const CONTACT_COMMANDS: [&str; 9] = [
     "listcontacts",
@@ -221,10 +226,12 @@ const CONTACT_COMMANDS: [&str; 9] = [
     "unblock",
     "listblocked",
 ];
-const MISC_COMMANDS: [&str; 10] = [
+const MISC_COMMANDS: [&str; 12] = [
     "getqr",
+    "getqrsvg",
     "getbadqr",
     "checkqr",
+    "joinqr",
     "event",
     "fileinfo",
     "clear",
@@ -290,7 +297,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
@@ -325,7 +332,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
 
         loop {
             let p = "> ";
-            let readline = rl.readline(&p);
+            let readline = rl.readline(p);
 
             match readline {
                 Ok(line) => {
@@ -392,7 +399,7 @@ async fn handle_cmd(
         "oauth2" => {
             if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
                 let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
+                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -409,18 +416,31 @@ async fn handle_cmd(
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
             let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
-                if !qr.is_empty() {
-                    if arg0 == "getbadqr" && qr.len() > 40 {
-                        qr.replace_range(12..22, "0000000000")
-                    }
-                    println!("{}", qr);
-                    let output = Command::new("qrencode")
-                        .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
-                        .output()
-                        .expect("failed to execute process");
-                    io::stdout().write_all(&output.stdout).unwrap();
-                    io::stderr().write_all(&output.stderr).unwrap();
+            let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
+            if !qr.is_empty() {
+                if arg0 == "getbadqr" && qr.len() > 40 {
+                    qr.replace_range(12..22, "0000000000")
+                }
+                println!("{}", qr);
+                let output = Command::new("qrencode")
+                    .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
+                    .output()
+                    .expect("failed to execute process");
+                io::stdout().write_all(&output.stdout).unwrap();
+                io::stderr().write_all(&output.stderr).unwrap();
+            }
+        }
+        "getqrsvg" => {
+            ctx.start_io().await;
+            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
+            match get_securejoin_qr_svg(&ctx, group).await {
+                Ok(svg) => {
+                    fs::write(&file, svg)?;
+                    println!("QR code svg written to: {:#?}", file);
+                }
+                Err(err) => {
+                    bail!("Failed to get QR code svg: {}", err);
                 }
             }
         }

examples/simple.rs

@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
+    let ctx = Context::new(dbfile.into(), 0)
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;

proptest-regressions/dc_tools.txt

@@ -7,3 +7,5 @@
 cc c310754465ee0261807b96fa9bcc4861ff9aa286e94667524b5960c69f9b6620 # shrinks to buf = "", approx_chars = 0, do_unwrap = false
 cc 5fd8d730b0a9cdf7308ce58818ca9aefc0255c9ba2a0878944fc48d43a67315b # shrinks to buf = "𑒀ὐ￠🜀\u{1e01b}A a🟠", approx_chars = 0, do_unwrap = false
 cc c6a0029a54137a4b9efc9ef2ea6d9a7dd1d60d1c937bb472b66a174618ba8013 # shrinks to buf = "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ  ", approx_chars = 0, do_unwrap = false
+cc 9796807baeda701227dcdcfc9fdaa93ddd556da2bb1630381bfe2e037bee73f6 # shrinks to buf = " ꫛ®a\u{11300}a", approx_chars = 0
+cc 063a4c42ac1ec9aa37af54521b210ba9cd82dcc9cc3be296ca2fedf8240072d4 # shrinks to buf = "a᪠  0A", approx_chars = 0

python/mypy.ini

@@ -0,0 +1,19 @@
+[mypy]
+
+[mypy-deltachat.capi.*]
+ignore_missing_imports = True
+
+[mypy-pluggy.*]
+ignore_missing_imports = True
+
+[mypy-cffi.*]
+ignore_missing_imports = True
+
+[mypy-imapclient.*]
+ignore_missing_imports = True
+
+[mypy-pytest.*]
+ignore_missing_imports = True
+
+[mypy-_pytest.*]
+ignore_missing_imports = True

python/pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = ".."
+tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
+git_describe_command = "git describe --dirty --tags --long --match py-*.*"

python/setup.py

@@ -8,17 +8,11 @@ def main():
         long_description = f.read()
     setuptools.setup(
         name='deltachat',
-        setup_requires=['setuptools_scm', 'cffi>=1.0.0'],
-        use_scm_version = {
-            "root": "..",
-            "relative_to": __file__,
-            'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
-            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
-        },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
         install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
+        setup_requires=['setuptools_scm'], # required for compatibility with `python3 setup.py sdist`
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -19,9 +19,9 @@ except DistributionNotFound:
 
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
-        for name, val in vars(const).items():
+        for name in dir(const):
             if name.startswith("DC_EVENT_"):
-                _DC_EVENTNAME_MAP[val] = name
+                _DC_EVENTNAME_MAP[getattr(const, name)] = name
     return _DC_EVENTNAME_MAP[integer]
 
 
@@ -75,7 +75,6 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("addr", args.email)
         ac.set_config("mail_pw", args.password)
         ac.set_config("mvbox_move", "0")
-        ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
         ac.set_config("bot", "1")
         configtracker = ac.configure()

python/src/deltachat/account.py

@@ -8,13 +8,14 @@ import os
 from array import array
 from . import const
 from .capi import ffi, lib
-from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
+from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
 from .tracker import ImexTracker, ConfigureTracker
 from . import hookspec
 from .events import EventThread
+from typing import Union, Any, Dict, Optional, List, Generator
 
 
 class MissingCredentials(ValueError):
@@ -28,7 +29,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True):
+    def __init__(self, db_path, os_name=None, logging=True) -> None:
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -58,11 +59,11 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
-    def disable_logging(self):
+    def disable_logging(self) -> None:
         """ disable logging. """
         self._logging = False
 
-    def enable_logging(self):
+    def enable_logging(self) -> None:
         """ re-enable logging. """
         self._logging = True
 
@@ -73,12 +74,12 @@ class Account(object):
         if self._logging:
             self._pm.hook.ac_log_line(message=msg)
 
-    def _check_config_key(self, name):
+    def _check_config_key(self, name: str) -> None:
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
                            name, self._configkeys))
 
-    def get_info(self):
+    def get_info(self) -> Dict[str, str]:
         """ return dictionary of built config parameters. """
         lines = from_dc_charpointer(lib.dc_get_info(self._dc_context))
         d = {}
@@ -105,19 +106,19 @@ class Account(object):
             cursor += len(entry) + 1
         log("")
 
-    def set_stock_translation(self, id, string):
+    def set_stock_translation(self, id: int, string: str) -> None:
         """ set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
         :returns: None
         """
-        string = string.encode("utf8")
-        res = lib.dc_set_stock_translation(self._dc_context, id, string)
+        bytestring = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, bytestring)
         if res == 0:
             raise ValueError("could not set translation string")
 
-    def set_config(self, name, value):
+    def set_config(self, name: str, value: Optional[str]) -> None:
         """ set configuration values.
 
         :param name: config key name (unicode)
@@ -125,16 +126,16 @@ class Account(object):
         :returns: None
         """
         self._check_config_key(name)
-        name = name.encode("utf8")
-        if name == b"addr" and self.is_configured():
+        namebytes = name.encode("utf8")
+        if namebytes == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
         if value is not None:
-            value = value.encode("utf8")
+            valuebytes = value.encode("utf8")
         else:
-            value = ffi.NULL
-        lib.dc_set_config(self._dc_context, name, value)
+            valuebytes = ffi.NULL
+        lib.dc_set_config(self._dc_context, namebytes, valuebytes)
 
-    def get_config(self, name):
+    def get_config(self, name: str) -> str:
         """ return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
@@ -143,12 +144,12 @@ class Account(object):
         """
         if name != "sys.config_keys":
             self._check_config_key(name)
-        name = name.encode("utf8")
-        res = lib.dc_get_config(self._dc_context, name)
+        namebytes = name.encode("utf8")
+        res = lib.dc_get_config(self._dc_context, namebytes)
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr, public, secret):
+    def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None:
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
@@ -160,7 +161,7 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def update_config(self, kwargs):
+    def update_config(self, kwargs: Dict[str, Any]) -> None:
         """ update config values.
 
         :param kwargs: name=value config settings for this account.
@@ -170,7 +171,7 @@ class Account(object):
         for key, value in kwargs.items():
             self.set_config(key, str(value))
 
-    def is_configured(self):
+    def is_configured(self) -> bool:
         """ determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
@@ -178,7 +179,7 @@ class Account(object):
         """
         return True if lib.dc_is_configured(self._dc_context) else False
 
-    def set_avatar(self, img_path):
+    def set_avatar(self, img_path: Optional[str]) -> None:
         """Set self avatar.
 
         :raises ValueError: if profile image could not be set
@@ -190,20 +191,18 @@ class Account(object):
             assert os.path.exists(img_path), img_path
             self.set_config("selfavatar", img_path)
 
-    def check_is_configured(self):
+    def check_is_configured(self) -> None:
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_latest_backupfile(self, backupdir):
+    def get_latest_backupfile(self, backupdir) -> Optional[str]:
         """ return the latest backup file in a given directory.
         """
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
-        if res == ffi.NULL:
-            return None
-        return from_dc_charpointer(res)
+        return from_optional_dc_charpointer(res)
 
-    def get_blobdir(self):
+    def get_blobdir(self) -> str:
         """ return the directory for files.
 
         All sent files are copied to this directory if necessary.
@@ -211,15 +210,15 @@ class Account(object):
         """
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
-    def get_self_contact(self):
+    def get_self_contact(self) -> Contact:
         """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name=None):
-        """ create a (new) Contact or return an existing one.
+    def create_contact(self, obj, name: Optional[str] = None) -> Contact:
+        """create a (new) Contact or return an existing one.
 
         Calling this method will always result in the same
         underlying contact id.  If there already is a Contact
@@ -236,13 +235,13 @@ class Account(object):
         contact_id = lib.dc_create_contact(self._dc_context, name, addr)
         return Contact(self, contact_id)
 
-    def get_contact(self, obj):
+    def get_contact(self, obj) -> Optional[Contact]:
         if isinstance(obj, Contact):
             return obj
         (_, addr) = self.get_contact_addr_and_name(obj)
         return self.get_contact_by_addr(addr)
 
-    def get_contact_addr_and_name(self, obj, name=None):
+    def get_contact_addr_and_name(self, obj, name: Optional[str] = None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -260,7 +259,7 @@ class Account(object):
             name = displayname
         return (name, addr)
 
-    def delete_contact(self, contact):
+    def delete_contact(self, contact: Contact) -> bool:
         """ delete a Contact.
 
         :param contact: contact object obtained
@@ -271,22 +270,23 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email):
+    def get_contact_by_addr(self, email: str) -> Optional[Contact]:
         """ get a contact for the email address or None if it's blocked or doesn't exist. """
         _, addr = parseaddr(email)
         addr = as_dc_charpointer(addr)
         contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
         if contact_id:
             return self.get_contact_by_id(contact_id)
+        return None
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
+    def get_contact_by_id(self, contact_id: int) -> Contact:
+        """ return Contact instance or raise an exception.
         :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
+        :returns: :class:`deltachat.contact.Contact` instance.
         """
         return Contact(self, contact_id)
 
-    def get_blocked_contacts(self):
+    def get_blocked_contacts(self) -> List[Contact]:
         """ return a list of all blocked contacts.
 
         :returns: list of :class:`deltachat.contact.Contact` objects.
@@ -297,8 +297,13 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_contacts(self, query=None, with_self=False, only_verified=False):
-        """ get a (filtered) list of contacts.
+    def get_contacts(
+        self,
+        query: Optional[str] = None,
+        with_self: bool = False,
+        only_verified: bool = False,
+    ) -> List[Contact]:
+        """get a (filtered) list of contacts.
 
         :param query: if a string is specified, only return contacts
                       whose name or e-mail matches query.
@@ -318,7 +323,7 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_fresh_messages(self):
+    def get_fresh_messages(self) -> Generator[Message, None, None]:
         """ yield all fresh messages from all chats. """
         dc_array = ffi.gc(
             lib.dc_get_fresh_msgs(self._dc_context),
@@ -326,12 +331,17 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj):
+    def create_chat(self, obj) -> Chat:
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
-    def create_group_chat(self, name, contacts=None, verified=False):
-        """ create a new group chat object.
+    def create_group_chat(
+        self,
+        name: str,
+        contacts: Optional[List[Contact]] = None,
+        verified: bool = False,
+    ) -> Chat:
+        """create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
@@ -347,7 +357,7 @@ class Account(object):
                 chat.add_contact(contact)
         return chat
 
-    def get_chats(self):
+    def get_chats(self) -> List[Chat]:
         """ return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
@@ -364,17 +374,17 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self):
+    def get_device_chat(self) -> Chat:
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
-    def get_message_by_id(self, msg_id):
+    def get_message_by_id(self, msg_id: int) -> Message:
         """ return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
-    def get_chat_by_id(self, chat_id):
+    def get_chat_by_id(self, chat_id: int) -> Chat:
         """ return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
@@ -386,19 +396,21 @@ class Account(object):
         lib.dc_chat_unref(res)
         return Chat(self, chat_id)
 
-    def mark_seen_messages(self, messages):
+    def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
         """ mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
         arr = array("i")
         for msg in messages:
-            msg = getattr(msg, "id", msg)
-            arr.append(msg)
+            if isinstance(msg, Message):
+                arr.append(msg.id)
+            else:
+                arr.append(msg)
         msg_ids = ffi.cast("uint32_t*", ffi.from_buffer(arr))
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
-    def forward_messages(self, messages, chat):
+    def forward_messages(self, messages: List[Message], chat: Chat) -> None:
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -408,7 +420,7 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
-    def delete_messages(self, messages):
+    def delete_messages(self, messages: List[Message]) -> None:
         """ delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -466,7 +478,7 @@ class Account(object):
     def imex(self, path, imex_cmd):
         lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
 
-    def initiate_key_transfer(self):
+    def initiate_key_transfer(self) -> str:
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
@@ -477,7 +489,7 @@ class Account(object):
             raise RuntimeError("could not send out autocrypt setup message")
         return from_dc_charpointer(res)
 
-    def get_setup_contact_qr(self):
+    def get_setup_contact_qr(self) -> str:
         """ get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -527,7 +539,9 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
+    def set_location(
+        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
+    ) -> None:
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -571,7 +585,7 @@ class Account(object):
     def get_connectivity(self):
         return lib.dc_get_connectivity(self._dc_context)
 
-    def get_connectivity_html(self):
+    def get_connectivity_html(self) -> str:
         return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
 
     def all_work_done(self):
@@ -621,7 +635,7 @@ class Account(object):
         """
         lib.dc_maybe_network(self._dc_context)
 
-    def configure(self, reconfigure=False):
+    def configure(self, reconfigure: bool = False) -> ConfigureTracker:
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
@@ -634,11 +648,11 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def wait_shutdown(self):
+    def wait_shutdown(self) -> None:
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def stop_io(self):
+    def stop_io(self) -> None:
         """ stop core IO scheduler if it is running. """
         self.log("stop_ongoing")
         self.stop_ongoing()
@@ -646,7 +660,7 @@ class Account(object):
         self.log("dc_stop_io (stop core IO scheduler)")
         lib.dc_stop_io(self._dc_context)
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         """ shutdown and destroy account (stop callback thread, close and remove
         underlying dc_context)."""
         if self._dc_context is None:

python/src/deltachat/chat.py

@@ -3,12 +3,13 @@
 import mimetypes
 import calendar
 import json
-from datetime import datetime
+from datetime import datetime, timezone
 import os
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
+from typing import Optional
 
 
 class Chat(object):
@@ -17,20 +18,20 @@ class Chat(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id):
+    def __init__(self, account, id) -> None:
         from .account import Account
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         return self.id == getattr(other, "id", None) and \
                self.account._dc_context == other.account._dc_context
 
-    def __ne__(self, other):
+    def __ne__(self, other) -> bool:
         return not (self == other)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
@@ -40,7 +41,7 @@ class Chat(object):
             lib.dc_chat_unref
         )
 
-    def delete(self):
+    def delete(self) -> None:
         """Delete this chat and all its messages.
 
         Note:
@@ -50,24 +51,24 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
-    def block(self):
+    def block(self) -> None:
         """Block this chat."""
         lib.dc_block_chat(self.account._dc_context, self.id)
 
-    def accept(self):
+    def accept(self) -> None:
         """Accept this contact request chat."""
         lib.dc_accept_chat(self.account._dc_context, self.id)
 
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self):
+    def is_group(self) -> bool:
         """ return true if this chat is a group chat.
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
-    def is_muted(self):
+    def is_muted(self) -> bool:
         """ return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
@@ -90,7 +91,7 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self):
+    def can_send(self) -> bool:
         """Check if messages can be sent to a give chat.
         This is not true eg. for the contact requests or for the device-talk
 
@@ -98,30 +99,30 @@ class Chat(object):
         """
         return lib.dc_chat_can_send(self._dc_chat)
 
-    def is_protected(self):
+    def is_protected(self) -> bool:
         """ return True if this chat is a protected chat.
 
         :returns: True if chat is protected, False otherwise.
         """
         return lib.dc_chat_is_protected(self._dc_chat)
 
-    def get_name(self):
+    def get_name(self) -> Optional[str]:
         """ return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
-    def set_name(self, name):
+    def set_name(self, name: str) -> bool:
         """ set name of this chat.
 
         :param name: as a unicode string.
-        :returns: None
+        :returns: True on success, False otherwise
         """
         name = as_dc_charpointer(name)
-        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
+        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
 
-    def mute(self, duration=None):
+    def mute(self, duration: Optional[int] = None) -> None:
         """ mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
@@ -135,7 +136,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
-    def unmute(self):
+    def unmute(self) -> None:
         """ unmutes the chat
 
         :returns: None
@@ -144,7 +145,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
-    def get_mute_duration(self):
+    def get_mute_duration(self) -> int:
         """ Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
@@ -152,37 +153,37 @@ class Chat(object):
         """
         return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
 
-    def get_ephemeral_timer(self):
+    def get_ephemeral_timer(self) -> int:
         """ get ephemeral timer.
 
         :returns: ephemeral timer value in seconds
         """
         return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
 
-    def set_ephemeral_timer(self, timer):
+    def set_ephemeral_timer(self, timer: int) -> bool:
         """ set ephemeral timer.
 
         :param: timer value in seconds
 
-        :returns: None
+        :returns: True on success, False otherwise
         """
-        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
+        return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
 
-    def get_type(self):
+    def get_type(self) -> int:
         """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self):
+    def get_encryption_info(self) -> Optional[str]:
         """Return encryption info for this chat.
 
         :returns: a string with encryption preferences of all chat members"""
         res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
         return from_dc_charpointer(res)
 
-    def get_join_qr(self):
+    def get_join_qr(self) -> Optional[str]:
         """ get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -194,7 +195,7 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
-    def send_msg(self, msg):
+    def send_msg(self, msg: Message) -> Message:
         """send a message by using a ready Message object.
 
         :param msg: a :class:`deltachat.message.Message` instance
@@ -512,8 +513,9 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.utcfromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i)
+                timestamp=datetime.fromtimestamp(
+                    lib.dc_array_get_timestamp(dc_array, i),
+                    timezone.utc
                 ),
                 marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )

python/src/deltachat/const.py

@@ -1,7 +1,13 @@
+from typing import Any, List
+
 from .capi import lib
 
 
-for name in dir(lib):
+def __getattr__(name: str) -> Any:
     if name.startswith("DC_"):
-        globals()[name] = getattr(lib, name)
-del name
+        return getattr(lib, name)
+    return globals()[name]
+
+
+def __dir__() -> List[str]:
+    return sorted(name for name in dir(lib) if name.startswith("DC_"))

python/src/deltachat/contact.py

@@ -1,10 +1,12 @@
 """ Contact object. """
 
-from . import props
-from .cutil import from_dc_charpointer
-from .capi import lib, ffi
+from datetime import date, datetime, timezone
+from typing import Optional
+
+from . import const, props
+from .capi import ffi, lib
 from .chat import Chat
-from . import const
+from .cutil import from_dc_charpointer, from_optional_dc_charpointer
 
 
 class Contact(object):
@@ -35,18 +37,25 @@ class Contact(object):
         )
 
     @props.with_doc
-    def addr(self):
+    def addr(self) -> str:
         """ normalized e-mail address for this account. """
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self):
+    def name(self) -> str:
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
     # deprecated alias
     display_name = name
 
+    @props.with_doc
+    def last_seen(self) -> date:
+        """Last seen timestamp."""
+        return datetime.fromtimestamp(
+            lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc
+        )
+
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -67,15 +76,13 @@ class Contact(object):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
 
-    def get_profile_image(self):
+    def get_profile_image(self) -> Optional[str]:
         """Get contact profile image.
 
         :returns: path to profile image, None if no profile image exists.
         """
         dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
-        if dc_res == ffi.NULL:
-            return None
-        return from_dc_charpointer(dc_res)
+        return from_optional_dc_charpointer(dc_res)
 
     @property
     def status(self):

python/src/deltachat/cutil.py

@@ -1,6 +1,9 @@
 from .capi import lib
 from .capi import ffi
-from datetime import datetime
+from datetime import datetime, timezone
+from typing import Optional, TypeVar, Generator, Callable
+
+T = TypeVar('T')
 
 
 def as_dc_charpointer(obj):
@@ -11,21 +14,28 @@ def as_dc_charpointer(obj):
     return obj
 
 
-def iter_array(dc_array_t, constructor):
+def iter_array(dc_array_t, constructor: Callable[[int], T]) -> Generator[T, None, None]:
     for i in range(0, lib.dc_array_get_cnt(dc_array_t)):
         yield constructor(lib.dc_array_get_id(dc_array_t, i))
 
 
-def from_dc_charpointer(obj):
+def from_dc_charpointer(obj) -> str:
     if obj != ffi.NULL:
         return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    raise ValueError
+
+
+def from_optional_dc_charpointer(obj) -> Optional[str]:
+    if obj != ffi.NULL:
+        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    return None
 
 
 class DCLot:
-    def __init__(self, dc_lot):
+    def __init__(self, dc_lot) -> None:
         self._dc_lot = dc_lot
 
-    def id(self):
+    def id(self) -> int:
         return lib.dc_lot_get_id(self._dc_lot)
 
     def state(self):
@@ -44,4 +54,4 @@ class DCLot:
         ts = lib.dc_lot_get_timestamp(self._dc_lot)
         if ts == 0:
             return None
-        return datetime.utcfromtimestamp(ts)
+        return datetime.fromtimestamp(ts, timezone.utc)

python/src/deltachat/direct_imap.py

@@ -11,7 +11,7 @@ from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
 import imaplib
 import deltachat
-from deltachat import const
+from deltachat import const, Account
 
 
 SEEN = b'\\Seen'
@@ -62,7 +62,7 @@ def dc_account_after_shutdown(account):
 
 
 class DirectImap:
-    def __init__(self, account):
+    def __init__(self, account: Account) -> None:
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
         self._idling = False

python/src/deltachat/events.py

@@ -9,11 +9,11 @@ from .hookspec import account_hookimpl
 from contextlib import contextmanager
 from .capi import ffi, lib
 from .message import map_system_message
-from .cutil import from_dc_charpointer
+from .cutil import from_optional_dc_charpointer
 
 
 class FFIEvent:
-    def __init__(self, name, data1, data2):
+    def __init__(self, name: str, data1, data2):
         self.name = name
         self.data1 = data1
         self.data2 = data2
@@ -29,18 +29,18 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account):
+    def __init__(self, account) -> None:
         self.account = account
         self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
+    def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
         self.account.log(str(ffi_event))
 
     @account_hookimpl
     def ac_log_line(self, message):
-        t = threading.currentThread()
+        t = threading.current_thread()
         tname = getattr(t, "name", t)
         if tname == "MainThread":
             tname = "MAIN"
@@ -69,7 +69,7 @@ class FFIEventTracker:
         self._event_queue = Queue()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
+    def ac_process_ffi_event(self, ffi_event: FFIEvent):
         self._event_queue.put(ffi_event)
 
     def set_timeout(self, timeout):
@@ -96,7 +96,7 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_contains(self, regex):
+    def get_info_contains(self, regex: str) -> FFIEvent:
         rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
@@ -176,6 +176,7 @@ class FFIEventTracker:
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
+        return None
 
     def wait_msg_delivered(self, msg):
         ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
@@ -189,10 +190,10 @@ class EventThread(threading.Thread):
 
     With each Account init this callback thread is started.
     """
-    def __init__(self, account):
+    def __init__(self, account) -> None:
         self.account = account
         super(EventThread, self).__init__(name="events")
-        self.setDaemon(True)
+        self.daemon = True
         self._marked_for_shutdown = False
         self.start()
 
@@ -202,17 +203,17 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def mark_shutdown(self):
+    def mark_shutdown(self) -> None:
         self._marked_for_shutdown = True
 
-    def wait(self, timeout=None):
+    def wait(self, timeout=None) -> None:
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
         self.join(timeout=timeout)
 
-    def run(self):
+    def run(self) -> None:
         """ get and run events until shutdown. """
         with self.log_execution("EVENT THREAD"):
             self._inner_run()
@@ -234,7 +235,7 @@ class EventThread(threading.Thread):
             # function which provides us signature info of an event call
             evt_name = deltachat.get_dc_event_name(evt)
             if lib.dc_event_has_string_data(evt):
-                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
+                data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event))
             else:
                 data2 = lib.dc_event_get_data2_int(event)
 
@@ -250,7 +251,7 @@ class EventThread(threading.Thread):
                 if self.account._dc_context is not None:
                     raise
 
-    def _map_ffi_event(self, ffi_event):
+    def _map_ffi_event(self, ffi_event: FFIEvent):
         name = ffi_event.name
         account = self.account
         if name == "DC_EVENT_CONFIGURE_PROGRESS":

python/src/deltachat/message.py

@@ -3,10 +3,11 @@
 import os
 import re
 from . import props
-from .cutil import from_dc_charpointer, as_dc_charpointer
+from .cutil import from_dc_charpointer, from_optional_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
 from . import const
-from datetime import datetime
+from datetime import datetime, timezone
+from typing import Optional
 
 
 class Message(object):
@@ -75,7 +76,7 @@ class Message(object):
         return lib.dc_msg_get_id(self._dc_msg)
 
     @props.with_doc
-    def text(self):
+    def text(self) -> str:
         """unicode text of this messages (might be empty if not a text message). """
         return from_dc_charpointer(lib.dc_msg_get_text(self._dc_msg))
 
@@ -84,9 +85,9 @@ class Message(object):
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
     @props.with_doc
-    def html(self):
+    def html(self) -> str:
         """html text of this messages (might be empty if not an html message). """
-        return from_dc_charpointer(
+        return from_optional_dc_charpointer(
             lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
 
     def has_html(self):
@@ -113,12 +114,13 @@ class Message(object):
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
-    def basename(self):
+    def basename(self) -> str:
         """basename of the attachment if it exists, otherwise empty string. """
+        # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 
     @props.with_doc
-    def filemime(self):
+    def filemime(self) -> str:
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
@@ -130,7 +132,7 @@ class Message(object):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
-    def get_setupcodebegin(self):
+    def get_setupcodebegin(self) -> str:
         """ return the first characters of a setup code in a setup message. """
         return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
 
@@ -146,7 +148,7 @@ class Message(object):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
 
-    def get_message_info(self):
+    def get_message_info(self) -> str:
         """ Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
@@ -170,7 +172,7 @@ class Message(object):
         :returns: naive datetime.datetime() object.
         """
         ts = lib.dc_msg_get_timestamp(self._dc_msg)
-        return datetime.utcfromtimestamp(ts)
+        return datetime.fromtimestamp(ts, timezone.utc)
 
     @props.with_doc
     def time_received(self):
@@ -180,7 +182,7 @@ class Message(object):
         """
         ts = lib.dc_msg_get_received_timestamp(self._dc_msg)
         if ts:
-            return datetime.utcfromtimestamp(ts)
+            return datetime.fromtimestamp(ts, timezone.utc)
 
     @props.with_doc
     def ephemeral_timer(self):
@@ -200,14 +202,14 @@ class Message(object):
         """
         ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
         if ts:
-            return datetime.utcfromtimestamp(ts)
+            return datetime.fromtimestamp(ts, timezone.utc)
 
     @property
-    def quoted_text(self):
+    def quoted_text(self) -> Optional[str]:
         """Text inside the quote
 
         :returns: Quoted text"""
-        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
+        return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
 
     @property
     def quote(self):
@@ -223,6 +225,10 @@ class Message(object):
         """Quote setter"""
         lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
 
+    def force_plaintext(self) -> None:
+        """Force the message to be sent in plain text."""
+        lib.dc_msg_force_plaintext(self._dc_msg)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -240,9 +246,9 @@ class Message(object):
             return email.message_from_string(s)
 
     @property
-    def error(self):
+    def error(self) -> Optional[str]:
         """Error message"""
-        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
+        return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
 
     @property
     def chat(self):
@@ -255,12 +261,12 @@ class Message(object):
         return Chat(self.account, chat_id)
 
     @props.with_doc
-    def override_sender_name(self):
+    def override_sender_name(self) -> Optional[str]:
         """the name that should be shown over the message instead of the contact display name.
 
         Usually used to impersonate someone else.
         """
-        return from_dc_charpointer(
+        return from_optional_dc_charpointer(
             lib.dc_msg_get_override_sender_name(self._dc_msg))
 
     def set_override_sender_name(self, name):

python/src/deltachat/provider.py

@@ -14,7 +14,7 @@ class Provider(object):
     :param domain: The email to get the provider info for.
     """
 
-    def __init__(self, account, addr):
+    def __init__(self, account, addr) -> None:
         provider = ffi.gc(
             lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
             lib.dc_provider_unref,
@@ -24,19 +24,19 @@ class Provider(object):
         self._provider = provider
 
     @property
-    def overview_page(self):
+    def overview_page(self) -> str:
         """URL to the overview page of the provider on providers.delta.chat."""
         return from_dc_charpointer(
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self):
+    def get_before_login_hints(self) -> str:
         """Should be shown to the user on login."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hints(self._provider))
+            lib.dc_provider_get_before_login_hint(self._provider))
 
     @property
-    def status(self):
+    def status(self) -> int:
         """The status of the provider information.
 
         This is one of the

python/src/deltachat/testplugin.py

@@ -9,6 +9,7 @@ import fnmatch
 import time
 import weakref
 import tempfile
+from typing import List, Dict, Callable
 
 import pytest
 import requests
@@ -126,7 +127,7 @@ def pytest_report_header(config, startdir):
 
 
 class SessionLiveConfigFromFile:
-    def __init__(self, fn):
+    def __init__(self, fn) -> None:
         self.fn = fn
         self.configlist = []
         for line in open(fn):
@@ -137,19 +138,21 @@ class SessionLiveConfigFromFile:
                     d[name] = value
                 self.configlist.append(d)
 
-    def get(self, index):
+    def get(self, index: int):
         return self.configlist[index]
 
-    def exists(self):
+    def exists(self) -> bool:
         return bool(self.configlist)
 
 
 class SessionLiveConfigFromURL:
-    def __init__(self, url):
+    configlist: List[Dict[str, str]]
+
+    def __init__(self, url: str) -> None:
         self.configlist = []
         self.url = url
 
-    def get(self, index):
+    def get(self, index: int):
         try:
             return self.configlist[index]
         except IndexError:
@@ -162,7 +165,7 @@ class SessionLiveConfigFromURL:
             self.configlist.append(config)
             return config
 
-    def exists(self):
+    def exists(self) -> bool:
         return bool(self.configlist)
 
 
@@ -179,7 +182,7 @@ def session_liveconfig(request):
 @pytest.fixture
 def data(request):
     class Data:
-        def __init__(self):
+        def __init__(self) -> None:
             # trying to find test data heuristically
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
@@ -210,7 +213,10 @@ def data(request):
 def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
     class AccountMaker:
-        def __init__(self):
+        _finalizers: List[Callable[[], None]]
+        _accounts: List[Account]
+
+        def __init__(self) -> None:
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
@@ -297,21 +303,20 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 self._preconfigure_key(ac, configdict['addr'])
             return ac, dict(configdict)
 
-        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
+        def get_online_configuring_account(self, sentbox=False, move=False,
                                            pre_generated_key=True, quiet=False, config={}):
             ac, configdict = self.get_online_config(
                 pre_generated_key=pre_generated_key, quiet=quiet)
             configdict.update(config)
-            configdict["mvbox_watch"] = str(int(mvbox))
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
             ac._configtracker = ac.configure()
             return ac
 
-        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
+        def get_one_online_account(self, pre_generated_key=True, move=False):
             ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
+                pre_generated_key=pre_generated_key, move=move)
             self.wait_configure_and_start_io([ac1])
             return ac1
 
@@ -330,7 +335,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
-            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
+            """ Clones addr, mail_pw, mvbox_move, sentbox_watch and the
             direct_imap object of an online account. This simulates the user setting
             up a new device without importing a backup.
 
@@ -345,7 +350,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
-                mvbox_watch=account.get_config("mvbox_watch"),
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
@@ -423,7 +427,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                         pass
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
-        def get_accepted_chat(self, ac1, ac2):
+        def get_accepted_chat(self, ac1: Account, ac2: Account):
             ac2.create_chat(ac1)
             return ac1.create_chat(ac2)
 
@@ -451,7 +455,9 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
 
 class BotProcess:
-    def __init__(self, popen, bot_cfg):
+    stdout_queue: queue.Queue
+
+    def __init__(self, popen, bot_cfg) -> None:
         self.popen = popen
         self.addr = bot_cfg["addr"]
 
@@ -459,10 +465,10 @@ class BotProcess:
         # the (unicode) lines available for readers through a queue.
         self.stdout_queue = queue.Queue()
         self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.setDaemon(1)
+        t.daemon = True
         t.start()
 
-    def _run_stdout_thread(self):
+    def _run_stdout_thread(self) -> None:
         try:
             while 1:
                 line = self.popen.stdout.readline()
@@ -474,10 +480,10 @@ class BotProcess:
         finally:
             self.stdout_queue.put(None)
 
-    def kill(self):
+    def kill(self) -> None:
         self.popen.kill()
 
-    def wait(self, timeout=30):
+    def wait(self, timeout=30) -> None:
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
@@ -509,14 +515,14 @@ def tmp_db_path(tmpdir):
 @pytest.fixture
 def lp():
     class Printer:
-        def sec(self, msg):
+        def sec(self, msg: str) -> None:
             print()
             print("=" * 10, msg, "=" * 10)
 
-        def step(self, msg):
+        def step(self, msg: str) -> None:
             print("-" * 5, "step " + msg, "-" * 5)
 
-        def indent(self, msg):
+        def indent(self, msg: str) -> None:
             print("  " + msg)
 
     return Printer()

python/tests/test_account.py

@@ -10,7 +10,7 @@ from deltachat.tracker import ImexTracker
 from deltachat.hookspec import account_hookimpl
 from deltachat.capi import ffi, lib
 from deltachat.cutil import iter_array
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 
 @pytest.mark.parametrize("msgtext,res", [
@@ -57,7 +57,7 @@ class TestOfflineAccountBasic:
         alice_public = data.read_path("key/alice-public.asc")
         alice_secret = data.read_path("key/alice-secret.asc")
         assert alice_public and alice_secret
-        ac._preconfigure_keypair("alice@example.com", alice_public, alice_secret)
+        ac._preconfigure_keypair("alice@example.org", alice_public, alice_secret)
 
     def test_getinfo(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
@@ -265,23 +265,23 @@ class TestOfflineChat:
         assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft()
 
     def test_group_chat_creation_with_translation(self, ac1):
-        ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %1$s")
+        ac1.set_stock_translation(const.DC_STR_MSGGRPNAME, "abc %1$s xyz %2$s")
         ac1._evtracker.consume_events()
         with pytest.raises(ValueError):
-            ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %2$s")
+            ac1.set_stock_translation(const.DC_STR_FILE, "xyz %1$s")
+        ac1._evtracker.get_matching("DC_EVENT_WARNING")
+        with pytest.raises(ValueError):
+            ac1.set_stock_translation(const.DC_STR_CONTACT_NOT_VERIFIED, "xyz %2$s")
         ac1._evtracker.get_matching("DC_EVENT_WARNING")
         with pytest.raises(ValueError):
             ac1.set_stock_translation(500, "xyz %1$s")
         ac1._evtracker.get_matching("DC_EVENT_WARNING")
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact2 = ac1.create_contact("some2@example.org", name="some2")
-        chat = ac1.create_group_chat(name="title1", contacts=[contact1, contact2])
-        assert chat.get_name() == "title1"
-        assert contact1 in chat.get_contacts()
-        assert contact2 in chat.get_contacts()
-        assert not chat.is_promoted()
-        msg = chat.get_draft()
-        assert msg.text == "xyz title1"
+        chat = ac1.create_group_chat(name="homework", contacts=[])
+        assert chat.get_name() == "homework"
+        chat.send_text("Now we have a group for homework")
+        assert chat.is_promoted()
+        chat.set_name("Homework")
+        assert chat.get_messages()[-1].text == "abc homework xyz Homework by me."
 
     @pytest.mark.parametrize("verified", [True, False])
     def test_group_chat_qr(self, acfactory, ac1, verified):
@@ -447,7 +447,7 @@ class TestOfflineChat:
         contact1.create_chat().send_text("hello")
 
     def test_chat_message_distinctions(self, ac1, chat1):
-        past1s = datetime.utcnow() - timedelta(seconds=1)
+        past1s = datetime.now(timezone.utc) - timedelta(seconds=1)
         msg = chat1.send_text("msg1")
         ts = msg.time_sent
         assert msg.time_received is None
@@ -860,7 +860,7 @@ class TestOnlineAccount:
 
     def test_mvbox_sentbox_threads(self, acfactory, lp):
         lp.sec("ac1: start with mvbox thread")
-        ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True, sentbox=True)
+        ac1 = acfactory.get_online_configuring_account(move=True, sentbox=True)
 
         lp.sec("ac2: start without mvbox/sentbox threads")
         ac2 = acfactory.get_online_configuring_account()
@@ -874,16 +874,20 @@ class TestOnlineAccount:
 
     def test_move_works(self, acfactory):
         ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account(mvbox=True, move=True)
+        ac2 = acfactory.get_online_configuring_account(move=True)
         acfactory.wait_configure_and_start_io()
         chat = acfactory.get_accepted_chat(ac1, ac2)
         chat.send_text("message1")
-        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
-        assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL
+
+        # Message is moved to the movebox
         ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
+        # Message is downloaded
+        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
+        assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL
+
     def test_move_works_on_self_sent(self, acfactory):
-        ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True)
+        ac1 = acfactory.get_online_configuring_account(move=True)
         ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure_and_start_io()
         ac1.set_config("bcc_self", "1")
@@ -953,7 +957,7 @@ class TestOnlineAccount:
         assert msg_in.is_forwarded()
 
     def test_send_self_message(self, acfactory, lp):
-        ac1 = acfactory.get_one_online_account(mvbox=True, move=True)
+        ac1 = acfactory.get_one_online_account(move=True)
         lp.sec("ac1: create self chat")
         chat = ac1.get_self_contact().create_chat()
         chat.send_text("hello")
@@ -973,7 +977,7 @@ class TestOnlineAccount:
         ac1._evtracker.wait_msg_delivered(msg1)
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         assert msg2.text == "message1"
         assert not msg2.is_forwarded()
         assert msg2.get_sender_contact().display_name == ac1.get_config("displayname")
@@ -983,7 +987,8 @@ class TestOnlineAccount:
         assert msg2 in chat2.get_messages()
         assert chat2.is_contact_request()
         assert chat2.count_fresh_messages() == 1
-        assert msg2.time_received >= msg1.time_sent
+        # Like it or not, this assert is flaky
+        # assert msg2.time_received >= msg1.time_sent
 
         lp.sec("create new chat with contact and verify it's proper")
         chat2b = msg2.create_chat()
@@ -1034,27 +1039,34 @@ class TestOnlineAccount:
 
     def test_moved_markseen(self, acfactory, lp):
         """Test that message already moved to DeltaChat folder is marked as seen."""
-        ac1 = acfactory.get_online_configuring_account(mvbox=True, config={"inbox_watch": "0"})
-        ac2 = acfactory.get_online_configuring_account()
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account(move=True)
         acfactory.wait_configure_and_start_io([ac1, ac2])
-        ac1.set_config("bcc_self", "1")
 
-        ac1.direct_imap.idle_start()
+        ac2.stop_io()
+        ac2.direct_imap.idle_start()
+
         ac1.create_chat(ac2).send_text("Hello!")
-        ac1.direct_imap.idle_check(terminate=True)
-        ac1.stop_io()
+
+        # Wait for the message to arrive.
+        ac2.direct_imap.idle_check(terminate=True)
 
         # Emulate moving of the message to DeltaChat folder by Sieve rule.
         # mailcow server contains this rule by default.
-        ac1.direct_imap.conn.move(["*"], "DeltaChat")
+        ac2.direct_imap.conn.move(["*"], "DeltaChat")
 
-        ac1.direct_imap.select_folder("DeltaChat")
-        ac1.direct_imap.idle_start()
-        ac1.start_io()
-        ac1.direct_imap.idle_wait_for_seen()
-        ac1.direct_imap.idle_done()
+        ac2.direct_imap.select_folder("DeltaChat")
+        ac2.direct_imap.idle_start()
+        ac2.start_io()
+        msg = ac2._evtracker.wait_next_incoming_message()
 
-        fetch = list(ac1.direct_imap.conn.fetch("*", b'FLAGS').values())
+        # Accept the contact request.
+        msg.chat.accept()
+        ac2.mark_seen_messages([msg])
+        ac2.direct_imap.idle_wait_for_seen()
+        ac2.direct_imap.idle_done()
+
+        fetch = list(ac2.direct_imap.conn.fetch("*", b'FLAGS').values())
         flags = fetch[-1][b'FLAGS']
         is_seen = b'\\Seen' in flags
         assert is_seen
@@ -1095,8 +1107,8 @@ class TestOnlineAccount:
     def test_markseen_message_and_mdn(self, acfactory, mvbox_move):
         # Please only change this test if you are very sure that it will still catch the issues it catches now.
         # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
-        ac1 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
-        ac2 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        ac1 = acfactory.get_online_configuring_account(move=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account(move=mvbox_move)
 
         acfactory.wait_configure_and_start_io()
         # Do not send BCC to self, we only want to test MDN on ac1.
@@ -1125,7 +1137,7 @@ class TestOnlineAccount:
         group1.add_contact(ac2)
         group1.send_text("hello")
 
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         group2 = msg2.create_chat()
         assert group2.get_name() == group1.get_name()
 
@@ -1142,7 +1154,7 @@ class TestOnlineAccount:
         assert not msg_reply1.chat.is_group()
         assert msg_reply1.chat.id == private_chat1.id
 
-    def test_mdn_asymetric(self, acfactory, lp):
+    def test_mdn_asymmetric(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts(move=True)
 
         lp.sec("ac1: create chat with ac2")
@@ -1166,6 +1178,9 @@ class TestOnlineAccount:
 
         assert len(msg.chat.get_messages()) == 1
 
+        ac1.direct_imap.select_config_folder("mvbox")
+        ac1.direct_imap.idle_start()
+
         lp.sec("ac2: mark incoming message as seen")
         ac2.mark_seen_messages([msg])
 
@@ -1175,6 +1190,9 @@ class TestOnlineAccount:
 
         assert len(chat.get_messages()) == 1
 
+        # Wait for the message to be marked as seen on IMAP.
+        assert ac1.direct_imap.idle_wait_for_seen()
+
         # MDN is received even though MDNs are already disabled
         assert msg_out.is_out_mdn_received()
 
@@ -1188,7 +1206,7 @@ class TestOnlineAccount:
         chat.send_text("message1")
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         assert msg2.text == "message1"
 
         lp.sec("create new chat with contact and send back (encrypted) message")
@@ -1221,6 +1239,40 @@ class TestOnlineAccount:
         assert not msg.is_encrypted()
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    def test_gossip_optimization(self, acfactory, lp):
+        """Test that gossip timestamp is updated when someone else sends gossip,
+        so we don't have to send gossip ourselves.
+        """
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+
+        acfactory.introduce_each_other([ac1, ac2])
+        acfactory.introduce_each_other([ac2, ac3])
+
+        lp.sec("ac1 creates a group chat with ac2")
+        group_chat = ac1.create_group_chat("hello")
+        group_chat.add_contact(ac2)
+        msg = group_chat.send_text("hi")
+
+        # No Autocrypt gossip was sent yet.
+        gossiped_timestamp = msg.chat.get_summary()["gossiped_timestamp"]
+        assert gossiped_timestamp == 0
+
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+        assert msg.text == "hi"
+
+        lp.sec("ac2 adds ac3 to the group")
+        msg.chat.add_contact(ac3)
+
+        lp.sec("ac1 receives message from ac2 and updates gossip timestamp")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+
+        # ac1 has updated the gossip timestamp even though no gossip was sent by ac1.
+        # ac1 does not need to send gossip because ac2 already did it.
+        gossiped_timestamp = msg.chat.get_summary()["gossiped_timestamp"]
+        assert gossiped_timestamp == int(msg.time_sent.timestamp())
+
     def test_gossip_encryption_preference(self, acfactory, lp):
         """Test that encryption preference of group members is gossiped to new members.
         This is a Delta Chat extension to Autocrypt 1.1.0, which Autocrypt-Gossip headers
@@ -1325,7 +1377,7 @@ class TestOnlineAccount:
         If the draft email is sent out later (i.e. moved to "Sent"), it must be shown."""
         ac1 = acfactory.get_online_configuring_account()
         ac1.set_config("show_emails", "2")
-        ac1.create_contact("alice@example.com").create_chat()
+        ac1.create_contact("alice@example.org").create_chat()
 
         acfactory.wait_configure(ac1)
         ac1.direct_imap.create_folder("Drafts")
@@ -1339,7 +1391,7 @@ class TestOnlineAccount:
         ac1.direct_imap.append("Drafts", """
             From: ac1 <{}>
             Subject: subj
-            To: alice@example.com
+            To: alice@example.org
             Message-ID: <aepiors@example.org>
             Content-Type: text/plain; charset=utf-8
 
@@ -1348,7 +1400,7 @@ class TestOnlineAccount:
         ac1.direct_imap.append("Sent", """
             From: ac1 <{}>
             Subject: subj
-            To: alice@example.com
+            To: alice@example.org
             Message-ID: <hsabaeni@example.org>
             Content-Type: text/plain; charset=utf-8
 
@@ -1379,6 +1431,35 @@ class TestOnlineAccount:
         assert msg2.text == "subj – message in Drafts that is moved to Sent later"
         assert len(msg.chat.get_messages()) == 2
 
+    def test_no_old_msg_is_fresh(self, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        ac1_clone = acfactory.clone_online_account(ac1)
+        acfactory.wait_configure_and_start_io()
+
+        ac1.set_config("e2ee_enabled", "0")
+        ac1_clone.set_config("e2ee_enabled", "0")
+        ac2.set_config("e2ee_enabled", "0")
+
+        ac1_clone.set_config("bcc_self", "1")
+
+        ac1.create_chat(ac2)
+        ac1_clone.create_chat(ac2)
+
+        lp.sec("Send a first message from ac2 to ac1 and check that it's 'fresh'")
+        first_msg_id = ac2.create_chat(ac1).send_text("Hi")
+        ac1._evtracker.wait_next_incoming_message()
+        assert ac1.create_chat(ac2).count_fresh_messages() == 1
+        assert len(list(ac1.get_fresh_messages())) == 1
+
+        lp.sec("Send a message from ac1_clone to ac2 and check that ac1 marks the first message as 'noticed'")
+        ac1_clone.create_chat(ac2).send_text("Hi back")
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+
+        assert ev.data1 == first_msg_id.id
+        assert ac1.create_chat(ac2).count_fresh_messages() == 0
+        assert len(list(ac1.get_fresh_messages())) == 0
+
     def test_prefer_encrypt(self, acfactory, lp):
         """Test quorum rule for encryption preference in 1:1 and group chat."""
         ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
@@ -1468,7 +1549,7 @@ class TestOnlineAccount:
         assert not msg1.is_encrypted()
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         assert msg2.text == "message1"
         assert not msg2.is_encrypted()
 
@@ -1517,7 +1598,7 @@ class TestOnlineAccount:
         chat1.send_text("hi")
 
         lp.sec("ac2 receives contact request from ac1")
-        received_message = ac2._evtracker.wait_next_messages_changed()
+        received_message = ac2._evtracker.wait_next_incoming_message()
         assert received_message.text == "hi"
 
         basename = "attachment.txt"
@@ -1552,7 +1633,7 @@ class TestOnlineAccount:
         assert msg_out.get_mime_headers() is None
 
         lp.sec("wait for ac2 to receive message")
-        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
         in_id = ev.data2
         mime = ac2.get_message_by_id(in_id).get_mime_headers()
         assert mime.get_all("From")
@@ -1851,7 +1932,7 @@ class TestOnlineAccount:
         ac1.create_chat(ac2).send_text("with avatar!")
 
         lp.sec("ac2: wait for receiving message and avatar from ac1")
-        msg2 = ac2._evtracker.wait_next_messages_changed()
+        msg2 = ac2._evtracker.wait_next_incoming_message()
         assert msg2.chat.is_contact_request()
         received_path = msg2.get_sender_contact().get_profile_image()
         assert open(received_path, "rb").read() == open(p, "rb").read()
@@ -2043,10 +2124,8 @@ class TestOnlineAccount:
         assert msg_back.chat == chat
         assert chat.get_profile_image() is None
 
-    @pytest.mark.parametrize("inbox_watch", ["0", "1"])
-    def test_connectivity(self, acfactory, lp, inbox_watch):
+    def test_connectivity(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-        ac1.set_config("inbox_watch", inbox_watch)
         ac1.set_config("scan_all_folders_debounce_secs", "0")
 
         ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTED)
@@ -2163,7 +2242,7 @@ class TestOnlineAccount:
                 break  # DC is done with reading messages
 
     def test_send_receive_locations(self, acfactory, lp):
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         lp.sec("ac1: create chat with ac2")
@@ -2286,7 +2365,7 @@ class TestOnlineAccount:
 
     def test_immediate_autodelete(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account(mvbox=False, move=False, sentbox=False)
+        ac2 = acfactory.get_online_configuring_account(move=False, sentbox=False)
 
         # "1" means delete immediately, while "0" means do not delete
         ac2.set_config("delete_server_after", "1")
@@ -2547,31 +2626,26 @@ class TestOnlineAccount:
         assert received_reply.quoted_text == "hello"
         assert received_reply.quote.id == out_msg.id
 
-    @pytest.mark.parametrize("folder,move,expected_destination,inbox_watch,", [
-        ("xyz", False, "xyz", "1"),  # Test that emails are recognized in a random folder but not moved
-        ("xyz", True, "DeltaChat", "1"),  # ...emails are found in a random folder and moved to DeltaChat
-        ("Spam", False, "INBOX", "1"),  # ...emails are moved from the spam folder to the Inbox
-        ("INBOX", False, "INBOX", "0"),  # ...emails are found in the `Inbox` folder even if `inbox_watch` is "0"
+    @pytest.mark.parametrize("folder,move,expected_destination,", [
+        ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
+        ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
+        ("Spam", False, "INBOX"),  # ...emails are moved from the spam folder to the Inbox
     ])
     # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
     # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
-    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination, inbox_watch):
+    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination):
         """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
         variant = folder + "-" + str(move) + "-" + expected_destination
         lp.sec("Testing variant " + variant)
         ac1 = acfactory.get_online_configuring_account(move=move)
         ac2 = acfactory.get_online_configuring_account()
-        ac1.set_config("inbox_watch", inbox_watch)
 
         acfactory.wait_configure(ac1)
         ac1.direct_imap.create_folder(folder)
 
         acfactory.wait_configure_and_start_io()
         # Wait until each folder was selected once and we are IDLEing:
-        if inbox_watch == "1":
-            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
-        else:
-            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
         ac1.stop_io()
 
         # Send a message to ac1 and move it to the mvbox:
@@ -2587,11 +2661,7 @@ class TestOnlineAccount:
         msg = ac1._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        # Wait until the message was moved (if at all) and we are IDLEing again:
-        if inbox_watch == "1":
-            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
-        else:
-            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
+        # The message has been downloaded, which means it has reached its destination.
         ac1.direct_imap.select_folder(expected_destination)
         assert len(ac1.direct_imap.get_all_messages()) == 1
         if folder != expected_destination:
@@ -2614,7 +2684,7 @@ class TestOnlineAccount:
             if mvbox_move:
                 assert ac.get_config("configured_mvbox_folder")
 
-        ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
+        ac1 = acfactory.get_online_configuring_account(move=mvbox_move)
         ac1.set_config("sentbox_move", "1")
         ac2 = acfactory.get_online_configuring_account()
 
@@ -2681,7 +2751,7 @@ class TestOnlineAccount:
 
         lp.sec("receive a message")
         ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
-        ac1._evtracker.wait_next_messages_changed()
+        ac1._evtracker.wait_next_incoming_message()
 
         lp.sec("send out message with bcc to ourselves")
         ac1.direct_imap.idle_start()
@@ -2712,6 +2782,22 @@ class TestOnlineAccount:
         # We can't decrypt the message in this chat, so the chat is empty:
         assert len(private_messages) == 0
 
+    def test_delete_deltachat_folder(self, acfactory):
+        """Test that DeltaChat folder is recreated if user deletes it manually."""
+        ac1 = acfactory.get_online_configuring_account(move=True)
+        ac2 = acfactory.get_online_configuring_account()
+        acfactory.wait_configure(ac1)
+
+        ac1.direct_imap.conn.delete_folder("DeltaChat")
+        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 0
+        acfactory.wait_configure_and_start_io()
+
+        ac2.create_chat(ac1).send_text("hello")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 1
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):
@@ -2826,7 +2912,8 @@ class TestGroupStressTests:
         lp.sec("ac2: check that ac3 is removed")
         msg = ac2._evtracker.wait_next_incoming_message()
 
-        assert msg.chat.num_contacts() == chat.num_contacts()
+        assert chat.num_contacts() == 2
+        assert msg.chat.num_contacts() == 2
         acfactory.dump_imap_summary(sys.stdout)
 
 

python/tests/test_increation.py

@@ -6,8 +6,6 @@ import shutil
 import pytest
 from filecmp import cmp
 
-from deltachat import const
-
 
 def wait_msg_delivered(account, msg_list):
     """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
@@ -102,14 +100,10 @@ class TestOnlineInCreation:
         ])
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1.data2)
+        received_original = ac2._evtracker.wait_next_incoming_message()
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2.data1 != ev1.data1
-        received_copy = ac2.get_message_by_id(ev2.data2)
+        received_copy = ac2._evtracker.wait_next_incoming_message()
+        assert received_copy.id != received_original.id
         assert cmp(received_copy.filename, orig, shallow=False)

python/tox.ini

@@ -1,7 +1,9 @@
 [tox]
+isolated_build = true
 envlist =
     py3
     lint
+    mypy
     auditwheels
 
 [testenv]
@@ -42,12 +44,20 @@ commands =
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
+[testenv:mypy]
+deps =
+    mypy
+    typing
+    types-setuptools
+    types-requests
+commands =
+    mypy --no-incremental src/
+
 [testenv:doc]
 changedir=doc
 deps =
-    # Pin dependencies to the versions which actually work with Python 3.5.
-    sphinx==3.4.3
-    breathe==4.28.0
+    sphinx
+    breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 

scripts/coverage.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-if ! which grcov 2>/dev/null 1>&2; then
+if ! command -v grcov >/dev/null; then
     echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
     echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
     exit 1

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -2,13 +2,13 @@
 
 set -x -e
 
-# we use the python3.6 environment as the base environment 
-/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel 
+# we use the python3.7 environment as the base environment 
+/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel 
 
 pushd /usr/bin
 
-ln -s /opt/_internal/cpython-3.6.*/bin/tox
-ln -s /opt/_internal/cpython-3.6.*/bin/devpi
-ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
+ln -s /opt/_internal/cpython-3.7.*/bin/tox
+ln -s /opt/_internal/cpython-3.7.*/bin/devpi
+ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
 
 popd

scripts/docker-coredeps/deps/build_python.sh

@@ -2,13 +2,13 @@
 
 set -x -e
 
-# we use the python3.6 environment as the base environment
-/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel
+# we use the python3.7 environment as the base environment
+/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel
 
 pushd /usr/bin
 
-ln -s /opt/_internal/cpython-3.6.*/bin/tox
-ln -s /opt/_internal/cpython-3.6.*/bin/devpi
-ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
+ln -s /opt/_internal/cpython-3.7.*/bin/tox
+ln -s /opt/_internal/cpython-3.7.*/bin/devpi
+ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
 
 popd

scripts/run_all.sh

@@ -19,11 +19,9 @@ export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp36-cp36m/bin
+export PATH=$PATH:/opt/python/cp37-cp37m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.6
-ln -s /opt/python/cp36-cp36m/bin/python3.6
 rm -f python3.7
 ln -s /opt/python/cp37-cp37m/bin/python3.7
 rm -f python3.8

scripts/set_core_version.py

@@ -86,6 +86,8 @@ def main():
     print("")
     print("   git tag -a {}".format(newversion))
     print("   git push origin {}".format(newversion))
+    print("   git tag -a py-{}".format(newversion))
+    print("   git push origin py-{}".format(newversion))
     print("")
 
 

src/accounts.rs

@@ -1,6 +1,8 @@
+//! # Account manager module.
+
 use std::collections::BTreeMap;
 
-use async_std::channel::{Receiver, Sender};
+use async_std::channel::{self, Receiver, Sender};
 use async_std::fs;
 use async_std::path::PathBuf;
 use async_std::prelude::*;
@@ -11,34 +13,37 @@ use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
-use crate::events::Event;
+use crate::events::{Event, EventType, Events};
 
 /// Account manager, that can handle multiple accounts in a single place.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct Accounts {
     dir: PathBuf,
     config: Config,
-    accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
+    accounts: BTreeMap<u32, Context>,
     emitter: EventEmitter,
+
+    /// Event channel to emit account manager errors.
+    events: Events,
 }
 
 impl Accounts {
     /// Loads or creates an accounts folder at the given `dir`.
-    pub async fn new(os_name: String, dir: PathBuf) -> Result<Self> {
+    pub async fn new(dir: PathBuf) -> Result<Self> {
         if !dir.exists().await {
-            Accounts::create(os_name, &dir).await?;
+            Accounts::create(&dir).await?;
         }
 
         Accounts::open(dir).await
     }
 
     /// Creates a new default structure.
-    pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
+    pub async fn create(dir: &PathBuf) -> Result<()> {
         fs::create_dir_all(dir)
             .await
             .context("failed to create folder")?;
 
-        Config::new(os_name.clone(), dir).await?;
+        Config::new(dir).await?;
 
         Ok(())
     }
@@ -55,6 +60,11 @@ impl Accounts {
         let accounts = config.load_accounts().await?;
 
         let emitter = EventEmitter::new();
+
+        let events = Events::default();
+
+        emitter.sender.send(events.get_emitter()).await?;
+
         for account in accounts.values() {
             emitter.add_account(account).await?;
         }
@@ -62,20 +72,21 @@ impl Accounts {
         Ok(Self {
             dir,
             config,
-            accounts: Arc::new(RwLock::new(accounts)),
+            accounts,
             emitter,
+            events,
         })
     }
 
     /// Get an account by its `id`:
     pub async fn get_account(&self, id: u32) -> Option<Context> {
-        self.accounts.read().await.get(&id).cloned()
+        self.accounts.get(&id).cloned()
     }
 
     /// Get the currently selected account.
     pub async fn get_selected_account(&self) -> Option<Context> {
         let id = self.config.get_selected_account().await;
-        self.accounts.read().await.get(&id).cloned()
+        self.accounts.get(&id).cloned()
     }
 
     /// Returns the currently selected account's id or None if no account is selected.
@@ -87,27 +98,26 @@ impl Accounts {
     }
 
     /// Select the given account.
-    pub async fn select_account(&self, id: u32) -> Result<()> {
+    pub async fn select_account(&mut self, id: u32) -> Result<()> {
         self.config.select_account(id).await?;
 
         Ok(())
     }
 
     /// Add a new account.
-    pub async fn add_account(&self) -> Result<u32> {
-        let os_name = self.config.os_name().await;
+    pub async fn add_account(&mut self) -> Result<u32> {
         let account_config = self.config.new_account(&self.dir).await?;
 
-        let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
+        let ctx = Context::new(account_config.dbfile().into(), account_config.id).await?;
         self.emitter.add_account(&ctx).await?;
-        self.accounts.write().await.insert(account_config.id, ctx);
+        self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
     }
 
     /// Remove an account.
-    pub async fn remove_account(&self, id: u32) -> Result<()> {
-        let ctx = self.accounts.write().await.remove(&id);
+    pub async fn remove_account(&mut self, id: u32) -> Result<()> {
+        let ctx = self.accounts.remove(&id);
         ensure!(ctx.is_some(), "no account with this id: {}", id);
         let ctx = ctx.unwrap();
         ctx.stop_io().await;
@@ -124,7 +134,7 @@ impl Accounts {
     }
 
     /// Migrate an existing account into this structure.
-    pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
+    pub async fn migrate_account(&mut self, dbfile: PathBuf) -> Result<u32> {
         let blobdir = Context::derive_blobdir(&dbfile);
         let walfile = Context::derive_walfile(&dbfile);
 
@@ -172,15 +182,9 @@ impl Accounts {
 
         match res {
             Ok(_) => {
-                let ctx = Context::with_blobdir(
-                    self.config.os_name().await,
-                    new_dbfile,
-                    new_blobdir,
-                    account_config.id,
-                )
-                .await?;
+                let ctx = Context::with_blobdir(new_dbfile, new_blobdir, account_config.id).await?;
                 self.emitter.add_account(&ctx).await?;
-                self.accounts.write().await.insert(account_config.id, ctx);
+                self.accounts.insert(account_config.id, ctx);
                 Ok(account_config.id)
             }
             Err(err) => {
@@ -201,7 +205,7 @@ impl Accounts {
 
     /// Get a list of all account ids.
     pub async fn get_all(&self) -> Vec<u32> {
-        self.accounts.read().await.keys().copied().collect()
+        self.accounts.keys().copied().collect()
     }
 
     /// This is meant especially for iOS, because iOS needs to tell the system when its background work is done.
@@ -215,7 +219,7 @@ impl Accounts {
     /// - while dc_accounts_all_work_done() returns false:
     ///   -  Wait for DC_EVENT_CONNECTIVITY_CHANGED
     pub async fn all_work_done(&self) -> bool {
-        for account in self.accounts.read().await.values() {
+        for account in self.accounts.values() {
             if !account.all_work_done().await {
                 return false;
             }
@@ -224,39 +228,41 @@ impl Accounts {
     }
 
     pub async fn start_io(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
+        for account in self.accounts.values() {
             account.start_io().await;
         }
     }
 
     pub async fn stop_io(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
+        for account in self.accounts.values() {
             account.stop_io().await;
         }
     }
 
     pub async fn maybe_network(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
+        for account in self.accounts.values() {
             account.maybe_network().await;
         }
     }
 
     pub async fn maybe_network_lost(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
+        for account in self.accounts.values() {
             account.maybe_network_lost().await;
         }
     }
 
-    /// Unified event emitter.
+    /// Emits a single event.
+    pub fn emit_event(&self, event: EventType) {
+        self.events.emit(Event { id: 0, typ: event })
+    }
+
+    /// Returns unified event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
         self.emitter.clone()
     }
 }
 
+/// Unified event emitter for multiple accounts.
 #[derive(Debug, Clone)]
 pub struct EventEmitter {
     /// Aggregate stream of events from all accounts.
@@ -271,7 +277,7 @@ pub struct EventEmitter {
 
 impl EventEmitter {
     pub fn new() -> Self {
-        let (sender, receiver) = async_std::channel::unbounded();
+        let (sender, receiver) = channel::unbounded();
         Self {
             stream: Arc::new(RwLock::new(futures::stream::SelectAll::new())),
             sender,
@@ -324,15 +330,18 @@ impl async_std::stream::Stream for EventEmitter {
 pub const CONFIG_NAME: &str = "accounts.toml";
 pub const DB_NAME: &str = "dc.db";
 
-#[derive(Debug, Clone)]
+/// Account manager configuration file.
+#[derive(Debug, Clone, PartialEq)]
 pub struct Config {
     file: PathBuf,
-    inner: Arc<RwLock<InnerConfig>>,
+    inner: InnerConfig,
 }
 
+/// Account manager configuration file contents.
+///
+/// This is serialized into TOML.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 struct InnerConfig {
-    pub os_name: String,
     /// The currently selected account.
     pub selected_account: u32,
     pub next_id: u32,
@@ -340,15 +349,15 @@ struct InnerConfig {
 }
 
 impl Config {
-    pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
+    pub async fn new(dir: &PathBuf) -> Result<Self> {
+        let inner = InnerConfig {
+            accounts: Vec::new(),
+            selected_account: 0,
+            next_id: 1,
+        };
         let cfg = Config {
             file: dir.join(CONFIG_NAME),
-            inner: Arc::new(RwLock::new(InnerConfig {
-                os_name,
-                accounts: Vec::new(),
-                selected_account: 0,
-                next_id: 1,
-            })),
+            inner,
         };
 
         cfg.sync().await?;
@@ -356,18 +365,11 @@ impl Config {
         Ok(cfg)
     }
 
-    pub async fn os_name(&self) -> String {
-        self.inner.read().await.os_name.clone()
-    }
-
     /// Sync the inmemory representation to disk.
     async fn sync(&self) -> Result<()> {
-        fs::write(
-            &self.file,
-            toml::to_string_pretty(&*self.inner.read().await)?,
-        )
-        .await
-        .context("failed to write config")
+        fs::write(&self.file, toml::to_string_pretty(&self.inner)?)
+            .await
+            .context("failed to write config")
     }
 
     /// Read a configuration from the given file into memory.
@@ -375,22 +377,13 @@ impl Config {
         let bytes = fs::read(&file).await.context("failed to read file")?;
         let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
 
-        Ok(Config {
-            file,
-            inner: Arc::new(RwLock::new(inner)),
-        })
+        Ok(Config { file, inner })
     }
 
     pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
-        let cfg = &*self.inner.read().await;
         let mut accounts = BTreeMap::new();
-        for account_config in &cfg.accounts {
-            let ctx = Context::new(
-                cfg.os_name.clone(),
-                account_config.dbfile().into(),
-                account_config.id,
-            )
-            .await?;
+        for account_config in &self.inner.accounts {
+            let ctx = Context::new(account_config.dbfile().into(), account_config.id).await?;
             accounts.insert(account_config.id, ctx);
         }
 
@@ -398,19 +391,18 @@ impl Config {
     }
 
     /// Create a new account in the given root directory.
-    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
+    async fn new_account(&mut self, dir: &PathBuf) -> Result<AccountConfig> {
         let id = {
-            let inner = &mut self.inner.write().await;
-            let id = inner.next_id;
+            let id = self.inner.next_id;
             let uuid = Uuid::new_v4();
             let target_dir = dir.join(uuid.to_simple_ref().to_string());
 
-            inner.accounts.push(AccountConfig {
+            self.inner.accounts.push(AccountConfig {
                 id,
                 dir: target_dir.into(),
                 uuid,
             });
-            inner.next_id += 1;
+            self.inner.next_id += 1;
             id
         };
 
@@ -422,46 +414,39 @@ impl Config {
     }
 
     /// Removes an existing acccount entirely.
-    pub async fn remove_account(&self, id: u32) -> Result<()> {
+    pub async fn remove_account(&mut self, id: u32) -> Result<()> {
         {
-            let inner = &mut *self.inner.write().await;
-            if let Some(idx) = inner.accounts.iter().position(|e| e.id == id) {
+            if let Some(idx) = self.inner.accounts.iter().position(|e| e.id == id) {
                 // remove account from the configs
-                inner.accounts.remove(idx);
+                self.inner.accounts.remove(idx);
             }
-            if inner.selected_account == id {
+            if self.inner.selected_account == id {
                 // reset selected account
-                inner.selected_account = inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+                self.inner.selected_account =
+                    self.inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
             }
         }
 
         self.sync().await
     }
 
-    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
-        self.inner
-            .read()
-            .await
-            .accounts
-            .iter()
-            .find(|e| e.id == id)
-            .cloned()
+    async fn get_account(&self, id: u32) -> Option<AccountConfig> {
+        self.inner.accounts.iter().find(|e| e.id == id).cloned()
     }
 
     pub async fn get_selected_account(&self) -> u32 {
-        self.inner.read().await.selected_account
+        self.inner.selected_account
     }
 
-    pub async fn select_account(&self, id: u32) -> Result<()> {
+    pub async fn select_account(&mut self, id: u32) -> Result<()> {
         {
-            let inner = &mut *self.inner.write().await;
             ensure!(
-                inner.accounts.iter().any(|e| e.id == id),
+                self.inner.accounts.iter().any(|e| e.id == id),
                 "invalid account id: {}",
                 id
             );
 
-            inner.selected_account = id;
+            self.inner.selected_account = id;
         }
 
         self.sync().await?;
@@ -469,8 +454,9 @@ impl Config {
     }
 }
 
+/// Configuration of a single account.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-pub struct AccountConfig {
+struct AccountConfig {
     /// Unique id.
     pub id: u32,
     /// Root directory for all data for this account.
@@ -494,23 +480,17 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts1").into();
 
-        let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let mut accounts1 = Accounts::new(p.clone()).await.unwrap();
         accounts1.add_account().await.unwrap();
 
         let accounts2 = Accounts::open(p).await.unwrap();
 
-        assert_eq!(accounts1.accounts.read().await.len(), 1);
+        assert_eq!(accounts1.accounts.len(), 1);
         assert_eq!(accounts1.config.get_selected_account().await, 1);
 
         assert_eq!(accounts1.dir, accounts2.dir);
-        assert_eq!(
-            &*accounts1.config.inner.read().await,
-            &*accounts2.config.inner.read().await,
-        );
-        assert_eq!(
-            accounts1.accounts.read().await.len(),
-            accounts2.accounts.read().await.len()
-        );
+        assert_eq!(accounts1.config, accounts2.config,);
+        assert_eq!(accounts1.accounts.len(), accounts2.accounts.len());
     }
 
     #[async_std::test]
@@ -518,26 +498,26 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 0);
+        let mut accounts = Accounts::new(p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
         let id = accounts.add_account().await.unwrap();
         assert_eq!(id, 1);
-        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.accounts.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         let id = accounts.add_account().await.unwrap();
         assert_eq!(id, 2);
         assert_eq!(accounts.config.get_selected_account().await, id);
-        assert_eq!(accounts.accounts.read().await.len(), 2);
+        assert_eq!(accounts.accounts.len(), 2);
 
         accounts.select_account(1).await.unwrap();
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         accounts.remove_account(1).await.unwrap();
         assert_eq!(accounts.config.get_selected_account().await, 2);
-        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.accounts.len(), 1);
     }
 
     #[async_std::test]
@@ -545,14 +525,14 @@ mod tests {
         let dir = tempfile::tempdir()?;
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+        let mut accounts = Accounts::new(p.clone()).await?;
         assert!(accounts.get_selected_account().await.is_none());
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
         let id = accounts.add_account().await?;
         assert!(accounts.get_selected_account().await.is_some());
         assert_eq!(id, 1);
-        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.accounts.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, id);
 
         accounts.remove_account(id).await?;
@@ -566,14 +546,12 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 0);
+        let mut accounts = Accounts::new(p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
         let extern_dbfile: PathBuf = dir.path().join("other").into();
-        let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
-            .await
-            .unwrap();
+        let ctx = Context::new(extern_dbfile.clone(), 0).await.unwrap();
         ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
             .await
             .unwrap();
@@ -584,7 +562,7 @@ mod tests {
             .migrate_account(extern_dbfile.clone())
             .await
             .unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.accounts.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         let ctx = accounts.get_selected_account().await.unwrap();
@@ -603,7 +581,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let mut accounts = Accounts::new(p.clone()).await.unwrap();
 
         for expected_id in 1..10 {
             let id = accounts.add_account().await.unwrap();
@@ -623,7 +601,7 @@ mod tests {
         let dummy_accounts = 10;
 
         let (id0, id1, id2) = {
-            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+            let mut accounts = Accounts::new(p.clone()).await?;
             accounts.add_account().await?;
             let ids = accounts.get_all().await;
             assert_eq!(ids.len(), 1);
@@ -658,7 +636,7 @@ mod tests {
         assert!(id2 > id1 + dummy_accounts);
 
         let (id0_reopened, id1_reopened, id2_reopened) = {
-            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+            let accounts = Accounts::new(p.clone()).await?;
             let ctx = accounts.get_selected_account().await.unwrap();
             assert_eq!(
                 ctx.get_config(crate::config::Config::Addr).await?,
@@ -697,4 +675,30 @@ mod tests {
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_no_accounts_event_emitter() -> Result<()> {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new(p.clone()).await?;
+
+        // Make sure there are no accounts.
+        assert_eq!(accounts.accounts.len(), 0);
+
+        // Create event emitter.
+        let mut event_emitter = accounts.get_event_emitter().await;
+
+        // Test that event emitter does not return `None` immediately.
+        let duration = std::time::Duration::from_millis(1);
+        assert!(async_std::future::timeout(duration, event_emitter.recv())
+            .await
+            .is_err());
+
+        // When account manager is dropped, event emitter is exhausted.
+        drop(accounts);
+        assert_eq!(event_emitter.recv().await?, None);
+
+        Ok(())
+    }
 }

src/aheader.rs

@@ -1,13 +1,13 @@
-//! # Autocrypt header module
+//! # Autocrypt header module.
 //!
 //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
 
+use anyhow::{bail, format_err, Error, Result};
 use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
 use crate::contact::addr_cmp;
-use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};
 
@@ -37,13 +37,13 @@ impl fmt::Display for EncryptPreference {
 }
 
 impl str::FromStr for EncryptPreference {
-    type Err = ();
+    type Err = Error;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    fn from_str(s: &str) -> Result<Self> {
         match s {
             "mutual" => Ok(EncryptPreference::Mutual),
             "nopreference" => Ok(EncryptPreference::NoPreference),
-            _ => Err(()),
+            _ => bail!("Cannot parse encryption preference {}", s),
         }
     }
 }
@@ -70,28 +70,27 @@ impl Aheader {
         }
     }
 
+    /// Tries to parse Autocrypt header.
+    ///
+    /// If there is none, returns None. If the header is present but cannot be parsed, returns an
+    /// error.
     pub fn from_headers(
-        context: &Context,
         wanted_from: &str,
         headers: &[mailparse::MailHeader<'_>],
-    ) -> Option<Self> {
+    ) -> Result<Option<Self>> {
         if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) {
-            match Self::from_str(&value) {
-                Ok(header) => {
-                    if addr_cmp(&header.addr, wanted_from) {
-                        return Some(header);
-                    }
-                }
-                Err(err) => {
-                    warn!(
-                        context,
-                        "found invalid autocrypt header {}: {:?}", value, err
-                    );
-                }
+            let header = Self::from_str(&value)?;
+            if !addr_cmp(&header.addr, wanted_from) {
+                bail!(
+                    "Autocrypt header address {:?} is not {:?}",
+                    header.addr,
+                    wanted_from
+                );
             }
+            Ok(Some(header))
+        } else {
+            Ok(None)
         }
-
-        None
     }
 }
 
@@ -120,9 +119,9 @@ impl fmt::Display for Aheader {
 }
 
 impl str::FromStr for Aheader {
-    type Err = ();
+    type Err = Error;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    fn from_str(s: &str) -> Result<Self> {
         let mut attributes: BTreeMap<String, String> = s
             .split(';')
             .filter_map(|a| {
@@ -136,15 +135,20 @@ impl str::FromStr for Aheader {
 
         let addr = match attributes.remove("addr") {
             Some(addr) => addr,
-            None => {
-                return Err(());
-            }
+            None => bail!("Autocrypt header has no addr"),
         };
         let public_key: SignedPublicKey = attributes
             .remove("keydata")
-            .ok_or(())
-            .and_then(|raw| SignedPublicKey::from_base64(&raw).or(Err(())))
-            .and_then(|key| key.verify().and(Ok(key)).or(Err(())))?;
+            .ok_or_else(|| format_err!("keydata attribute is not found"))
+            .and_then(|raw| {
+                SignedPublicKey::from_base64(&raw)
+                    .map_err(|_| format_err!("Autocrypt key cannot be decoded"))
+            })
+            .and_then(|key| {
+                key.verify()
+                    .and(Ok(key))
+                    .map_err(|_| format_err!("Autocrypt key cannot be verified"))
+            })?;
 
         let prefer_encrypt = attributes
             .remove("prefer-encrypt")
@@ -154,7 +158,7 @@ impl str::FromStr for Aheader {
         // Autocrypt-Level0: unknown attributes starting with an underscore can be safely ignored
         // Autocrypt-Level0: unknown attribute, treat the header as invalid
         if attributes.keys().any(|k| !k.starts_with('_')) {
-            return Err(());
+            bail!("Unknown Autocrypt attribute found");
         }
 
         Ok(Aheader {
@@ -172,38 +176,40 @@ mod tests {
     const RAWKEY: &str = "xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=";
 
     #[test]
-    fn test_from_str() {
+    fn test_from_str() -> Result<()> {
         let h: Aheader = format!(
             "addr=me@mail.com; prefer-encrypt=mutual; keydata={}",
             RAWKEY
         )
-        .parse()
-        .expect("failed to parse");
+        .parse()?;
 
         assert_eq!(h.addr, "me@mail.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::Mutual);
+        Ok(())
     }
 
     // EncryptPreference::Reset is an internal value, parser should never return it
     #[test]
-    fn test_from_str_reset() {
+    fn test_from_str_reset() -> Result<()> {
         let raw = format!(
             "addr=reset@example.com; prefer-encrypt=reset; keydata={}",
             RAWKEY
         );
-        let h: Aheader = raw.parse().expect("failed to parse");
+        let h: Aheader = raw.parse()?;
 
         assert_eq!(h.addr, "reset@example.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
+        Ok(())
     }
 
     #[test]
-    fn test_from_str_non_critical() {
+    fn test_from_str_non_critical() -> Result<()> {
         let raw = format!("addr=me@mail.com; _foo=one; _bar=two; keydata={}", RAWKEY);
-        let h: Aheader = raw.parse().expect("failed to parse");
+        let h: Aheader = raw.parse()?;
 
         assert_eq!(h.addr, "me@mail.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
+        Ok(())
     }
 
     #[test]
@@ -216,7 +222,7 @@ mod tests {
     }
 
     #[test]
-    fn test_good_headers() {
+    fn test_good_headers() -> Result<()> {
         let fixed_header = concat!(
             "addr=a@b.example.org; prefer-encrypt=mutual; ",
             "keydata=xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJg",
@@ -242,7 +248,7 @@ mod tests {
             " wm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc="
         );
 
-        let ah = Aheader::from_str(fixed_header).expect("failed to parse");
+        let ah = Aheader::from_str(fixed_header)?;
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
         assert_eq!(format!("{}", ah), fixed_header);
@@ -250,18 +256,17 @@ mod tests {
         let rendered = ah.to_string();
         assert_eq!(rendered, fixed_header);
 
-        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY)).expect("failed to parse");
+        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY))?;
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
 
         Aheader::from_str(&format!(
             "addr=a@b.example.org; prefer-encrypt=ignoreUnknownValues; keydata={}",
             RAWKEY
-        ))
-        .expect("failed to parse");
+        ))?;
 
-        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))
-            .expect("failed to parse");
+        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))?;
+        Ok(())
     }
 
     #[test]

src/blob.rs

@@ -1,4 +1,4 @@
-//! # Blob directory management
+//! # Blob directory management.
 
 use core::cmp::max;
 use std::ffi::OsStr;
@@ -621,7 +621,13 @@ mod tests {
 
     use super::*;
 
-    use crate::{message::Message, test_utils::TestContext};
+    use crate::chat::{create_group_chat, ProtectionStatus};
+    use crate::{
+        chat,
+        message::Message,
+        test_utils::{self, TestContext},
+    };
+    use anyhow::Result;
     use image::Pixel;
 
     #[async_std::test]
@@ -870,11 +876,10 @@ mod tests {
     async fn test_selfavatar_in_blobdir() {
         let t = TestContext::new().await;
         let avatar_src = t.get_blobdir().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
             .await
             .unwrap()
-            .write_all(avatar_bytes)
+            .write_all(test_utils::AVATAR_900x900_BYTES)
             .await
             .unwrap();
 
@@ -1064,4 +1069,38 @@ mod tests {
         assert_eq!(img.height() as u32, compressed_height);
         Ok(img)
     }
+
+    #[async_std::test]
+    async fn test_increation_in_blobdir() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
+
+        let file = t.get_blobdir().join("anyfile.dat");
+        File::create(&file).await?.write_all("bla".as_ref()).await?;
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file(file.to_str().unwrap(), None);
+        let prepared_id = chat::prepare_msg(&t, chat_id, &mut msg).await?;
+        assert_eq!(prepared_id, msg.id);
+        assert!(msg.is_increation());
+
+        let msg = Message::load_from_db(&t, prepared_id).await?;
+        assert!(msg.is_increation());
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_increation_not_blobdir() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
+        assert_ne!(t.get_blobdir().to_str(), t.dir.path().to_str());
+
+        let file = t.dir.path().join("anyfile.dat");
+        File::create(&file).await?.write_all("bla".as_ref()).await?;
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file(file.to_str().unwrap(), None);
+        assert!(chat::prepare_msg(&t, chat_id, &mut msg).await.is_err());
+
+        Ok(())
+    }
 }

src/chatlist.rs

@@ -1,4 +1,4 @@
-//! # Chat list module
+//! # Chat list module.
 
 use anyhow::{bail, ensure, Result};
 
@@ -11,9 +11,9 @@ use crate::constants::{
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::ephemeral::delete_expired_messages;
-use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock_str;
+use crate::summary::Summary;
 
 /// An object representing a single chatlist in memory.
 ///
@@ -288,26 +288,13 @@ impl Chatlist {
         }
     }
 
-    /// Get a summary for a chatlist index.
-    ///
-    /// The summary is returned by a dc_lot_t object with the following fields:
-    ///
-    /// - dc_lot_t::text1: contains the username or the strings "Me", "Draft" and so on.
-    ///   The string may be colored by having a look at text1_meaning.
-    ///   If there is no such name or it should not be displayed, the element is NULL.
-    /// - dc_lot_t::text1_meaning: one of DC_TEXT1_USERNAME, DC_TEXT1_SELF or DC_TEXT1_DRAFT.
-    ///   Typically used to show dc_lot_t::text1 with different colors. 0 if not applicable.
-    /// - dc_lot_t::text2: contains an excerpt of the message text or strings as
-    ///   "No messages".  May be NULL of there is no such text (eg. for the archive link)
-    /// - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
-    /// - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
-    //    0 if not applicable.
+    /// Returns a summary for a given chatlist index.
     pub async fn get_summary(
         &self,
         context: &Context,
         index: usize,
         chat: Option<&Chat>,
-    ) -> Result<Lot> {
+    ) -> Result<Summary> {
         // The summary is created by the chat, not by the last message.
         // This is because we may want to display drafts here or stuff as
         // "is typing".
@@ -320,14 +307,13 @@ impl Chatlist {
         Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
     }
 
+    /// Returns a summary for a given chatlist item.
     pub async fn get_summary2(
         context: &Context,
         chat_id: ChatId,
         lastmsg_id: Option<MsgId>,
         chat: Option<&Chat>,
-    ) -> Result<Lot> {
-        let mut ret = Lot::new();
-
+    ) -> Result<Summary> {
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
@@ -343,10 +329,9 @@ impl Chatlist {
                 (Some(lastmsg), None)
             } else {
                 match chat.typ {
-                    Chattype::Group | Chattype::Mailinglist => {
-                        let lastcontact =
-                            Contact::load_from_db(context, lastmsg.from_id).await.ok();
-                        (Some(lastmsg), lastcontact)
+                    Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
+                        let lastcontact = Contact::load_from_db(context, lastmsg.from_id).await?;
+                        (Some(lastmsg), Some(lastcontact))
                     }
                     Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
                 }
@@ -356,17 +341,15 @@ impl Chatlist {
         };
 
         if chat.id.is_archived_link() {
-            ret.text2 = None;
-        } else if let Some(mut lastmsg) =
-            lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED)
-        {
-            ret.fill(&mut lastmsg, chat, lastcontact.as_ref(), context)
-                .await;
+            Ok(Default::default())
+        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED) {
+            Ok(Summary::new(context, &lastmsg, chat, lastcontact.as_ref()).await)
         } else {
-            ret.text2 = Some(stock_str::no_messages(context).await);
+            Ok(Summary {
+                text: stock_str::no_messages(context).await,
+                ..Default::default()
+            })
         }
-
-        Ok(ret)
     }
 
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
@@ -380,7 +363,7 @@ pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
         .sql
         .count(
             "SELECT COUNT(*) FROM chats WHERE blocked!=? AND archived=?;",
-            paramsv![Blocked::Manually, ChatVisibility::Archived],
+            paramsv![Blocked::Yes, ChatVisibility::Archived],
         )
         .await?;
     Ok(count)
@@ -416,10 +399,20 @@ mod tests {
         assert_eq!(chats.get_chat_id(1), chat_id2);
         assert_eq!(chats.get_chat_id(2), chat_id1);
 
-        // drafts are sorted to the top
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t, Some(&mut msg)).await.unwrap();
+        // New drafts are sorted to the top
+        // We have to set a draft on the other two messages, too, as
+        // chat timestamps are only exact to the second and sorting by timestamp
+        // would not work.
+        // Message timestamps are "smeared" and unique, so we don't have this problem
+        // if we have any message (can be a draft) in all chats.
+        // Instead of setting drafts for chat_id1 and chat_id3, we could also sleep
+        // 2s here.
+        for chat_id in &[chat_id1, chat_id3, chat_id2] {
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_text(Some("hello".to_string()));
+            chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
+        }
+
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
@@ -506,7 +499,7 @@ mod tests {
         dc_receive_imf(
             &t,
             b"From: Bob Authname <bob@example.org>\n\
-                 To: alice@example.com\n\
+                 To: alice@example.org\n\
                  Subject: foo\n\
                  Message-ID: <msg1234@example.org>\n\
                  Chat-Version: 1.0\n\
@@ -514,7 +507,6 @@ mod tests {
                  \n\
                  hello foo\n",
             "INBOX",
-            1,
             false,
         )
         .await?;
@@ -568,7 +560,7 @@ mod tests {
         dc_receive_imf(
             &t,
             b"From: bob@example.org\n\
-                 To: alice@example.com\n\
+                 To: alice@example.org\n\
                  Subject: foo\n\
                  Message-ID: <msg5678@example.org>\n\
                  Chat-Version: 1.0\n\
@@ -576,7 +568,6 @@ mod tests {
                  \n\
                  hello foo\n",
             "INBOX",
-            1,
             false,
         )
         .await?;
@@ -637,6 +628,6 @@ mod tests {
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         let summary = chats.get_summary(&t, 0, None).await.unwrap();
-        assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
+        assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/color.rs

@@ -1,4 +1,4 @@
-//! Implementation of Consistent Color Generation
+//! Implementation of Consistent Color Generation.
 //!
 //! Consistent Color Generation is defined in XEP-0392.
 //!
@@ -35,6 +35,10 @@ pub(crate) fn str_to_color(s: &str) -> u32 {
     rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
 }
 
+pub fn color_int_to_hex_string(color: u32) -> String {
+    format!("{:#08x}", color).replace("0x", "#")
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/config.rs

@@ -1,6 +1,6 @@
-//! # Key-value configuration management
+//! # Key-value configuration management.
 
-use anyhow::Result;
+use anyhow::{ensure, Result};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
@@ -48,6 +48,12 @@ pub enum Config {
     SmtpCertificateChecks,
     ServerFlags,
 
+    Socks5Enabled,
+    Socks5Host,
+    Socks5Port,
+    Socks5User,
+    Socks5Password,
+
     Displayname,
     Selfstatus,
     Selfavatar,
@@ -61,15 +67,9 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MdnsEnabled,
 
-    #[strum(props(default = "1"))]
-    InboxWatch,
-
     #[strum(props(default = "1"))]
     SentboxWatch,
 
-    #[strum(props(default = "1"))]
-    MvboxWatch,
-
     #[strum(props(default = "1"))]
     MvboxMove,
 
@@ -150,6 +150,11 @@ pub enum Config {
     #[strum(props(default = "0"))]
     NotifyAboutWrongPw,
 
+    /// If a warning about exceeding quota was shown recently,
+    /// this is the percentage of quota at the time the warning was given.
+    /// Unset, when quota falls below minimal warning threshold again.
+    QuotaExceeding,
+
     /// address to webrtc instance to use for videochats
     WebrtcInstance,
 
@@ -159,6 +164,16 @@ pub enum Config {
     /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
     #[strum(props(default = "60"))]
     ScanAllFoldersDebounceSecs,
+
+    /// Defines the max. size (in bytes) of messages downloaded automatically.
+    /// 0 = no limit.
+    #[strum(props(default = "0"))]
+    DownloadLimit,
+
+    /// Send sync messages, requires `BccSelf` to be set as well.
+    /// In a future versions, this switch may be removed.
+    #[strum(props(default = "0"))]
+    SendSyncMsgs,
 }
 
 impl Context {
@@ -263,13 +278,13 @@ impl Context {
                         let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
                         blob.recode_to_avatar_size(self).await?;
                         self.sql.set_raw_config(key, Some(blob.as_name())).await?;
-                        Ok(())
                     }
                     None => {
                         self.sql.set_raw_config(key, None).await?;
-                        Ok(())
                     }
                 }
+                self.emit_event(EventType::SelfavatarChanged);
+                Ok(())
             }
             Config::Selfstatus => {
                 let def = stock_str::status_line(self).await;
@@ -306,7 +321,7 @@ impl Context {
                     .set_raw_config(key, value)
                     .await
                     .map_err(Into::into);
-                job::schedule_resync(self).await;
+                job::schedule_resync(self).await?;
                 ret
             }
             _ => {
@@ -321,6 +336,21 @@ impl Context {
             .await?;
         Ok(())
     }
+
+    /// Sets an ui-specific key-value pair.
+    /// Keys must be prefixed by `ui.`
+    /// and should be followed by the name of the system and maybe subsystem,
+    /// eg. `ui.desktop.linux.foo`, `ui.desktop.macos.bar`, `ui.ios.foobar`.
+    pub async fn set_ui_config(&self, key: &str, value: Option<&str>) -> Result<()> {
+        ensure!(key.starts_with("ui."), "set_ui_config(): prefix missing.");
+        self.sql.set_raw_config(key, value).await
+    }
+
+    /// Gets an ui-specific value set by set_ui_config().
+    pub async fn get_ui_config(&self, key: &str) -> Result<Option<String>> {
+        ensure!(key.starts_with("ui."), "get_ui_config(): prefix missing.");
+        self.sql.get_raw_config(key).await
+    }
 }
 
 /// Returns all available configuration keys concated together.
@@ -373,4 +403,25 @@ mod tests {
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
         assert_eq!(media_quality, constants::MediaQuality::Worse);
     }
+
+    #[async_std::test]
+    async fn test_ui_config() -> Result<()> {
+        let t = TestContext::new().await;
+
+        assert_eq!(t.get_ui_config("ui.desktop.linux.systray").await?, None);
+
+        t.set_ui_config("ui.android.screen_security", Some("safe"))
+            .await?;
+        assert_eq!(
+            t.get_ui_config("ui.android.screen_security").await?,
+            Some("safe".to_string())
+        );
+
+        t.set_ui_config("ui.android.screen_security", None).await?;
+        assert_eq!(t.get_ui_config("ui.android.screen_security").await?, None);
+
+        assert!(t.set_ui_config("configured", Some("bar")).await.is_err());
+
+        Ok(())
+    }
 }

src/configure.rs

@@ -1,4 +1,4 @@
-//! Email accounts autoconfiguration process module
+//! Email accounts autoconfiguration process module.
 
 mod auto_mozilla;
 mod auto_outlook;
@@ -8,13 +8,12 @@ mod server_params;
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
 use async_std::task;
-use itertools::Itertools;
 use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
-use crate::login_param::{LoginParam, ServerLoginParam};
+use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
 use crate::message::Message;
 use crate::oauth2::dc_get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
@@ -170,12 +169,17 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         DC_LP_AUTH_NORMAL as i32
     };
 
+    let socks5_config = param.socks5_config.clone();
+    let socks5_enabled = socks5_config.is_some();
+
     let ctx2 = ctx.clone();
     let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
 
     // Step 1: Load the parameters and check email-address and password
 
-    if oauth2 {
+    // Do oauth2 only if socks5 is disabled. As soon as we have a http library that can do
+    // socks5 requests, this can work with socks5 too
+    if oauth2 && !socks5_enabled {
         // the used oauth2 addr may differ, check this.
         // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
@@ -217,7 +221,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             "checking internal provider-info for offline autoconfig"
         );
 
-        if let Some(provider) = provider::get_provider_info(&param_domain).await {
+        if let Some(provider) =
+            provider::get_provider_info(ctx, &param_domain, socks5_enabled).await
+        {
             param.provider = Some(provider);
             match provider.status {
                 provider::Status::Ok | provider::Status::Preparation => {
@@ -244,6 +250,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                                         }
                                     }
                                 },
+                                strict_tls: Some(provider.strict_tls),
                             })
                             .collect();
 
@@ -256,9 +263,16 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                 }
             }
         } else {
+            // Try receiving autoconfig
             info!(ctx, "no offline autoconfig found");
-            param_autoconfig =
-                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
+            param_autoconfig = if socks5_enabled {
+                // Currently we can't do http requests through socks5, to not leak
+                // the ip, just don't do online autoconfig
+                info!(ctx, "socks5 enabled, skipping autoconfig");
+                None
+            } else {
+                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await
+            }
         }
     } else {
         param_autoconfig = None;
@@ -277,6 +291,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.imap.port,
             socket: param.imap.security,
             username: param.imap.user.clone(),
+            strict_tls: None,
         })
     }
     if !servers
@@ -289,8 +304,24 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.smtp.port,
             socket: param.smtp.security,
             username: param.smtp.user.clone(),
+            strict_tls: None,
         })
     }
+
+    // respect certificate setting from function parameters
+    for mut server in &mut servers {
+        let certificate_checks = match server.protocol {
+            Protocol::Imap => param.imap.certificate_checks,
+            Protocol::Smtp => param.smtp.certificate_checks,
+        };
+        server.strict_tls = match certificate_checks {
+            CertificateChecks::AcceptInvalidCertificates
+            | CertificateChecks::AcceptInvalidCertificates2 => Some(false),
+            CertificateChecks::Strict => Some(true),
+            CertificateChecks::Automatic => server.strict_tls,
+        };
+    }
+
     let servers = expand_param_vector(servers, &param.addr, &param_domain);
 
     progress!(ctx, 550);
@@ -306,7 +337,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         .filter(|params| params.protocol == Protocol::Smtp)
         .cloned()
         .collect();
-    let provider_strict_tls = param.provider.map_or(false, |provider| provider.strict_tls);
+    let provider_strict_tls = param
+        .provider
+        .map_or(socks5_config.is_some(), |provider| provider.strict_tls);
 
     let smtp_config_task = task::spawn(async move {
         let mut smtp_configured = false;
@@ -316,10 +349,16 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             smtp_param.server = smtp_server.hostname.clone();
             smtp_param.port = smtp_server.port;
             smtp_param.security = smtp_server.socket;
+            smtp_param.certificate_checks = match smtp_server.strict_tls {
+                Some(true) => CertificateChecks::Strict,
+                Some(false) => CertificateChecks::AcceptInvalidCertificates,
+                None => CertificateChecks::Automatic,
+            };
 
             match try_smtp_one_param(
                 &context_smtp,
                 &smtp_param,
+                &socks5_config,
                 &smtp_addr,
                 oauth2,
                 provider_strict_tls,
@@ -358,8 +397,22 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         param.imap.server = imap_server.hostname.clone();
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
+        param.imap.certificate_checks = match imap_server.strict_tls {
+            Some(true) => CertificateChecks::Strict,
+            Some(false) => CertificateChecks::AcceptInvalidCertificates,
+            None => CertificateChecks::Automatic,
+        };
 
-        match try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, provider_strict_tls).await {
+        match try_imap_one_param(
+            ctx,
+            &param.imap,
+            &param.socks5_config,
+            &param.addr,
+            oauth2,
+            provider_strict_tls,
+        )
+        .await
+        {
             Ok(configured_imap) => {
                 imap = Some(configured_imap);
                 break;
@@ -390,8 +443,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 900);
 
-    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await?
-        || ctx.get_config_bool(Config::MvboxMove).await?;
+    let create_mvbox = ctx.get_config_bool(Config::MvboxMove).await?;
 
     imap.configure_folders(ctx, create_mvbox).await?;
 
@@ -419,7 +471,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         ctx,
         job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
     )
-    .await;
+    .await?;
 
     progress!(ctx, 940);
     update_device_chats_handle.await?;
@@ -507,6 +559,7 @@ async fn get_autoconfig(
 async fn try_imap_one_param(
     context: &Context,
     param: &ServerLoginParam,
+    socks5_config: &Option<Socks5Config>,
     addr: &str,
     oauth2: bool,
     provider_strict_tls: bool,
@@ -519,7 +572,16 @@ async fn try_imap_one_param(
 
     let (_s, r) = async_std::channel::bounded(1);
 
-    let mut imap = match Imap::new(param, addr, oauth2, provider_strict_tls, r).await {
+    let mut imap = match Imap::new(
+        param,
+        socks5_config.clone(),
+        addr,
+        oauth2,
+        provider_strict_tls,
+        r,
+    )
+    .await
+    {
         Err(err) => {
             info!(context, "failure: {}", err);
             return Err(ConfigurationError {
@@ -548,19 +610,37 @@ async fn try_imap_one_param(
 async fn try_smtp_one_param(
     context: &Context,
     param: &ServerLoginParam,
+    socks5_config: &Option<Socks5Config>,
     addr: &str,
     oauth2: bool,
     provider_strict_tls: bool,
     smtp: &mut Smtp,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
-        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
-        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
+        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
+        param.user,
+        param.server,
+        param.port,
+        param.security,
+        param.certificate_checks,
+        oauth2,
+        if let Some(socks5_config) = socks5_config {
+            socks5_config.to_string()
+        } else {
+            "None".to_string()
+        }
     );
     info!(context, "Trying: {}", inf);
 
     if let Err(err) = smtp
-        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .connect(
+            context,
+            param,
+            socks5_config,
+            addr,
+            oauth2,
+            provider_strict_tls,
+        )
         .await
     {
         info!(context, "failure: {}", err);
@@ -602,7 +682,11 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
         return first_err.msg.to_string();
     }
 
-    errors.iter().map(|e| e.to_string()).join("\n\n")
+    errors
+        .iter()
+        .map(|e| e.to_string())
+        .collect::<Vec<String>>()
+        .join("\n\n")
 }
 
 #[derive(Debug, thiserror::Error)]

src/configure/auto_mozilla.rs

@@ -243,6 +243,7 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
                 hostname: server.hostname,
                 port: server.port,
                 username: server.username,
+                strict_tls: None,
             })
         })
         .collect();

src/configure/auto_outlook.rs

@@ -180,6 +180,7 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
                 hostname: protocol.server,
                 port: protocol.port,
                 username: String::new(),
+                strict_tls: None,
             })
         })
         .collect()

src/configure/server_params.rs

@@ -22,6 +22,9 @@ pub(crate) struct ServerParams {
 
     /// Username, empty if unknown.
     pub username: String,
+
+    /// Whether TLS certificates should be strictly checked or not, `None` for automatic.
+    pub strict_tls: Option<bool>,
 }
 
 impl ServerParams {
@@ -128,6 +131,23 @@ impl ServerParams {
             vec![self]
         }
     }
+
+    fn expand_strict_tls(self) -> Vec<ServerParams> {
+        if self.strict_tls.is_none() {
+            vec![
+                Self {
+                    strict_tls: Some(true), // Strict.
+                    ..self.clone()
+                },
+                Self {
+                    strict_tls: None, // Automatic.
+                    ..self
+                },
+            ]
+        } else {
+            vec![self]
+        }
+    }
 }
 
 /// Expands vector of `ServerParams`, replacing placeholders with
@@ -138,10 +158,32 @@ pub(crate) fn expand_param_vector(
     domain: &str,
 ) -> Vec<ServerParams> {
     v.into_iter()
-        // The order of expansion is important: ports are expanded the
-        // last, so they are changed the first. Username is only
-        // changed if default value (address with domain) didn't work
-        // for all available hosts and ports.
+        .map(|params| {
+            if params.socket == Socket::Plain {
+                ServerParams {
+                    // Avoid expanding plaintext configuration into configuration with and without
+                    // `strict_tls` if `strict_tls` is set to `None` as `strict_tls` is not used for
+                    // plaintext connections. Always setting it to "enabled", just in case.
+                    strict_tls: Some(true),
+                    ..params
+                }
+            } else {
+                params
+            }
+        })
+        // The order of expansion is important.
+        //
+        // Ports are expanded the last, so they are changed the first.  Username is only changed if
+        // default value (address with domain) didn't work for all available hosts and ports.
+        //
+        // Strict TLS must be expanded first, so we try all configurations with strict TLS first
+        // and only then try again without strict TLS. Otherwise we may lock to wrong hostname
+        // without strict TLS when another hostname with strict TLS is available.  For example, if
+        // both smtp.example.net and mail.example.net are running an SMTP server, but both use a
+        // certificate that is only valid for mail.example.net, we want to skip smtp.example.net
+        // and use mail.example.net with strict TLS instead of using smtp.example.net without
+        // strict TLS.
+        .flat_map(|params| params.expand_strict_tls().into_iter())
         .flat_map(|params| params.expand_usernames(addr).into_iter())
         .flat_map(|params| params.expand_hostnames(domain).into_iter())
         .flat_map(|params| params.expand_ports().into_iter())
@@ -161,6 +203,7 @@ mod tests {
                 port: 0,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
+                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -174,6 +217,7 @@ mod tests {
                 port: 993,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
+                strict_tls: Some(true)
             }],
         );
 
@@ -184,6 +228,7 @@ mod tests {
                 port: 123,
                 socket: Socket::Automatic,
                 username: "foobar".to_string(),
+                strict_tls: None,
             }],
             "foobar@example.net",
             "example.net",
@@ -197,16 +242,59 @@ mod tests {
                     hostname: "example.net".to_string(),
                     port: 123,
                     socket: Socket::Ssl,
-                    username: "foobar".to_string()
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true),
                 },
                 ServerParams {
                     protocol: Protocol::Smtp,
                     hostname: "example.net".to_string(),
                     port: 123,
                     socket: Socket::Starttls,
-                    username: "foobar".to_string()
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                },
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: None,
+                },
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Starttls,
+                    username: "foobar".to_string(),
+                    strict_tls: None
                 }
             ],
         );
+
+        // Test that strict_tls is not expanded for plaintext connections.
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Smtp,
+                hostname: "example.net".to_string(),
+                port: 123,
+                socket: Socket::Plain,
+                username: "foobar".to_string(),
+                strict_tls: None,
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+        assert_eq!(
+            v,
+            vec![ServerParams {
+                protocol: Protocol::Smtp,
+                hostname: "example.net".to_string(),
+                port: 123,
+                socket: Socket::Plain,
+                username: "foobar".to_string(),
+                strict_tls: Some(true)
+            }],
+        );
     }
 }

src/constants.rs

@@ -1,4 +1,4 @@
-//! # Constants
+//! # Constants.
 use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
@@ -24,7 +24,7 @@ pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION")
 #[repr(i8)]
 pub enum Blocked {
     Not = 0,
-    Manually = 1,
+    Yes = 1,
     Request = 2,
 }
 
@@ -153,6 +153,7 @@ pub enum Chattype {
     Single = 100,
     Group = 120,
     Mailinglist = 140,
+    Broadcast = 160,
 }
 
 impl Default for Chattype {
@@ -165,36 +166,18 @@ pub const DC_MSG_ID_MARKER1: u32 = 1;
 pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
-/// string that indicates sth. is left out or truncated
-pub const DC_ELLIPSE: &str = "[...]";
+/// String that indicates that something is left out or truncated.
+pub const DC_ELLIPSIS: &str = "[...]";
 
-/// to keep bubbles and chat flow usable,
-/// and to avoid problems with controls using very long texts,
-/// we limit the text length to DC_DESIRED_TEXT_LEN.
-/// if the text is longer, the full text can be retrieved using has_html()/get_html().
+/// Message length limit.
 ///
-/// we are using a bit less than DC_MAX_GET_TEXT_LEN to avoid cutting twice
-/// (a bit less as truncation may not be exact and ellipses may be added).
+/// To keep bubbles and chat flow usable and to avoid problems with controls using very long texts,
+/// we limit the text length to `DC_DESIRED_TEXT_LEN`.  If the text is longer, the full text can be
+/// retrieved using has_html()/get_html().
 ///
-/// note, that DC_DESIRED_TEXT_LEN and DC_MAX_GET_TEXT_LEN
-/// define max. number of bytes, _not_ unicode graphemes.
-/// in general, that seems to be okay for such an upper limit,
-/// esp. as calculating the number of graphemes is not simple
-/// (one graphemes may be a sequence of code points which is a sequence of bytes).
-/// also even if we have the exact number of graphemes,
-/// that would not always help on getting an idea about the screen space used
-/// (to keep bubbles and chat flow usable).
-///
-/// therefore, the number of bytes is only a very rough estimation,
-/// however, the ~30K seems to work okayish for a while,
-/// if it turns out, it is too few for some alphabet, we can still increase.
-pub const DC_DESIRED_TEXT_LEN: usize = 29_000;
-
-/// approx. max. length (number of bytes) returned by dc_msg_get_text()
-pub const DC_MAX_GET_TEXT_LEN: usize = 30_000;
-
-/// approx. max. length returned by dc_get_msg_info()
-pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
+/// Note that for simplicity maximum length is defined as the number of Unicode Scalar Values (Rust
+/// `char`s), not Unicode Grapheme Clusters.
+pub const DC_DESIRED_TEXT_LEN: usize = 5000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -366,6 +349,7 @@ mod tests {
         assert_eq!(Chattype::Single, Chattype::from_i32(100).unwrap());
         assert_eq!(Chattype::Group, Chattype::from_i32(120).unwrap());
         assert_eq!(Chattype::Mailinglist, Chattype::from_i32(140).unwrap());
+        assert_eq!(Chattype::Broadcast, Chattype::from_i32(160).unwrap());
     }
 
     #[test]
@@ -401,7 +385,7 @@ mod tests {
         // values may be written to disk and must not change
         assert_eq!(Blocked::Not, Blocked::default());
         assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
-        assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
+        assert_eq!(Blocked::Yes, Blocked::from_i32(1).unwrap());
         assert_eq!(Blocked::Request, Blocked::from_i32(2).unwrap());
     }
 

src/contact.rs

@@ -2,10 +2,9 @@
 
 use std::convert::{TryFrom, TryInto};
 
-use anyhow::{bail, ensure, format_err, Result};
+use anyhow::{bail, ensure, Context as _, Result};
 use async_std::path::PathBuf;
 use deltachat_derive::{FromSql, ToSql};
-use itertools::Itertools;
 use once_cell::sync::Lazy;
 use regex::Regex;
 
@@ -67,6 +66,9 @@ pub struct Contact {
     /// Blocked state. Use dc_contact_is_blocked to access this field.
     pub blocked: bool,
 
+    /// Time when the contact was seen last time, Unix time in seconds.
+    last_seen: i64,
+
     /// The origin/source of the contact.
     pub origin: Origin,
 
@@ -174,12 +176,19 @@ pub enum VerifiedStatus {
     BidirectVerified = 2,
 }
 
+impl Default for VerifiedStatus {
+    fn default() -> Self {
+        Self::Unverified
+    }
+}
+
 impl Contact {
     pub async fn load_from_db(context: &Context, contact_id: u32) -> Result<Self> {
         let mut contact = context
             .sql
             .query_row(
-                "SELECT c.name, c.addr, c.origin, c.blocked, c.authname, c.param, c.status
+                "SELECT c.name, c.addr, c.origin, c.blocked, c.last_seen,
+                c.authname, c.param, c.status
                FROM contacts c
               WHERE c.id=?;",
                 paramsv![contact_id as i32],
@@ -188,15 +197,17 @@ impl Contact {
                     let addr: String = row.get(1)?;
                     let origin: Origin = row.get(2)?;
                     let blocked: Option<bool> = row.get(3)?;
-                    let authname: String = row.get(4)?;
-                    let param: String = row.get(5)?;
-                    let status: Option<String> = row.get(6)?;
+                    let last_seen: i64 = row.get(4)?;
+                    let authname: String = row.get(5)?;
+                    let param: String = row.get(6)?;
+                    let status: Option<String> = row.get(7)?;
                     let contact = Self {
                         id: contact_id,
                         name,
                         authname,
                         addr,
                         blocked: blocked.unwrap_or_default(),
+                        last_seen,
                         origin,
                         param: param.parse().unwrap_or_default(),
                         status: status.unwrap_or_default(),
@@ -228,12 +239,15 @@ impl Contact {
         self.blocked
     }
 
+    /// Returns last seen timestamp.
+    pub fn last_seen(&self) -> i64 {
+        self.last_seen
+    }
+
     /// Check if a contact is blocked.
-    pub async fn is_blocked_load(context: &Context, id: u32) -> bool {
-        Self::load_from_db(context, id)
-            .await
-            .map(|contact| contact.blocked)
-            .unwrap_or_default()
+    pub async fn is_blocked_load(context: &Context, id: u32) -> Result<bool> {
+        let blocked = Self::load_from_db(context, id).await?.blocked;
+        Ok(blocked)
     }
 
     /// Block the given contact.
@@ -263,7 +277,7 @@ impl Contact {
 
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated).await?;
-        let blocked = Contact::is_blocked_load(context, contact_id).await;
+        let blocked = Contact::is_blocked_load(context, contact_id).await?;
         match sth_modified {
             Modifier::None => {}
             Modifier::Modified | Modifier::Created => {
@@ -297,17 +311,17 @@ impl Contact {
     /// use `dc_may_be_valid_addr()`.
     pub async fn lookup_id_by_addr(
         context: &Context,
-        addr: impl AsRef<str>,
+        addr: &str,
         min_origin: Origin,
     ) -> Result<Option<u32>> {
-        if addr.as_ref().is_empty() {
+        if addr.is_empty() {
             bail!("lookup_id_by_addr: empty address");
         }
 
-        let addr_normalized = addr_normalize(addr.as_ref());
+        let addr_normalized = addr_normalize(addr);
 
         if let Some(addr_self) = context.get_config(Config::ConfiguredAddr).await? {
-            if addr_cmp(addr_normalized, addr_self) {
+            if addr_cmp(addr_normalized, &addr_self) {
                 return Ok(Some(DC_CONTACT_ID_SELF));
             }
         }
@@ -369,7 +383,7 @@ impl Contact {
             .await?
             .unwrap_or_default();
 
-        if addr_cmp(&addr, addr_self) {
+        if addr_cmp(&addr, &addr_self) {
             return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
@@ -568,7 +582,7 @@ impl Contact {
 
         for (name, addr) in split_address_book(addr_book).into_iter() {
             let (name, addr) = sanitize_name_and_addr(name, addr);
-            let name = normalize_name(name);
+            let name = normalize_name(&name);
             match Contact::add_or_lookup(context, &name, &addr, Origin::AddressBook).await {
                 Err(err) => {
                     warn!(
@@ -707,7 +721,7 @@ impl Contact {
             .sql
             .query_map(
                 "SELECT name, grpid FROM chats WHERE type=? AND blocked=?;",
-                paramsv![Chattype::Mailinglist, Blocked::Manually],
+                paramsv![Chattype::Mailinglist, Blocked::Yes],
                 |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
                 |rows| {
                     rows.collect::<std::result::Result<Vec<_>, _>>()
@@ -754,12 +768,9 @@ impl Contact {
 
     /// Get blocked contacts.
     pub async fn get_all_blocked(context: &Context) -> Result<Vec<u32>> {
-        if let Err(e) = Contact::update_blocked_mailinglist_contacts(context).await {
-            warn!(
-                context,
-                "Cannot update blocked mailinglist contacts: {:?}", e
-            );
-        }
+        Contact::update_blocked_mailinglist_contacts(context)
+            .await
+            .context("cannot update blocked mailinglist contacts")?;
 
         let list = context
             .sql
@@ -1018,7 +1029,7 @@ impl Contact {
     ///
     /// The UI may draw a checkbox or something like that beside verified contacts.
     ///
-    pub async fn is_verified(&self, context: &Context) -> VerifiedStatus {
+    pub async fn is_verified(&self, context: &Context) -> Result<VerifiedStatus> {
         self.is_verified_ex(context, None).await
     }
 
@@ -1029,54 +1040,46 @@ impl Contact {
         &self,
         context: &Context,
         peerstate: Option<&Peerstate>,
-    ) -> VerifiedStatus {
+    ) -> Result<VerifiedStatus> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
         if self.id == DC_CONTACT_ID_SELF {
-            return VerifiedStatus::BidirectVerified;
+            return Ok(VerifiedStatus::BidirectVerified);
         }
 
         if let Some(peerstate) = peerstate {
             if peerstate.verified_key.is_some() {
-                return VerifiedStatus::BidirectVerified;
+                return Ok(VerifiedStatus::BidirectVerified);
             }
         }
 
-        let peerstate = match Peerstate::from_addr(context, &self.addr).await {
-            Ok(peerstate) => peerstate,
-            Err(err) => {
-                warn!(
-                    context,
-                    "Failed to load peerstate for address {}: {}", self.addr, err
-                );
-                return VerifiedStatus::Unverified;
-            }
-        };
-
-        if let Some(ps) = peerstate {
-            if ps.verified_key.is_some() {
-                return VerifiedStatus::BidirectVerified;
+        if let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? {
+            if peerstate.verified_key.is_some() {
+                return Ok(VerifiedStatus::BidirectVerified);
             }
         }
 
-        VerifiedStatus::Unverified
+        Ok(VerifiedStatus::Unverified)
     }
 
-    pub async fn addr_equals_contact(context: &Context, addr: &str, contact_id: u32) -> bool {
+    pub async fn addr_equals_contact(
+        context: &Context,
+        addr: &str,
+        contact_id: u32,
+    ) -> Result<bool> {
         if addr.is_empty() {
-            return false;
+            return Ok(false);
         }
 
-        if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            if !contact.addr.is_empty() {
-                let normalized_addr = addr_normalize(addr);
-                if contact.addr == normalized_addr {
-                    return true;
-                }
+        let contact = Contact::load_from_db(context, contact_id).await?;
+        if !contact.addr.is_empty() {
+            let normalized_addr = addr_normalize(addr);
+            if contact.addr == normalized_addr {
+                return Ok(true);
             }
         }
 
-        false
+        Ok(false)
     }
 
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
@@ -1094,30 +1097,34 @@ impl Contact {
         Ok(count)
     }
 
-    pub async fn real_exists_by_id(context: &Context, contact_id: u32) -> bool {
-        if !context.sql.is_open().await || contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
-            return false;
+    pub async fn real_exists_by_id(context: &Context, contact_id: u32) -> Result<bool> {
+        if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+            return Ok(false);
         }
 
-        context
+        let exists = context
             .sql
             .exists(
                 "SELECT COUNT(*) FROM contacts WHERE id=?;",
                 paramsv![contact_id as i32],
             )
-            .await
-            .unwrap_or_default()
+            .await?;
+        Ok(exists)
     }
 
-    pub async fn scaleup_origin_by_id(context: &Context, contact_id: u32, origin: Origin) -> bool {
+    pub async fn scaleup_origin_by_id(
+        context: &Context,
+        contact_id: u32,
+        origin: Origin,
+    ) -> Result<()> {
         context
             .sql
             .execute(
                 "UPDATE contacts SET origin=? WHERE id=? AND origin<?;",
                 paramsv![origin, contact_id as i32, origin],
             )
-            .await
-            .is_ok()
+            .await?;
+        Ok(())
     }
 }
 
@@ -1203,7 +1210,8 @@ WHERE type=? AND id IN (
         // also unblock mailinglist
         // if the contact is a mailinglist address explicitly created to allow unblocking
         if !new_blocking && contact.origin == Origin::MailinglistAddress {
-            if let Some((chat_id, _, _)) = chat::get_chat_id_by_grpid(context, contact.addr).await?
+            if let Some((chat_id, _, _)) =
+                chat::get_chat_id_by_grpid(context, &contact.addr).await?
             {
                 chat_id.unblock(context).await?;
             }
@@ -1292,14 +1300,35 @@ pub(crate) async fn set_status(
     Ok(())
 }
 
+/// Updates last seen timestamp of the contact if it is earlier than the given `timestamp`.
+pub(crate) async fn update_last_seen(
+    context: &Context,
+    contact_id: u32,
+    timestamp: i64,
+) -> Result<()> {
+    ensure!(
+        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+        "Can not update special contact last seen timestamp"
+    );
+
+    context
+        .sql
+        .execute(
+            "UPDATE contacts SET last_seen = ?1 WHERE last_seen < ?1 AND id = ?2",
+            paramsv![timestamp, contact_id],
+        )
+        .await?;
+    Ok(())
+}
+
 /// Normalize a name.
 ///
 /// - Remove quotes (come from some bad MUA implementations)
 /// - Trims the resulting string
 ///
 /// Typically, this function is not needed as it is called implicitly by `Contact::add_address_book`.
-pub fn normalize_name(full_name: impl AsRef<str>) -> String {
-    let full_name = full_name.as_ref().trim();
+pub fn normalize_name(full_name: &str) -> String {
+    let full_name = full_name.trim();
     if full_name.is_empty() {
         return full_name.into();
     }
@@ -1342,44 +1371,45 @@ fn cat_fingerprint(
 impl Context {
     /// determine whether the specified addr maps to the/a self addr
     pub async fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        let self_addr = self
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .ok_or_else(|| format_err!("Not configured"))?;
-
-        Ok(addr_cmp(self_addr, addr))
+        if let Some(self_addr) = self.get_config(Config::ConfiguredAddr).await? {
+            Ok(addr_cmp(&self_addr, addr))
+        } else {
+            Ok(false)
+        }
     }
 }
 
-pub fn addr_cmp(addr1: impl AsRef<str>, addr2: impl AsRef<str>) -> bool {
-    let norm1 = addr_normalize(addr1.as_ref()).to_lowercase();
-    let norm2 = addr_normalize(addr2.as_ref()).to_lowercase();
+pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
+    let norm1 = addr_normalize(addr1).to_lowercase();
+    let norm2 = addr_normalize(addr2).to_lowercase();
 
     norm1 == norm2
 }
 
 fn split_address_book(book: &str) -> Vec<(&str, &str)> {
     book.lines()
+        .collect::<Vec<&str>>()
         .chunks(2)
         .into_iter()
-        .filter_map(|mut chunk| {
-            let name = chunk.next().unwrap();
-            let addr = match chunk.next() {
-                Some(a) => a,
-                None => return None,
-            };
-            Some((name, addr))
+        .filter_map(|chunk| {
+            let name = chunk.get(0)?;
+            let addr = chunk.get(1)?;
+            Some((*name, *addr))
         })
         .collect()
 }
 
 #[cfg(test)]
 mod tests {
+    use async_std::fs::File;
+    use async_std::io::WriteExt;
+
     use super::*;
 
     use crate::chat::send_text_msg;
+    use crate::dc_receive_imf::dc_receive_imf;
     use crate::message::Message;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::{self, TestContext};
 
     #[test]
     fn test_may_be_valid_addr() {
@@ -1494,11 +1524,11 @@ mod tests {
     #[async_std::test]
     async fn test_is_self_addr() -> Result<()> {
         let t = TestContext::new().await;
-        assert!(t.is_self_addr("me@me.org").await.is_err());
-
-        let addr = t.configure_alice().await;
         assert_eq!(t.is_self_addr("me@me.org").await?, false);
-        assert_eq!(t.is_self_addr(&addr).await?, true);
+
+        t.configure_addr("you@you.net").await;
+        assert_eq!(t.is_self_addr("me@me.org").await?, false);
+        assert_eq!(t.is_self_addr("you@you.net").await?, true);
 
         Ok(())
     }
@@ -1861,7 +1891,7 @@ mod tests {
 
         let alice = TestContext::new_alice().await;
 
-        let id = Contact::lookup_id_by_addr(&alice.ctx, "alice@example.com", Origin::Unknown)
+        let id = Contact::lookup_id_by_addr(&alice.ctx, "alice@example.org", Origin::Unknown)
             .await
             .unwrap();
         assert_eq!(id, Some(DC_CONTACT_ID_SELF));
@@ -1886,7 +1916,7 @@ mod tests {
 
         let bob = TestContext::new_bob().await;
         let chat_alice = bob
-            .create_chat_with_contact("Alice", "alice@example.com")
+            .create_chat_with_contact("Alice", "alice@example.org")
             .await;
         send_text_msg(&bob, chat_alice.id, "Hello".to_string()).await?;
         let msg = bob.pop_sent_msg().await;
@@ -1898,7 +1928,7 @@ mod tests {
             "End-to-end encryption preferred.
 Fingerprints:
 
-alice@example.com:
+alice@example.org:
 2E6F A2CB 23B5 32D7 2863
 4B58 64B0 8F61 A9ED 9443
 
@@ -1949,7 +1979,7 @@ CCCB 5AA9 F6E1 141C 9431
 
         // Bob replies.
         let chat = bob
-            .create_chat_with_contact("Alice", "alice@example.com")
+            .create_chat_with_contact("Alice", "alice@example.org")
             .await;
 
         send_text_msg(&bob, chat.id, "Reply".to_string()).await?;
@@ -1975,4 +2005,94 @@ CCCB 5AA9 F6E1 141C 9431
 
         Ok(())
     }
+
+    /// Tests that DC_EVENT_SELFAVATAR_CHANGED is emitted on avatar changes.
+    #[async_std::test]
+    async fn test_selfavatar_changed_event() -> Result<()> {
+        // Alice has two devices.
+        let alice1 = TestContext::new_alice().await;
+        let alice2 = TestContext::new_alice().await;
+
+        // Bob has one device.
+        let bob = TestContext::new_bob().await;
+
+        assert_eq!(alice1.get_config(Config::Selfavatar).await?, None);
+
+        let avatar_src = alice1.get_blobdir().join("avatar.png");
+        File::create(&avatar_src)
+            .await?
+            .write_all(test_utils::AVATAR_900x900_BYTES)
+            .await?;
+
+        alice1
+            .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await?;
+
+        alice1
+            .evtracker
+            .get_matching(|e| matches!(e, EventType::SelfavatarChanged))
+            .await;
+
+        // Bob sends a message so that Alice can encrypt to him.
+        let chat = bob
+            .create_chat_with_contact("Alice", "alice@example.org")
+            .await;
+
+        send_text_msg(&bob, chat.id, "Reply".to_string()).await?;
+        let sent_msg = bob.pop_sent_msg().await;
+        alice1.recv_msg(&sent_msg).await;
+        alice2.recv_msg(&sent_msg).await;
+
+        // Alice sends a message.
+        let alice1_chat_id = alice1.get_last_msg().await.chat_id;
+        alice1_chat_id.accept(&alice1).await?;
+        send_text_msg(&alice1, alice1_chat_id, "Hello".to_string()).await?;
+        let sent_msg = alice1.pop_sent_msg().await;
+
+        // The message is encrypted.
+        let message = Message::load_from_db(&alice1, sent_msg.sender_msg_id).await?;
+        assert!(message.get_showpadlock());
+
+        // Alice's second device receives a copy of the outgoing message.
+        alice2.recv_msg(&sent_msg).await;
+
+        // Alice's second device applies the selfavatar.
+        assert!(alice2.get_config(Config::Selfavatar).await?.is_some());
+        alice2
+            .evtracker
+            .get_matching(|e| matches!(e, EventType::SelfavatarChanged))
+            .await;
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_last_seen() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        let (contact_id, _) =
+            Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated)
+                .await?;
+        let contact = Contact::load_from_db(&alice, contact_id).await?;
+        assert_eq!(contact.last_seen(), 0);
+
+        let mime = br#"Subject: Hello
+Message-ID: message@example.net
+To: Alice <alice@example.org>
+From: Bob <bob@example.net>
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+Chat-Version: 1.0
+Date: Sun, 22 Mar 2020 22:37:55 +0000
+
+Hi."#;
+        dc_receive_imf(&alice, mime, "Inbox", false).await?;
+        let msg = alice.get_last_msg().await;
+
+        let timestamp = msg.get_timestamp();
+        assert!(timestamp > 0);
+        let contact = Contact::load_from_db(&alice, contact_id).await?;
+        assert_eq!(contact.last_seen(), timestamp);
+
+        Ok(())
+    }
 }

src/context.rs

@@ -1,4 +1,4 @@
-//! Context module
+//! Context module.
 
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
@@ -47,7 +47,6 @@ pub struct InnerContext {
     /// Blob directory path
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
-    pub(crate) os_name: Option<String>,
     pub(crate) bob: Bob,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
@@ -76,6 +75,11 @@ pub struct InnerContext {
     pub(crate) id: u32,
 
     creation_time: SystemTime,
+
+    /// The text of the last error logged and emitted as an event.
+    /// If the ui wants to display an error after a failure,
+    /// `last_error` should be used to avoid races with the event thread.
+    pub(crate) last_error: RwLock<String>,
 }
 
 #[derive(Debug)]
@@ -103,7 +107,7 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 
 impl Context {
     /// Creates new context.
-    pub async fn new(os_name: String, dbfile: PathBuf, id: u32) -> Result<Context> {
+    pub async fn new(dbfile: PathBuf, id: u32) -> Result<Context> {
         // pretty_env_logger::try_init_timed().ok();
 
         let mut blob_fname = OsString::new();
@@ -113,11 +117,10 @@ impl Context {
         if !blobdir.exists().await {
             async_std::fs::create_dir_all(&blobdir).await?;
         }
-        Context::with_blobdir(os_name, dbfile, blobdir, id).await
+        Context::with_blobdir(dbfile, blobdir, id).await
     }
 
     pub(crate) async fn with_blobdir(
-        os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
         id: u32,
@@ -132,7 +135,6 @@ impl Context {
             id,
             blobdir,
             dbfile,
-            os_name: Some(os_name),
             running_state: RwLock::new(Default::default()),
             sql: Sql::new(),
             bob: Default::default(),
@@ -147,6 +149,7 @@ impl Context {
             quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
+            last_error: RwLock::new("".to_string()),
         };
 
         let ctx = Context {
@@ -289,6 +292,7 @@ impl Context {
         let request_msgs = message::get_request_msg_cnt(self).await as usize;
         let contacts = Contact::get_real_cnt(self).await? as usize;
         let is_configured = self.get_config_int(Config::Configured).await?;
+        let socks5_enabled = self.get_config_int(Config::Socks5Enabled).await?;
         let dbversion = self
             .sql
             .get_raw_config_int("dbversion")
@@ -302,6 +306,7 @@ impl Context {
         let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await?;
         let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await?;
         let bcc_self = self.get_config_int(Config::BccSelf).await?;
+        let send_sync_msgs = self.get_config_int(Config::SendSyncMsgs).await?;
 
         let prv_key_cnt = self
             .sql
@@ -317,9 +322,7 @@ impl Context {
             Err(err) => format!("<key failure: {}>", err),
         };
 
-        let inbox_watch = self.get_config_int(Config::InboxWatch).await?;
         let sentbox_watch = self.get_config_int(Config::SentboxWatch).await?;
-        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await?;
         let mvbox_move = self.get_config_int(Config::MvboxMove).await?;
         let sentbox_move = self.get_config_int(Config::SentboxMove).await?;
         let folders_configured = self
@@ -357,6 +360,7 @@ impl Context {
                 .unwrap_or_else(|| "<unset>".to_string()),
         );
         res.insert("is_configured", is_configured.to_string());
+        res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
         res.insert(
@@ -369,9 +373,13 @@ impl Context {
             "show_emails",
             self.get_config_int(Config::ShowEmails).await?.to_string(),
         );
-        res.insert("inbox_watch", inbox_watch.to_string());
+        res.insert(
+            "download_limit",
+            self.get_config_int(Config::DownloadLimit)
+                .await?
+                .to_string(),
+        );
         res.insert("sentbox_watch", sentbox_watch.to_string());
-        res.insert("mvbox_watch", mvbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
         res.insert("sentbox_move", sentbox_move.to_string());
         res.insert("folders_configured", folders_configured.to_string());
@@ -384,6 +392,7 @@ impl Context {
             self.get_config_int(Config::KeyGenType).await?.to_string(),
         );
         res.insert("bcc_self", bcc_self.to_string());
+        res.insert("send_sync_msgs", send_sync_msgs.to_string());
         res.insert("private_key_count", prv_key_cnt.to_string());
         res.insert("public_key_count", pub_key_cnt.to_string());
         res.insert("fingerprint", fingerprint_str);
@@ -421,6 +430,12 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "quota_exceeding",
+            self.get_config_int(Config::QuotaExceeding)
+                .await?
+                .to_string(),
+        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
@@ -631,7 +646,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new("FakeOs".into(), dbfile.into(), 1).await;
+        let res = Context::new(dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -649,7 +664,7 @@ mod tests {
             .unwrap();
         let msg = format!(
             "From: {}\n\
-             To: alice@example.com\n\
+             To: alice@example.org\n\
              Message-ID: <{}>\n\
              Chat-Version: 1.0\n\
              Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
@@ -659,7 +674,7 @@ mod tests {
             dc_create_outgoing_rfc724_mid(None, contact.get_addr())
         );
         println!("{}", msg);
-        dc_receive_imf(t, msg.as_bytes(), "INBOX", 1, false)
+        dc_receive_imf(t, msg.as_bytes(), "INBOX", false)
             .await
             .unwrap();
     }
@@ -782,9 +797,7 @@ mod tests {
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new("FakeOS".into(), dbfile.into(), 1)
-            .await
-            .unwrap();
+        Context::new(dbfile.into(), 1).await.unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -795,7 +808,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new("FakeOS".into(), dbfile.into(), 1).await;
+        let res = Context::new(dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -805,9 +818,7 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new("FakeOS".into(), dbfile.into(), 1)
-            .await
-            .unwrap();
+        Context::new(dbfile.into(), 1).await.unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -817,7 +828,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir, 1).await;
+        let res = Context::with_blobdir(dbfile.into(), blobdir, 1).await;
         assert!(res.is_err());
     }
 
@@ -826,7 +837,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
+        let res = Context::with_blobdir(dbfile.into(), blobdir.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -878,6 +889,10 @@ mod tests {
             "send_security",
             "server_flags",
             "smtp_certificate_checks",
+            "socks5_host",
+            "socks5_port",
+            "socks5_user",
+            "socks5_password",
         ];
         let t = TestContext::new().await;
         let info = t.get_info().await.unwrap();

src/dc_tools.rs

@@ -5,6 +5,7 @@ use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
 use std::io::Cursor;
+use std::str::from_utf8;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
@@ -14,10 +15,13 @@ use async_std::{fs, io};
 
 use anyhow::{bail, Error};
 use chrono::{Local, TimeZone};
+use mailparse::dateparse;
+use mailparse::headers::Headers;
+use mailparse::MailHeaderMap;
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
-use crate::constants::{Viewtype, DC_ELLIPSE, DC_OUTDATED_WARNING_DAYS};
+use crate::constants::{Viewtype, DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::events::EventType;
 use crate::message::Message;
@@ -29,7 +33,7 @@ use crate::stock_str;
 #[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
     let count = buf.chars().count();
-    if approx_chars > 0 && count > approx_chars + DC_ELLIPSE.len() {
+    if count > approx_chars + DC_ELLIPSIS.len() {
         let end_pos = buf
             .char_indices()
             .nth(approx_chars)
@@ -37,9 +41,9 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
             .unwrap_or_default();
 
         if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
-            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSE))
+            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSIS))
         } else {
-            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSE))
+            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSIS))
         }
     } else {
         Cow::Borrowed(buf)
@@ -84,9 +88,9 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
 //   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
 const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 
-// returns the currently smeared timestamp,
-// may be used to check if call to dc_create_smeared_timestamp() is needed or not.
-// the returned timestamp MUST NOT be used to be sent out or saved in the database!
+/// Returns the current smeared timestamp,
+///
+/// The returned timestamp MUST NOT be sent out.
 pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
     let mut now = time();
     let ts = *context.last_smeared_timestamp.read().await;
@@ -97,7 +101,7 @@ pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
     now
 }
 
-// returns a timestamp that is guaranteed to be unique.
+/// Returns a timestamp that is guaranteed to be unique.
 pub(crate) async fn dc_create_smeared_timestamp(context: &Context) -> i64 {
     let now = time();
     let mut ret = now;
@@ -670,13 +674,144 @@ pub fn remove_subject_prefix(last_subject: &str) -> String {
         .to_string()
 }
 
+// Types and methods to create hop-info for message-info
+
+fn extract_address_from_receive_header<'a>(header: &'a str, start: &str) -> Option<&'a str> {
+    let header_len = header.len();
+    header.find(start).and_then(|mut begin| {
+        begin += start.len();
+        let end = header
+            .get(begin..)?
+            .find(|c: char| c.is_whitespace())
+            .unwrap_or(header_len);
+        header.get(begin..begin + end)
+    })
+}
+
+pub(crate) fn parse_receive_header(header: &str) -> String {
+    let header = header.replace(&['\r', '\n'][..], "");
+    let mut hop_info = String::from("Hop: ");
+
+    if let Some(from) = extract_address_from_receive_header(&header, "from ") {
+        hop_info += &format!("From: {}; ", from.trim());
+    }
+
+    if let Some(by) = extract_address_from_receive_header(&header, "by ") {
+        hop_info += &format!("By: {}; ", by.trim());
+    }
+
+    if let Ok(date) = dateparse(&header) {
+        // In tests, use the UTC timezone so that the test is reproducible
+        #[cfg(test)]
+        let date_obj = chrono::Utc.timestamp(date, 0);
+        #[cfg(not(test))]
+        let date_obj = Local.timestamp(date, 0);
+
+        hop_info += &format!("Date: {}", date_obj.to_rfc2822());
+    };
+
+    hop_info
+}
+
+/// parses "receive"-headers
+pub(crate) fn parse_receive_headers(headers: &Headers) -> String {
+    headers
+        .get_all_headers("Received")
+        .iter()
+        .rev()
+        .filter_map(|header_map_item| from_utf8(header_map_item.get_value_raw()).ok())
+        .map(parse_receive_header)
+        .collect::<Vec<_>>()
+        .join("\n")
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
 
-    use crate::test_utils::TestContext;
+    use crate::{
+        config::Config, dc_receive_imf::dc_receive_imf, message::get_msg_info,
+        test_utils::TestContext,
+    };
+
+    #[test]
+    fn test_parse_receive_headers() {
+        // Test `parse_receive_headers()` with some more-or-less random emails from the test-data
+        let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
+        let expected =
+            "Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000\n\
+             Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000";
+        check_parse_receive_headers(raw, expected);
+
+        let raw = include_bytes!("../test-data/message/wrong-html.eml");
+        let expected =
+            "Hop: From: oxbsltgw18.schlund.de; By: mrelayeu.kundenserver.de; Date: Thu, 06 Aug 2020 16:40:31 +0000\n\
+             Hop: From: mout.kundenserver.de; By: dd37930.kasserver.com; Date: Thu, 06 Aug 2020 16:40:32 +0000";
+        check_parse_receive_headers(raw, expected);
+
+        let raw = include_bytes!("../test-data/message/posteo_ndn.eml");
+        let expected =
+            "Hop: By: mout01.posteo.de; Date: Tue, 09 Jun 2020 18:44:22 +0000\n\
+             Hop: From: mout01.posteo.de; By: mx04.posteo.de; Date: Tue, 09 Jun 2020 18:44:22 +0000\n\
+             Hop: From: mx04.posteo.de; By: mailin06.posteo.de; Date: Tue, 09 Jun 2020 18:44:23 +0000\n\
+             Hop: From: mailin06.posteo.de; By: proxy02.posteo.de; Date: Tue, 09 Jun 2020 18:44:23 +0000\n\
+             Hop: From: proxy02.posteo.de; By: proxy02.posteo.name; Date: Tue, 09 Jun 2020 18:44:23 +0000\n\
+             Hop: From: proxy02.posteo.name; By: dovecot03.posteo.local; Date: Tue, 09 Jun 2020 18:44:24 +0000";
+        check_parse_receive_headers(raw, expected);
+    }
+
+    fn check_parse_receive_headers(raw: &[u8], expected: &str) {
+        let mail = mailparse::parse_mail(raw).unwrap();
+        let hop_info = parse_receive_headers(&mail.get_headers());
+        assert_eq!(hop_info, expected)
+    }
+
+    #[async_std::test]
+    async fn test_parse_receive_headers_integration() {
+        let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
+        let expected = r"State: Fresh
+
+hi
+
+Message-ID: 2dfdbde7@example.org
+
+Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000
+Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000";
+        check_parse_receive_headers_integration(raw, expected).await;
+
+        let raw = include_bytes!("../test-data/message/encrypted_with_received_headers.eml");
+        let expected = "State: Fresh, Encrypted
+
+Re: Message from alice@example.org
+
+hi back\r\n\
+\r\n\
+-- \r\n\
+Sent with my Delta Chat Messenger: https://delta.chat
+
+Message-ID: Mr.adQpEwndXLH.LPDdlFVJ7wG@example.net
+
+Hop: From: [127.0.0.1]; By: mail.example.org; Date: Mon, 27 Dec 2021 11:21:21 +0000
+Hop: From: mout.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
+Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000";
+        check_parse_receive_headers_integration(raw, expected).await;
+    }
+
+    async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
+        let t = TestContext::new_alice().await;
+        t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
+        dc_receive_imf(&t, raw, "INBOX", false).await.unwrap();
+        let msg = t.get_last_msg().await;
+        let msg_info = get_msg_info(&t, msg.id).await.unwrap();
+
+        // Ignore the first rows of the msg_info because they contain a
+        // received time that depends on the test time which makes it impossible to
+        // compare with a static string
+        let capped_result = &msg_info[msg_info.find("State").unwrap()..];
+        assert_eq!(expected, capped_result);
+    }
 
     #[test]
     fn test_rust_ftoa() {
@@ -711,10 +846,7 @@ mod tests {
         assert_eq!(dc_truncate("\n  hello \n world", 4), "\n  [...]");
 
         assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
-        assert_eq!(
-            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0),
-            "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ"
-        );
+        assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0), "[...]");
 
         // 9 characters, so no truncation
         assert_eq!(dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
@@ -838,8 +970,8 @@ mod tests {
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
-    use crate::chat;
     use crate::chatlist::Chatlist;
+    use crate::{chat, test_utils};
     use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
     use proptest::prelude::*;
 
@@ -847,22 +979,18 @@ mod tests {
         #[test]
         fn test_dc_truncate(
             buf: String,
-            approx_chars in 0..10000usize
+            approx_chars in 0..100usize
         ) {
             let res = dc_truncate(&buf, approx_chars);
             let el_len = 5;
             let l = res.chars().count();
-            if approx_chars > 0 {
-                assert!(
-                    l <= approx_chars + el_len,
-                    "buf: '{}' - res: '{}' - len {}, approx {}",
-                    &buf, &res, res.len(), approx_chars
-                );
-            } else {
-                assert_eq!(&res, &buf);
-            }
+            assert!(
+                l <= approx_chars + el_len,
+                "buf: '{}' - res: '{}' - len {}, approx {}",
+                &buf, &res, res.len(), approx_chars
+            );
 
-            if approx_chars > 0 && buf.chars().count() > approx_chars + el_len {
+            if buf.chars().count() > approx_chars + el_len {
                 let l = res.len();
                 assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
             }
@@ -989,8 +1117,7 @@ mod tests {
 
     #[test]
     fn test_get_filemeta() {
-        let data = include_bytes!("../test-data/image/avatar900x900.png");
-        let (w, h) = dc_get_filemeta(data).unwrap();
+        let (w, h) = dc_get_filemeta(test_utils::AVATAR_900x900_BYTES).unwrap();
         assert_eq!(w, 900);
         assert_eq!(h, 900);
 

src/dehtml.rs

@@ -1,4 +1,4 @@
-//! De-HTML
+//! De-HTML.
 //!
 //! A module to remove HTML tags from the email text
 
@@ -36,9 +36,9 @@ impl Dehtml {
             ""
         }
     }
-    fn append_prefix(&self, line_end: impl AsRef<str>) -> String {
+    fn append_prefix(&self, line_end: &str) -> String {
         // line_end is e.g. "\n\n". We add "> " if necessary.
-        line_end.as_ref().to_owned() + self.line_prefix()
+        line_end.to_string() + self.line_prefix()
     }
     fn get_add_text(&self) -> AddText {
         if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {

src/download.rs

@@ -0,0 +1,396 @@
+//! # Download large messages manually.
+
+use anyhow::{anyhow, Result};
+use deltachat_derive::{FromSql, ToSql};
+use serde::{Deserialize, Serialize};
+
+use crate::config::Config;
+use crate::constants::Viewtype;
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::imap::{Imap, ImapActionResult};
+use crate::job::{self, Action, Job, Status};
+use crate::message::{Message, MsgId};
+use crate::mimeparser::{MimeMessage, Part};
+use crate::param::Params;
+use crate::{job_try, stock_str, EventType};
+use std::cmp::max;
+
+/// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`.
+///
+/// Some messages as non-delivery-reports (NDN) or read-receipts (MDN)
+/// need to be downloaded completely to handle them correctly,
+/// eg. to assign them to the correct chat.
+/// As these messages are typically small,
+/// they're catched by `MIN_DOWNLOAD_LIMIT`.
+const MIN_DOWNLOAD_LIMIT: u32 = 32768;
+
+/// If a message is downloaded only partially
+/// and `delete_server_after` is set to small timeouts (eg. "at once"),
+/// the user might have no chance to actually download that message.
+/// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case.
+pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
+
+#[derive(
+    Debug,
+    Display,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    FromPrimitive,
+    ToPrimitive,
+    FromSql,
+    ToSql,
+    Serialize,
+    Deserialize,
+)]
+#[repr(u32)]
+pub enum DownloadState {
+    Done = 0,
+    Available = 10,
+    Failure = 20,
+    InProgress = 1000,
+}
+
+impl Default for DownloadState {
+    fn default() -> Self {
+        DownloadState::Done
+    }
+}
+
+impl Context {
+    // Returns validated download limit or `None` for "no limit".
+    pub(crate) async fn download_limit(&self) -> Result<Option<u32>> {
+        let download_limit = self.get_config_int(Config::DownloadLimit).await?;
+        if download_limit <= 0 {
+            Ok(None)
+        } else {
+            Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32)))
+        }
+    }
+}
+
+impl MsgId {
+    /// Schedules full message download for partially downloaded message.
+    pub async fn download_full(self, context: &Context) -> Result<()> {
+        let msg = Message::load_from_db(context, self).await?;
+        match msg.download_state() {
+            DownloadState::Done => return Err(anyhow!("Nothing to download.")),
+            DownloadState::InProgress => return Err(anyhow!("Download already in progress.")),
+            DownloadState::Available | DownloadState::Failure => {
+                self.update_download_state(context, DownloadState::InProgress)
+                    .await?;
+                job::add(
+                    context,
+                    Job::new(Action::DownloadMsg, self.to_u32(), Params::new(), 0),
+                )
+                .await?;
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) async fn update_download_state(
+        self,
+        context: &Context,
+        download_state: DownloadState,
+    ) -> Result<()> {
+        let msg = Message::load_from_db(context, self).await?;
+        context
+            .sql
+            .execute(
+                "UPDATE msgs SET download_state=? WHERE id=?;",
+                paramsv![download_state, self],
+            )
+            .await?;
+        context.emit_event(EventType::MsgsChanged {
+            chat_id: msg.chat_id,
+            msg_id: self,
+        });
+        Ok(())
+    }
+}
+
+impl Message {
+    /// Returns the download state of the message.
+    pub fn download_state(&self) -> DownloadState {
+        self.download_state
+    }
+}
+
+impl Job {
+    /// Actually download a message.
+    /// Called in response to `Action::DownloadMsg`.
+    pub(crate) async fn download_msg(&self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.prepare(context).await {
+            warn!(context, "download: could not connect: {:?}", err);
+            return Status::RetryNow;
+        }
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let row = job_try!(
+            context
+                .sql
+                .query_row_optional(
+                    "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''",
+                    paramsv![msg.rfc724_mid],
+                    |row| {
+                        let server_uid: u32 = row.get(0)?;
+                        let server_folder: String = row.get(1)?;
+                        Ok((server_uid, server_folder))
+                    }
+                )
+                .await
+        );
+
+        if let Some((server_uid, server_folder)) = row {
+            match imap
+                .fetch_single_msg(context, &server_folder, server_uid)
+                .await
+            {
+                ImapActionResult::RetryLater | ImapActionResult::Failed => {
+                    job_try!(
+                        msg.id
+                            .update_download_state(context, DownloadState::Failure)
+                            .await
+                    );
+                    Status::Finished(Err(anyhow!("Call download_full() again to try over.")))
+                }
+                ImapActionResult::Success => {
+                    // update_download_state() not needed as receive_imf() already
+                    // set the state and emitted the event.
+                    Status::Finished(Ok(()))
+                }
+            }
+        } else {
+            // No IMAP record found, we don't know the UID and folder.
+            job_try!(
+                msg.id
+                    .update_download_state(context, DownloadState::Failure)
+                    .await
+            );
+            Status::Finished(Err(anyhow!("Call download_full() again to try over.")))
+        }
+    }
+}
+
+impl Imap {
+    /// Download a single message and pipe it to receive_imf().
+    ///
+    /// receive_imf() is not directly aware that this is a result of a call to download_msg(),
+    /// however, implicitly knows that as the existing message is flagged as being partly.
+    async fn fetch_single_msg(
+        &mut self,
+        context: &Context,
+        folder: &str,
+        uid: u32,
+    ) -> ImapActionResult {
+        if let Some(imapresult) = self
+            .prepare_imap_operation_on_msg(context, folder, uid)
+            .await
+        {
+            return imapresult;
+        }
+
+        // we are connected, and the folder is selected
+        info!(context, "Downloading message {}/{} fully...", folder, uid);
+
+        let (last_uid, _received) = self
+            .fetch_many_msgs(context, folder, vec![uid], false, false)
+            .await;
+        if last_uid.is_none() {
+            ImapActionResult::Failed
+        } else {
+            ImapActionResult::Success
+        }
+    }
+}
+
+impl MimeMessage {
+    /// Creates a placeholder part and add that to `parts`.
+    ///
+    /// To create the placeholder, only the outermost header can be used,
+    /// the mime-structure itself is not available.
+    ///
+    /// The placeholder part currently contains a text with size and availability of the message;
+    /// in the future, we may do more advanced things as previews here.
+    pub(crate) async fn create_stub_from_partial_download(
+        &mut self,
+        context: &Context,
+        org_bytes: u32,
+    ) -> Result<()> {
+        let mut text = format!(
+            "[{}]",
+            stock_str::partial_download_msg_body(context, org_bytes).await
+        );
+        if let Some(delete_server_after) = context.get_config_delete_server_after().await? {
+            let until = stock_str::download_availability(
+                context,
+                time() + max(delete_server_after, MIN_DELETE_SERVER_AFTER),
+            )
+            .await;
+            text += format!(" [{}]", until).as_str();
+        };
+
+        info!(context, "Partial download: {}", text);
+
+        self.parts.push(Part {
+            typ: Viewtype::Text,
+            msg: text,
+            ..Default::default()
+        });
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chat::send_msg;
+    use crate::constants::Viewtype;
+    use crate::dc_receive_imf::dc_receive_imf_inner;
+    use crate::ephemeral::Timer;
+    use crate::test_utils::TestContext;
+    use num_traits::FromPrimitive;
+
+    #[test]
+    fn test_downloadstate_values() {
+        // values may be written to disk and must not change
+        assert_eq!(DownloadState::Done, DownloadState::default());
+        assert_eq!(DownloadState::Done, DownloadState::from_i32(0).unwrap());
+        assert_eq!(
+            DownloadState::Available,
+            DownloadState::from_i32(10).unwrap()
+        );
+        assert_eq!(DownloadState::Failure, DownloadState::from_i32(20).unwrap());
+        assert_eq!(
+            DownloadState::InProgress,
+            DownloadState::from_i32(1000).unwrap()
+        );
+    }
+
+    #[async_std::test]
+    async fn test_download_limit() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        assert_eq!(t.download_limit().await?, None);
+
+        t.set_config(Config::DownloadLimit, Some("200000")).await?;
+        assert_eq!(t.download_limit().await?, Some(200000));
+
+        t.set_config(Config::DownloadLimit, Some("20000")).await?;
+        assert_eq!(t.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
+
+        t.set_config(Config::DownloadLimit, None).await?;
+        assert_eq!(t.download_limit().await?, None);
+
+        for val in &["0", "-1", "-100", "", "foo"] {
+            t.set_config(Config::DownloadLimit, Some(val)).await?;
+            assert_eq!(t.download_limit().await?, None);
+        }
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_update_download_state() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat = t.create_chat_with_contact("Bob", "bob@example.org").await;
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("Hi Bob".to_owned()));
+        let msg_id = send_msg(&t, chat.id, &mut msg).await?;
+        let msg = Message::load_from_db(&t, msg_id).await?;
+        assert_eq!(msg.download_state(), DownloadState::Done);
+
+        for s in &[
+            DownloadState::Available,
+            DownloadState::InProgress,
+            DownloadState::Failure,
+            DownloadState::Done,
+        ] {
+            msg_id.update_download_state(&t, *s).await?;
+            let msg = Message::load_from_db(&t, msg_id).await?;
+            assert_eq!(msg.download_state(), *s);
+        }
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_partial_receive_imf() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        let header =
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+             From: bob@example.com\n\
+             To: alice@example.org\n\
+             Subject: foo\n\
+             Message-ID: <Mr.12345678901@example.com>\n\
+             Chat-Version: 1.0\n\
+             Date: Sun, 22 Mar 2020 22:37:57 +0000\
+             Content-Type: text/plain";
+
+        dc_receive_imf_inner(&t, header.as_bytes(), "INBOX", false, Some(100000), false).await?;
+        let msg = t.get_last_msg().await;
+        assert_eq!(msg.download_state(), DownloadState::Available);
+        assert_eq!(msg.get_subject(), "foo");
+        assert!(msg
+            .get_text()
+            .unwrap()
+            .contains(&stock_str::partial_download_msg_body(&t, 100000).await));
+
+        dc_receive_imf_inner(
+            &t,
+            format!("{}\n\n100k text...", header).as_bytes(),
+            "INBOX",
+            false,
+            None,
+            false,
+        )
+        .await?;
+        let msg = t.get_last_msg().await;
+        assert_eq!(msg.download_state(), DownloadState::Done);
+        assert_eq!(msg.get_subject(), "foo");
+        assert_eq!(msg.get_text(), Some("100k text...".to_string()));
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_partial_download_and_ephemeral() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = t
+            .create_chat_with_contact("bob", "bob@example.org")
+            .await
+            .id;
+        chat_id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
+            .await?;
+
+        // download message from bob partially, this must not change the ephemeral timer
+        dc_receive_imf_inner(
+            &t,
+            b"From: Bob <bob@example.org>\n\
+                    To: Alice <alice@example.org>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: subject\n\
+                    Message-ID: <first@example.org>\n\
+                    Date: Sun, 14 Nov 2021 00:10:00 +0000\
+                    Content-Type: text/plain",
+            "INBOX",
+            false,
+            Some(100000),
+            false,
+        )
+        .await?;
+        assert_eq!(
+            chat_id.get_ephemeral_timer(&t).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        Ok(())
+    }
+}

src/e2ee.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashSet;
 
-use anyhow::{bail, ensure, format_err, Result};
+use anyhow::{bail, format_err, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
@@ -161,21 +161,24 @@ pub async fn try_decrypt(
     let mut peerstate = Peerstate::from_addr(context, &from).await?;
 
     // Apply Autocrypt header
-    if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
-        if let Some(ref mut peerstate) = peerstate {
-            peerstate.apply_header(header, message_time);
-            peerstate.save_to_db(&context.sql, false).await?;
-        } else {
-            let p = Peerstate::from_header(header, message_time);
-            p.save_to_db(&context.sql, true).await?;
-            peerstate = Some(p);
+    match Aheader::from_headers(&from, &mail.headers) {
+        Ok(Some(ref header)) => {
+            if let Some(ref mut peerstate) = peerstate {
+                peerstate.apply_header(header, message_time);
+                peerstate.save_to_db(&context.sql, false).await?;
+            } else {
+                let p = Peerstate::from_header(header, message_time);
+                p.save_to_db(&context.sql, true).await?;
+                peerstate = Some(p);
+            }
         }
+        Ok(None) => {}
+        Err(err) => warn!(context, "Failed to parse Autocrypt header: {}", err),
     }
 
     // Possibly perform decryption
     let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
     let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
-    let mut signatures = HashSet::default();
 
     if let Some(ref mut peerstate) = peerstate {
         peerstate
@@ -188,14 +191,17 @@ pub async fn try_decrypt(
         }
     }
 
-    let out_mail = decrypt_if_autocrypt_message(
+    let (out_mail, signatures) = match decrypt_if_autocrypt_message(
         context,
         mail,
         private_keyring,
         public_keyring_for_validate,
-        &mut signatures,
     )
-    .await?;
+    .await?
+    {
+        Some((out_mail, signatures)) => (Some(out_mail), signatures),
+        None => (None, Default::default()),
+    };
 
     if let Some(mut peerstate) = peerstate {
         // If message is not encrypted and it is not a read receipt, degrade encryption.
@@ -271,8 +277,7 @@ async fn decrypt_if_autocrypt_message(
     mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<Fingerprint>,
-) -> Result<Option<Vec<u8>>> {
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
     let encrypted_data_part = match get_autocrypt_mime(mail).or_else(|| get_mixed_up_mime(mail)) {
         None => {
             // not an autocrypt mime message, abort and ignore
@@ -286,36 +291,60 @@ async fn decrypt_if_autocrypt_message(
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
-        ret_valid_signatures,
     )
     .await
 }
 
+/// Validates signatures of Multipart/Signed message part, as defined in RFC 1847.
+///
+/// Returns `None` if the part is not a Multipart/Signed part, otherwise retruns the set of key
+/// fingerprints for which there is a valid signature.
+async fn validate_detached_signature(
+    mail: &ParsedMail<'_>,
+    public_keyring_for_validate: &Keyring<SignedPublicKey>,
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
+    if mail.ctype.mimetype != "multipart/signed" {
+        return Ok(None);
+    }
+
+    if let [first_part, second_part] = &mail.subparts[..] {
+        // First part is the content, second part is the signature.
+        let content = first_part.raw_bytes;
+        let signature = second_part.get_body_raw()?;
+        let ret_valid_signatures =
+            pgp::pk_validate(content, &signature, public_keyring_for_validate).await?;
+
+        Ok(Some((content.to_vec(), ret_valid_signatures)))
+    } else {
+        Ok(None)
+    }
+}
+
 /// Returns Ok(None) if nothing encrypted was found.
 async fn decrypt_part(
     mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<Fingerprint>,
-) -> Result<Option<Vec<u8>>> {
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
     let data = mail.get_body_raw()?;
 
     if has_decrypted_pgp_armor(&data) {
-        // we should only have one decryption happening
-        ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
+        let (plain, ret_valid_signatures) =
+            pgp::pk_decrypt(data, private_keyring, &public_keyring_for_validate).await?;
 
-        let plain = pgp::pk_decrypt(
-            data,
-            private_keyring,
-            public_keyring_for_validate,
-            Some(ret_valid_signatures),
-        )
-        .await?;
+        // Check for detached signatures.
+        // If decrypted part is a multipart/signed, then there is a detached signature.
+        let decrypted_part = mailparse::parse_mail(&plain)?;
+        if let Some((content, valid_detached_signatures)) =
+            validate_detached_signature(&decrypted_part, &public_keyring_for_validate).await?
+        {
+            return Ok(Some((content, valid_detached_signatures)));
+        } else {
+            // If the message was wrongly or not signed, still return the plain text.
+            // The caller has to check the signatures then.
 
-        // If the message was wrongly or not signed, still return the plain text.
-        // The caller has to check the signatures then.
-
-        return Ok(Some(plain));
+            return Ok(Some((plain, ret_valid_signatures)));
+        }
     }
 
     Ok(None)
@@ -335,7 +364,7 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     false
 }
 
-/// Check if a MIME structure contains a multipart/report part.
+/// Checks if a MIME structure contains a multipart/report part.
 ///
 /// As reports are often unencrypted, we do not reset the Autocrypt header in
 /// this case.
@@ -387,9 +416,11 @@ mod tests {
 
         #[async_std::test]
         async fn test_prexisting() {
-            let t = TestContext::new().await;
-            let test_addr = t.configure_alice().await;
-            assert_eq!(ensure_secret_key_exists(&t).await.unwrap(), test_addr);
+            let t = TestContext::new_alice().await;
+            assert_eq!(
+                ensure_secret_key_exists(&t).await.unwrap(),
+                "alice@example.org"
+            );
         }
 
         #[async_std::test]
@@ -460,7 +491,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert!(!msg.was_encrypted());
 
         // Parsing a message is enough to update peerstate
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
@@ -491,28 +522,28 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
         let msg = bob.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
         // Alice sends plaintext message with Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
-        msg.param.set_int(Param::ForcePlaintext, 1);
+        msg.force_plaintext();
         chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
         chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
         let sent = alice.pop_sent_msg().await;
 
         let msg = bob.parse_msg(&sent).await;
         assert!(!msg.was_encrypted());
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
         // Alice sends plaintext message without Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
-        msg.param.set_int(Param::ForcePlaintext, 1);
+        msg.force_plaintext();
         msg.param.set_int(Param::SkipAutocrypt, 1);
         chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
         chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
@@ -520,7 +551,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
         let msg = bob.parse_msg(&sent).await;
         assert!(!msg.was_encrypted());
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Reset);

src/ephemeral.rs

@@ -1,4 +1,4 @@
-//! # Ephemeral messages
+//! # Ephemeral messages.
 //!
 //! Ephemeral messages are messages that have an Ephemeral-Timer
 //! header attached to them, which specifies time in seconds after
@@ -52,7 +52,7 @@
 //! `MsgsChanged` event is emitted when a message deletion is due, to
 //! make UI reload displayed messages and cause actual deletion.
 //!
-//! Server deletion happens by generating IMAP deletion jobs based on
+//! Server deletion happens by updating the `imap` table based on
 //! the database entries which are expired either according to their
 //! ephemeral message timers or global `delete_server_after` setting.
 
@@ -71,11 +71,12 @@ use crate::constants::{
 };
 use crate::context::Context;
 use crate::dc_tools::time;
+use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
-use crate::job;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::stock_str;
+use std::cmp::max;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -261,7 +262,7 @@ pub(crate) async fn stock_ephemeral_timer_changed(
 
 impl MsgId {
     /// Returns ephemeral message timer value for the message.
-    pub(crate) async fn ephemeral_timer(self, context: &Context) -> anyhow::Result<Timer> {
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> Result<Timer> {
         let res = match context
             .sql
             .query_get_value(
@@ -277,9 +278,9 @@ impl MsgId {
     }
 
     /// Starts ephemeral message timer for the message if it is not started yet.
-    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> anyhow::Result<()> {
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> Result<()> {
         if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
-            let ephemeral_timestamp = time() + i64::from(duration);
+            let ephemeral_timestamp = time().saturating_add(duration.into());
 
             context
                 .sql
@@ -416,58 +417,52 @@ pub async fn schedule_ephemeral_task(context: &Context) {
             let context1 = context.clone();
             let ephemeral_task = task::spawn(async move {
                 async_std::task::sleep(duration).await;
-                emit_event!(
-                    context1,
-                    EventType::MsgsChanged {
-                        chat_id: ChatId::new(0),
-                        msg_id: MsgId::new(0)
-                    }
-                );
+                context1.emit_event(EventType::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0),
+                });
             });
             *context.ephemeral_task.write().await = Some(ephemeral_task);
         } else {
             // Emit event immediately
-            emit_event!(
-                context,
-                EventType::MsgsChanged {
-                    chat_id: ChatId::new(0),
-                    msg_id: MsgId::new(0)
-                }
-            );
+            context.emit_event(EventType::MsgsChanged {
+                chat_id: ChatId::new(0),
+                msg_id: MsgId::new(0),
+            });
         }
     }
 }
 
-/// Returns ID of any expired message that should be deleted from the server.
-///
-/// It looks up the trash chat too, to find messages that are already
-/// deleted locally, but not deleted on the server.
-pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
+/// Schedules expired IMAP messages for deletion.
+pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<()> {
     let now = time();
 
-    let threshold_timestamp = match context.get_config_delete_server_after().await? {
-        None => 0,
-        Some(delete_server_after) => now - delete_server_after,
-    };
+    let (threshold_timestamp, threshold_timestamp_extended) =
+        match context.get_config_delete_server_after().await? {
+            None => (0, 0),
+            Some(delete_server_after) => (
+                now - delete_server_after,
+                now - max(delete_server_after, MIN_DELETE_SERVER_AFTER),
+            ),
+        };
 
     context
         .sql
-        .query_row_optional(
-            "SELECT id FROM msgs \
-         WHERE ( \
-         timestamp < ? \
-         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
-         ) \
-         AND server_uid != 0 \
-         AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
-         LIMIT 1",
-            paramsv![threshold_timestamp, now, job::Action::DeleteMsgOnImap],
-            |row| {
-                let msg_id: MsgId = row.get(0)?;
-                Ok(msg_id)
-            },
+        .execute(
+            "UPDATE imap
+             SET target=''
+             WHERE EXISTS (
+               SELECT * FROM msgs
+               WHERE rfc724_mid=imap.rfc724_mid
+               AND ((download_state = 0 AND timestamp < ?) OR
+                    (download_state != 0 AND timestamp < ?) OR
+                    (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?))
+             )",
+            paramsv![threshold_timestamp, threshold_timestamp_extended, now],
         )
-        .await
+        .await?;
+
+    Ok(())
 }
 
 /// Start ephemeral timers for seen messages if they are not started
@@ -502,10 +497,10 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
 
 #[cfg(test)]
 mod tests {
-    use crate::param::Params;
-    use async_std::task::sleep;
-
     use super::*;
+    use crate::config::Config;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
         chat::{self, Chat, ChatItem},
@@ -641,8 +636,83 @@ mod tests {
         );
     }
 
+    /// Test enabling and disabling ephemeral timer remotely.
     #[async_std::test]
-    async fn test_ephemeral_timer() -> anyhow::Result<()> {
+    async fn test_ephemeral_enable_disable() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
+
+        chat_alice
+            .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
+            .await?;
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        chat_alice
+            .set_ephemeral_timer(&alice.ctx, Timer::Disabled)
+            .await?;
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Disabled
+        );
+
+        Ok(())
+    }
+
+    /// Test that timer is enabled even if the message explicitly enabling the timer is lost.
+    #[async_std::test]
+    async fn test_ephemeral_enable_lost() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
+
+        // Alice enables the timer.
+        chat_alice
+            .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
+            .await?;
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+        // The message enabling the timer is lost.
+        let _sent = alice.pop_sent_msg().await;
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Disabled,
+        );
+
+        // Alice sends a text message.
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        // Bob receives text message and enables the timer, even though explicit timer update was
+        // lost previously.
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        Ok(())
+    }
+
+    /// Test that Alice replying to the chat without a timer at the same time as Bob enables the
+    /// timer does not result in disabling the timer on the Bob's side.
+    #[async_std::test]
+    async fn test_ephemeral_timer_rollback() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
@@ -700,18 +770,30 @@ mod tests {
             Timer::Enabled { duration: 60 }
         );
 
+        // Bob disables the chat timer.
+        // Note that the last message in the Bob's chat is from Alice and has no timer,
+        // but the chat timer is enabled.
+        chat_bob
+            .set_ephemeral_timer(&bob.ctx, Timer::Disabled)
+            .await?;
+        alice.recv_msg(&bob.pop_sent_msg().await).await;
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Disabled
+        );
+
         Ok(())
     }
 
     #[async_std::test]
-    async fn test_ephemeral_delete_msgs() {
+    async fn test_ephemeral_delete_msgs() -> Result<()> {
         let t = TestContext::new_alice().await;
         let chat = t.get_self_chat().await;
 
         t.send_text(chat.id, "Saved message, which we delete manually")
             .await;
         let msg = t.get_last_msg_in(chat.id).await;
-        msg.id.delete_from_db(&t).await.unwrap();
+        msg.id.delete_from_db(&t).await?;
         check_msg_was_deleted(&t, &chat, msg.id).await;
 
         chat.id
@@ -722,36 +804,12 @@ mod tests {
             .send_text(chat.id, "Saved message, disappearing after 1s")
             .await;
 
-        sleep(Duration::from_millis(1100)).await;
+        async_std::task::sleep(Duration::from_millis(1100)).await;
 
-        // Check checks that the msg was deleted locally
+        // Check that the msg was deleted locally.
         check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
 
-        // Check that the msg will be deleted on the server
-        // First of all, set a server_uid so that DC thinks that it's actually possible to delete
-        t.sql
-            .execute(
-                "UPDATE msgs SET server_uid=1 WHERE id=?",
-                paramsv![msg.sender_msg_id],
-            )
-            .await
-            .unwrap();
-        let job = job::load_imap_deletion_job(&t).await.unwrap();
-        assert_eq!(
-            job,
-            Some(job::Job::new(
-                job::Action::DeleteMsgOnImap,
-                msg.sender_msg_id.to_u32(),
-                Params::new(),
-                0,
-            ))
-        );
-        // Let's assume that executing the job fails on first try and the job is saved to the db
-        job.unwrap().save(&t).await.unwrap();
-
-        // Make sure that we don't get yet another job when loading from db
-        let job2 = job::load_imap_deletion_job(&t).await.unwrap();
-        assert_eq!(job2, None);
+        Ok(())
     }
 
     async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
@@ -777,4 +835,202 @@ mod tests {
             assert!(rawtxt.is_none_or_empty(), "{:?}", rawtxt);
         }
     }
+
+    #[async_std::test]
+    async fn test_delete_expired_imap_messages() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        const HOUR: i64 = 60 * 60;
+        let now = time();
+        for (id, timestamp, ephemeral_timestamp) in &[
+            (900, now - 2 * HOUR, 0),
+            (1000, now - 23 * HOUR - MIN_DELETE_SERVER_AFTER, 0),
+            (1010, now - 23 * HOUR, 0),
+            (1020, now - 21 * HOUR, 0),
+            (1030, now - 19 * HOUR, 0),
+            (2000, now - 18 * HOUR, now - HOUR),
+            (2020, now - 17 * HOUR, now + HOUR),
+        ] {
+            let message_id = id.to_string();
+            t.sql
+                   .execute(
+                       "INSERT INTO msgs (id, rfc724_mid, timestamp, ephemeral_timestamp) VALUES (?,?,?,?);",
+                       paramsv![id, message_id, timestamp, ephemeral_timestamp],
+                   )
+                   .await?;
+            t.sql
+                   .execute(
+                       "INSERT INTO imap (rfc724_mid, folder, uid, target) VALUES (?,'INBOX',?, 'INBOX');",
+                       paramsv![message_id, id],
+                   )
+                   .await?;
+        }
+
+        async fn test_marked_for_deletion(context: &Context, id: u32) -> Result<()> {
+            assert_eq!(
+                context
+                    .sql
+                    .count(
+                        "SELECT COUNT(*) FROM imap WHERE target='' AND rfc724_mid=?",
+                        paramsv![id.to_string()],
+                    )
+                    .await?,
+                1
+            );
+            Ok(())
+        }
+
+        async fn remove_uid(context: &Context, id: u32) -> Result<()> {
+            context
+                .sql
+                .execute(
+                    "DELETE FROM imap WHERE rfc724_mid=?",
+                    paramsv![id.to_string()],
+                )
+                .await?;
+            Ok(())
+        }
+
+        // This should mark message 2000 for deletion.
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 2000).await?;
+        remove_uid(&t, 2000).await?;
+        // No other messages are marked for deletion.
+        assert_eq!(
+            t.sql
+                .count("SELECT COUNT(*) FROM imap WHERE target=''", paramsv![],)
+                .await?,
+            0
+        );
+
+        t.set_config(Config::DeleteServerAfter, Some(&*(25 * HOUR).to_string()))
+            .await?;
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 1000).await?;
+
+        MsgId::new(1000)
+            .update_download_state(&t, DownloadState::Available)
+            .await?;
+        t.sql
+            .execute(
+                "UPDATE imap SET target=folder WHERE rfc724_mid='1000'",
+                paramsv![],
+            )
+            .await?;
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 1000).await?; // Delete downloadable anyway.
+        remove_uid(&t, 1000).await?;
+
+        t.set_config(Config::DeleteServerAfter, Some(&*(22 * HOUR).to_string()))
+            .await?;
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 1010).await?;
+        t.sql
+            .execute(
+                "UPDATE imap SET target=folder WHERE rfc724_mid='1010'",
+                paramsv![],
+            )
+            .await?;
+
+        MsgId::new(1010)
+            .update_download_state(&t, DownloadState::Available)
+            .await?;
+        delete_expired_imap_messages(&t).await?;
+        // Keep downloadable for now.
+        assert_eq!(
+            t.sql
+                .count("SELECT COUNT(*) FROM imap WHERE target=''", paramsv![],)
+                .await?,
+            0
+        );
+
+        Ok(())
+    }
+
+    // Regression test for a bug in the timer rollback protection.
+    #[async_std::test]
+    async fn test_ephemeral_timer_references() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        // Message with Message-ID <first@example.com> and no timer is received.
+        dc_receive_imf(
+            &alice,
+            b"From: Bob <bob@example.com>\n\
+                    To: Alice <alice@example.org>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: Subject\n\
+                    Message-ID: <first@example.com>\n\
+                    Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
+                    \n\
+                    hello\n",
+            "INBOX",
+            false,
+        )
+        .await?;
+
+        let msg = alice.get_last_msg().await;
+        let chat_id = msg.chat_id;
+        assert_eq!(chat_id.get_ephemeral_timer(&alice).await?, Timer::Disabled);
+
+        // Message with Message-ID <second@example.com> is received.
+        dc_receive_imf(
+            &alice,
+            b"From: Bob <bob@example.com>\n\
+                    To: Alice <alice@example.org>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: Subject\n\
+                    Message-ID: <second@example.com>\n\
+                    Date: Sun, 22 Mar 2020 00:11:00 +0000\n\
+                    Ephemeral-Timer: 60\n\
+                    \n\
+                    second message\n",
+            "INBOX",
+            false,
+        )
+        .await?;
+        assert_eq!(
+            chat_id.get_ephemeral_timer(&alice).await?,
+            Timer::Enabled { duration: 60 }
+        );
+        let msg = alice.get_last_msg().await;
+
+        // Message is deleted from the database when its timer expires.
+        msg.id.delete_from_db(&alice).await?;
+
+        // Message with Message-ID <third@example.com>, referencing <first@example.com> and
+        // <second@example.com>, is received.  The message <second@example.come> is not in the
+        // database anymore, so the timer should be applied unconditionally without rollback
+        // protection.
+        //
+        // Previously Delta Chat fallen back to using <first@example.com> in this case and
+        // compared received timer value to the timer value of the <first@examle.com>. Because
+        // their timer values are the same ("disabled"), Delta Chat assumed that the timer was not
+        // changed explicitly and the change should be ignored.
+        //
+        // The message also contains a quote of the first message to test that only References:
+        // header and not In-Reply-To: is consulted by the rollback protection.
+        dc_receive_imf(
+            &alice,
+            b"From: Bob <bob@example.com>\n\
+                    To: Alice <alice@example.org>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: Subject\n\
+                    Message-ID: <third@example.com>\n\
+                    Date: Sun, 22 Mar 2020 00:12:00 +0000\n\
+                    References: <first@example.com> <second@example.com>\n\
+                    In-Reply-To: <first@example.com>\n\
+                    \n\
+                    > hello\n",
+            "INBOX",
+            false,
+        )
+        .await?;
+
+        let msg = alice.get_last_msg().await;
+        assert_eq!(
+            msg.chat_id.get_ephemeral_timer(&alice).await?,
+            Timer::Disabled
+        );
+
+        Ok(())
+    }
 }

src/events.rs

@@ -1,4 +1,4 @@
-//! # Events specification
+//! # Events specification.
 
 use std::ops::Deref;
 
@@ -199,7 +199,8 @@ pub enum EventType {
     /// - Chats created, deleted or archived
     /// - A draft has been set
     ///
-    /// The `chat_id` and `msg_id` values will be 0 if more than one message is changed.
+    /// `chat_id` is set if only a single chat is affected by the changes, otherwise 0.
+    /// `msg_id` is set if only a single message is affected by the changes, otherwise 0.
     #[strum(props(id = "2000"))]
     MsgsChanged { chat_id: ChatId, msg_id: MsgId },
 
@@ -322,4 +323,7 @@ pub enum EventType {
     /// dc_get_connectivity_html() for details.
     #[strum(props(id = "2100"))]
     ConnectivityChanged,
+
+    #[strum(props(id = "2110"))]
+    SelfavatarChanged,
 }

src/format_flowed.rs

@@ -1,13 +1,13 @@
-///! # format=flowed support
-///!
-///! Format=flowed is defined in
-///! [RFC 3676](https://tools.ietf.org/html/rfc3676).
-///!
-///! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
-///! during formatting, i.e., DelSp parameter introduced in RFC 3676
-///! is assumed to be set to "no".
-///!
-///! For received messages, DelSp parameter is honoured.
+//! # format=flowed support.
+//!
+//! Format=flowed is defined in
+//! [RFC 3676](https://tools.ietf.org/html/rfc3676).
+//!
+//! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
+//! during formatting, i.e., DelSp parameter introduced in RFC 3676
+//! is assumed to be set to "no".
+//!
+//! For received messages, DelSp parameter is honoured.
 
 /// Wraps line to 72 characters using format=flowed soft breaks.
 ///

src/headerdef.rs

@@ -1,7 +1,8 @@
-use crate::strum::AsStaticRef;
+//! # List of email headers.
+
 use mailparse::{MailHeader, MailHeaderMap};
 
-#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
+#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, IntoStaticStr)]
 #[strum(serialize_all = "kebab_case")]
 pub enum HeaderDef {
     MessageId,
@@ -32,6 +33,7 @@ pub enum HeaderDef {
     XMozillaDraftInfo,
 
     ListId,
+    ListPost,
     References,
     InReplyTo,
     Precedence,
@@ -65,9 +67,9 @@ pub enum HeaderDef {
 }
 
 impl HeaderDef {
-    /// Returns the corresponding Event id.
+    /// Returns the corresponding header string.
     pub fn get_headername(&self) -> &'static str {
-        self.as_static()
+        self.into()
     }
 }
 

src/html.rs

@@ -1,11 +1,12 @@
-///! # Get message as HTML.
-///!
-///! Use `Message.has_html()` to check if the UI shall render a
-///! corresponding button and `MsgId.get_html()` to get the full message.
-///!
-///! Even when the original mime-message is not HTML,
-///! `MsgId.get_html()` will return HTML -
-///! this allows nice quoting, handling linebreaks properly etc.
+//! # Get message as HTML.
+//!
+//! Use `Message.has_html()` to check if the UI shall render a
+//! corresponding button and `MsgId.get_html()` to get the full message.
+//!
+//! Even when the original mime-message is not HTML,
+//! `MsgId.get_html()` will return HTML -
+//! this allows nice quoting, handling linebreaks properly etc.
+
 use futures::future::FutureExt;
 use std::future::Future;
 use std::pin::Pin;
@@ -439,9 +440,7 @@ test some special html-characters as < > and & but also " and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", 1, false)
-            .await
-            .unwrap();
+        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
         assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
@@ -467,7 +466,7 @@ test some special html-characters as < > and & but also " and &#x
 
         // bob: check that bob also got the html-part of the forwarded message
         let bob = TestContext::new_bob().await;
-        let chat = bob.create_chat_with_contact("", "alice@example.com").await;
+        let chat = bob.create_chat_with_contact("", "alice@example.org").await;
         bob.recv_msg(&alice.pop_sent_msg().await).await;
         let msg = bob.get_last_msg_in(chat.get_id()).await;
         assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
@@ -490,9 +489,7 @@ test some special html-characters as < > and & but also " and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", 1, false)
-            .await
-            .unwrap();
+        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
 
         // forward the message to saved-messages,
@@ -559,7 +556,6 @@ test some special html-characters as < > and & but also " and &#x
             &t,
             include_bytes!("../test-data/message/cp1252-html.eml"),
             "INBOX",
-            0,
             false,
         )
         .await?;

src/imap/client.rs

@@ -1,16 +1,23 @@
-use std::ops::{Deref, DerefMut};
-
-use async_imap::{
-    error::{Error as ImapError, Result as ImapResult},
-    Client as ImapClient,
+use std::{
+    ops::{Deref, DerefMut},
+    time::Duration,
 };
+
+use anyhow::{Context as _, Result};
+
+use async_imap::Client as ImapClient;
+
+use async_smtp::ServerAddress;
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::dc_build_tls;
+use crate::login_param::{dc_build_tls, Socks5Config};
 
 use super::session::SessionStream;
 
+/// IMAP write and read timeout in seconds.
+const IMAP_TIMEOUT: u64 = 30;
+
 #[derive(Debug)]
 pub(crate) struct Client {
     is_secure: bool,
@@ -32,24 +39,12 @@ impl DerefMut for Client {
 }
 
 impl Client {
-    pub async fn login(
-        self,
-        username: &str,
-        password: &str,
-    ) -> std::result::Result<Session, (ImapError, Self)> {
-        let Client { inner, is_secure } = self;
+    pub async fn login(self, username: &str, password: &str) -> Result<Session> {
+        let Client { inner, .. } = self;
         let session = inner
             .login(username, password)
             .await
-            .map_err(|(err, client)| {
-                (
-                    err,
-                    Client {
-                        is_secure,
-                        inner: client,
-                    },
-                )
-            })?;
+            .map_err(|(err, _client)| err)?;
         Ok(Session { inner: session })
     }
 
@@ -57,21 +52,12 @@ impl Client {
         self,
         auth_type: &str,
         authenticator: impl async_imap::Authenticator,
-    ) -> std::result::Result<Session, (ImapError, Self)> {
-        let Client { inner, is_secure } = self;
-        let session =
-            inner
-                .authenticate(auth_type, authenticator)
-                .await
-                .map_err(|(err, client)| {
-                    (
-                        err,
-                        Client {
-                            is_secure,
-                            inner: client,
-                        },
-                    )
-                })?;
+    ) -> Result<Session> {
+        let Client { inner, .. } = self;
+        let session = inner
+            .authenticate(auth_type, authenticator)
+            .await
+            .map_err(|(err, _client)| err)?;
         Ok(Session { inner: session })
     }
 
@@ -79,7 +65,7 @@ impl Client {
         addr: impl net::ToSocketAddrs,
         domain: &str,
         strict_tls: bool,
-    ) -> ImapResult<Self> {
+    ) -> Result<Self> {
         let stream = TcpStream::connect(addr).await?;
         let tls = dc_build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> = Box::new(tls.connect(domain, stream).await?);
@@ -88,7 +74,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .context("failed to read greeting")?;
 
         Ok(Client {
             is_secure: true,
@@ -96,14 +82,14 @@ impl Client {
         })
     }
 
-    pub async fn connect_insecure(addr: impl net::ToSocketAddrs) -> ImapResult<Self> {
+    pub async fn connect_insecure(addr: impl net::ToSocketAddrs) -> Result<Self> {
         let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .context("failed to read greeting")?;
 
         Ok(Client {
             is_secure: false,
@@ -111,7 +97,56 @@ impl Client {
         })
     }
 
-    pub async fn secure(self, domain: &str, strict_tls: bool) -> ImapResult<Client> {
+    pub async fn connect_secure_socks5(
+        target_addr: &ServerAddress,
+        strict_tls: bool,
+        socks5_config: Socks5Config,
+    ) -> Result<Self> {
+        let socks5_stream: Box<dyn SessionStream> = Box::new(
+            socks5_config
+                .connect(target_addr, Some(Duration::from_secs(IMAP_TIMEOUT)))
+                .await?,
+        );
+
+        let tls = dc_build_tls(strict_tls);
+        let tls_stream: Box<dyn SessionStream> =
+            Box::new(tls.connect(target_addr.host.clone(), socks5_stream).await?);
+        let mut client = ImapClient::new(tls_stream);
+
+        let _greeting = client
+            .read_response()
+            .await
+            .context("failed to read greeting")?;
+
+        Ok(Client {
+            is_secure: true,
+            inner: client,
+        })
+    }
+
+    pub async fn connect_insecure_socks5(
+        target_addr: &ServerAddress,
+        socks5_config: Socks5Config,
+    ) -> Result<Self> {
+        let socks5_stream: Box<dyn SessionStream> = Box::new(
+            socks5_config
+                .connect(target_addr, Some(Duration::from_secs(IMAP_TIMEOUT)))
+                .await?,
+        );
+
+        let mut client = ImapClient::new(socks5_stream);
+        let _greeting = client
+            .read_response()
+            .await
+            .context("failed to read greeting")?;
+
+        Ok(Client {
+            is_secure: false,
+            inner: client,
+        })
+    }
+
+    pub async fn secure(self, domain: &str, strict_tls: bool) -> Result<Client> {
         if self.is_secure {
             Ok(self)
         } else {

src/imap/idle.rs

@@ -150,7 +150,7 @@ impl Imap {
                     }
                     if self.config.can_idle {
                         // we only fake-idled because network was gone during IDLE, probably
-                        break InterruptInfo::new(false, None);
+                        break InterruptInfo::new(false);
                     }
                     info!(context, "fake_idle is connected");
                     // we are connected, let's see if fetching messages results
@@ -162,7 +162,7 @@ impl Imap {
                         Ok(res) => {
                             info!(context, "fetch_new_messages returned {:?}", res);
                             if res {
-                                break InterruptInfo::new(false, None);
+                                break InterruptInfo::new(false);
                             }
                         }
                         Err(err) => {

src/imap/scan_folders.rs

@@ -10,7 +10,7 @@ use async_std::prelude::*;
 use super::{get_folder_meaning, get_folder_meaning_by_name};
 
 impl Imap {
-    pub async fn scan_folders(&mut self, context: &Context) -> Result<()> {
+    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<()> {
         // First of all, debounce to once per minute:
         let mut last_scan = context.last_full_folder_scan.lock().await;
         if let Some(last_scan) = *last_scan {
@@ -29,7 +29,7 @@ impl Imap {
         let session = self.session.as_mut();
         let session = session.context("scan_folders(): IMAP No Connection established")?;
         let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
-        let watched_folders = get_watched_folders(context).await;
+        let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
 
@@ -74,7 +74,7 @@ impl Imap {
                 self.server_sent_unsolicited_exists(context);
 
                 loop {
-                    self.fetch_new_messages(context, folder.name(), false)
+                    self.fetch_move_delete(context, folder.name())
                         .await
                         .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
 
@@ -102,19 +102,21 @@ impl Imap {
     }
 }
 
-pub(crate) async fn get_watched_folders(context: &Context) -> Vec<String> {
+pub(crate) async fn get_watched_folders(context: &Context) -> Result<Vec<String>> {
     let mut res = Vec::new();
+    if let Some(inbox_folder) = context.get_config(Config::ConfiguredInboxFolder).await? {
+        res.push(inbox_folder);
+    }
     let folder_watched_configured = &[
         (Config::SentboxWatch, Config::ConfiguredSentboxFolder),
-        (Config::MvboxWatch, Config::ConfiguredMvboxFolder),
-        (Config::InboxWatch, Config::ConfiguredInboxFolder),
+        (Config::MvboxMove, Config::ConfiguredMvboxFolder),
     ];
     for (watched, configured) in folder_watched_configured {
-        if context.get_config_bool(*watched).await.unwrap_or_default() {
-            if let Ok(Some(folder)) = context.get_config(*configured).await {
+        if context.get_config_bool(*watched).await? {
+            if let Some(folder) = context.get_config(*configured).await? {
                 res.push(folder);
             }
         }
     }
-    res
+    Ok(res)
 }

src/imap/select_folder.rs

@@ -15,6 +15,9 @@ pub enum Error {
     #[error("IMAP Folder name invalid: {0}")]
     BadFolderName(String),
 
+    #[error("IMAP folder does not exist: {0}")]
+    NoFolder(String),
+
     #[error("IMAP close/expunge failed")]
     CloseExpungeFailed(#[from] async_imap::error::Error),
 
@@ -110,6 +113,9 @@ impl Imap {
                     Err(async_imap::error::Error::Validate(_)) => {
                         Err(Error::BadFolderName(folder.to_string()))
                     }
+                    Err(async_imap::error::Error::No(_)) => {
+                        Err(Error::NoFolder(folder.to_string()))
+                    }
                     Err(err) => {
                         self.config.selected_folder = None;
                         self.trigger_reconnect(context).await;
@@ -123,6 +129,28 @@ impl Imap {
             Ok(NewlySelected::No)
         }
     }
+
+    /// Selects a folder. Tries to create it once and select again if the folder does not exist.
+    pub(super) async fn select_or_create_folder(
+        &mut self,
+        context: &Context,
+        folder: &str,
+    ) -> Result<NewlySelected> {
+        match self.select_folder(context, Some(folder)).await {
+            Ok(newly_selected) => Ok(newly_selected),
+            Err(err) => match err {
+                Error::NoFolder(_) => {
+                    if let Some(ref mut session) = self.session {
+                        session.create(folder).await?;
+                    } else {
+                        return Err(Error::NoSession);
+                    }
+                    self.select_folder(context, Some(folder)).await
+                }
+                _ => Err(err),
+            },
+        }
+    }
 }
 #[derive(PartialEq, Debug, Copy, Clone, Eq)]
 pub(super) enum NewlySelected {

src/imap/session.rs

@@ -3,6 +3,7 @@ use std::ops::{Deref, DerefMut};
 use async_imap::Session as ImapSession;
 use async_native_tls::TlsStream;
 use async_std::net::TcpStream;
+use fast_socks5::client::Socks5Stream;
 
 #[derive(Debug)]
 pub(crate) struct Session {
@@ -17,6 +18,7 @@ pub(crate) trait SessionStream:
 impl SessionStream for TlsStream<Box<dyn SessionStream>> {}
 impl SessionStream for TlsStream<TcpStream> {}
 impl SessionStream for TcpStream {}
+impl SessionStream for Socks5Stream<TcpStream> {}
 
 impl Deref for Session {
     type Target = ImapSession<Box<dyn SessionStream>>;

src/imex.rs

@@ -1,4 +1,4 @@
-//! # Import/export module
+//! # Import/export module.
 
 use std::any::Any;
 use std::ffi::OsStr;
@@ -66,15 +66,15 @@ pub enum ImexMode {
 
 /// Import/export things.
 ///
-/// What to do is defined by the *what* parameter.
+/// What to do is defined by the `what` parameter.
 ///
 /// During execution of the job,
 /// some events are sent out:
 ///
-/// - A number of #DC_EVENT_IMEX_PROGRESS events are sent and may be used to create
+/// - A number of `DC_EVENT_IMEX_PROGRESS` events are sent and may be used to create
 ///   a progress bar or stuff like that. Moreover, you'll be informed when the imex-job is done.
 ///
-/// - For each file written on export, the function sends #DC_EVENT_IMEX_FILE_WRITTEN
+/// - For each file written on export, the function sends `DC_EVENT_IMEX_FILE_WRITTEN`
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
@@ -204,6 +204,7 @@ pub async fn has_backup_old(context: &Context, dir_name: &Path) -> Result<String
     }
 }
 
+/// Initiates key transfer via Autocrypt Setup Message.
 pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
     use futures::future::FutureExt;
 
@@ -233,11 +234,11 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     msg = Message::default();
     msg.viewtype = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
-
+    msg.subject = stock_str::ac_setup_msg_subject(context).await;
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.param.set_int(Param::ForcePlaintext, 1);
+    msg.force_plaintext();
     msg.param.set_int(Param::SkipAutocrypt, 1);
 
     let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
@@ -435,7 +436,7 @@ async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
     Ok(plain_text)
 }
 
-pub fn normalize_setup_code(s: &str) -> String {
+fn normalize_setup_code(s: &str) -> String {
     let mut out = String::new();
     for c in s.chars() {
         if ('0'..='9').contains(&c) {
@@ -506,14 +507,16 @@ async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()>
     let archive = Archive::new(backup_file);
 
     let mut entries = archive.entries()?;
+    let mut last_progress = 0;
     while let Some(file) = entries.next().await {
         let f = &mut file?;
 
         let current_pos = f.raw_file_position();
         let progress = 1000 * current_pos / file_size;
-        if progress > 10 && progress < 1000 {
+        if progress != last_progress && progress > 10 && progress < 1000 {
             // We already emitted ImexProgress(10) above
             context.emit_event(EventType::ImexProgress(progress as usize));
+            last_progress = progress;
         }
 
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
@@ -736,6 +739,7 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
     let count = read_dir.len();
     let mut written_files = 0;
 
+    let mut last_progress = 0;
     for entry in read_dir.into_iter() {
         let entry = entry?;
         let name = entry.file_name();
@@ -753,9 +757,10 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
 
         written_files += 1;
         let progress = 1000 * written_files / count;
-        if progress > 10 && progress < 1000 {
+        if progress != last_progress && progress > 10 && progress < 1000 {
             // We already emitted ImexProgress(10) above
-            emit_event!(context, EventType::ImexProgress(progress));
+            context.emit_event(EventType::ImexProgress(progress));
+            last_progress = progress;
         }
     }
 
@@ -931,9 +936,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file() {
-        let t = TestContext::new().await;
-
-        t.configure_alice().await;
+        let t = TestContext::new_alice().await;
         let msg = render_setup_file(&t, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
@@ -950,11 +953,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
-        t.configure_alice().await;
         let msg = render_setup_file(&t, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
@@ -1007,15 +1009,13 @@ mod tests {
 
     #[async_std::test]
     async fn test_export_and_import_key() {
-        let context = TestContext::new().await;
-        context.configure_alice().await;
+        let context = TestContext::new_alice().await;
         let blobdir = context.ctx.get_blobdir();
         if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir).await {
             panic!("got error on export: {:?}", err);
         }
 
-        let context2 = TestContext::new().await;
-        context2.configure_alice().await;
+        let context2 = TestContext::new_alice().await;
         if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir).await {
             panic!("got error on import: {:?}", err);
         }

src/job.rs

@@ -1,15 +1,13 @@
-//! # Job module
+//! # Job module.
 //!
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
+use std::fmt;
 use std::future::Future;
-use std::{fmt, time::Duration};
 
 use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
 use async_smtp::smtp::response::{Category, Code, Detail};
-use async_std::task::sleep;
 use deltachat_derive::{FromSql, ToSql};
-use itertools::Itertools;
 use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
@@ -18,7 +16,6 @@ use crate::config::Config;
 use crate::contact::{normalize_name, Contact, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_tools::{dc_delete_file, dc_read_file, time};
-use crate::ephemeral::load_imap_deletion_msgid;
 use crate::events::EventType;
 use crate::imap::{Imap, ImapActionResult};
 use crate::location;
@@ -98,10 +95,11 @@ pub enum Action {
     // this is user initiated so it should have a fairly high priority
     UpdateRecentQuota = 140,
 
-    // Moving message is prioritized lower than deletion so we don't
-    // bother moving message if it is already scheduled for deletion.
-    MoveMsg = 200,
-    DeleteMsgOnImap = 210,
+    // This job will download partially downloaded messages completely
+    // and is added when download_full() is called.
+    // Most messages are downloaded automatically on fetch
+    // and do not go through this job.
+    DownloadMsg = 250,
 
     // UID synchronization is high-priority to make sure correct UIDs
     // are used by message moving/deletion.
@@ -129,11 +127,10 @@ impl From<Action> for Thread {
 
             Housekeeping => Thread::Imap,
             FetchExistingMsgs => Thread::Imap,
-            DeleteMsgOnImap => Thread::Imap,
             ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
-            MoveMsg => Thread::Imap,
             UpdateRecentQuota => Thread::Imap,
+            DownloadMsg => Thread::Imap,
 
             MaybeSendLocations => Thread::Smtp,
             MaybeSendLocationsEnded => Thread::Smtp,
@@ -427,6 +424,13 @@ impl Job {
                 }
                 // now also delete the generated file
                 dc_delete_file(context, filename).await;
+
+                // finally, create another send-job if there are items to be synced.
+                // triggering sync-job after msg-send-job guarantees, the recipient has grpid etc.
+                // once the sync message arrives.
+                // if there are no items to sync, this function returns fast.
+                context.send_sync_msg().await?;
+
                 Ok(())
             }
         })
@@ -541,149 +545,6 @@ impl Job {
         .await
     }
 
-    async fn move_msg(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.prepare(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let server_folder = &job_try!(msg
-            .server_folder
-            .context("Can't move message out of folder if we don't know the current folder"));
-
-        let move_res = msg.id.needs_move(context, server_folder).await;
-        let dest_folder = match move_res {
-            Err(e) => {
-                warn!(context, "could not load dest folder: {}", e);
-                return Status::RetryLater;
-            }
-            Ok(None) => {
-                warn!(
-                    context,
-                    "msg {} does not need to be moved from {}", msg.id, server_folder
-                );
-                return Status::Finished(Ok(()));
-            }
-            Ok(Some(config)) => match context.get_config(config).await {
-                Ok(folder) => folder,
-                Err(err) => {
-                    warn!(context, "failed to load config: {}", err);
-                    return Status::RetryLater;
-                }
-            },
-        };
-
-        if let Some(dest_folder) = dest_folder {
-            match imap
-                .mv(context, server_folder, msg.server_uid, &dest_folder)
-                .await
-            {
-                ImapActionResult::RetryLater => Status::RetryLater,
-                ImapActionResult::Success => {
-                    // Rust-Imap provides no target uid on mv, so just set it to 0, update again when precheck_imf() is called for the moved message
-                    message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, 0).await;
-                    Status::Finished(Ok(()))
-                }
-                ImapActionResult::Failed => {
-                    Status::Finished(Err(format_err!("IMAP action failed")))
-                }
-                ImapActionResult::AlreadyDone => Status::Finished(Ok(())),
-            }
-        } else {
-            Status::Finished(Err(format_err!("No mvbox folder configured")))
-        }
-    }
-
-    /// Deletes a message on the server.
-    ///
-    /// `foreign_id` is a MsgId.
-    ///
-    /// If the message is in the trash chat or hidden, this job
-    /// removes database record, otherwise it only clears the
-    /// `server_uid` column.  If there are no more records pointing to
-    /// the same message on the server, the job actually removes the
-    /// message on the server.
-    async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.prepare(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-
-        if !msg.rfc724_mid.is_empty() {
-            let cnt = message::rfc724_mid_cnt(context, &msg.rfc724_mid).await;
-            info!(
-                context,
-                "Running delete job for message {} which has {} entries in the database",
-                &msg.rfc724_mid,
-                cnt
-            );
-            if cnt > 1 {
-                info!(
-                    context,
-                    "The message is deleted from the server when all parts are deleted.",
-                );
-            } else if cnt == 0 {
-                warn!(
-                    context,
-                    "The message {} has no UID on the server to delete", &msg.rfc724_mid
-                );
-            } else {
-                /* if this is the last existing part of the message,
-                we delete the message from the server */
-                let mid = msg.rfc724_mid;
-                let server_folder = msg.server_folder.as_ref().unwrap();
-                let res = if msg.server_uid == 0 {
-                    // Message is already deleted on IMAP server.
-                    ImapActionResult::AlreadyDone
-                } else {
-                    imap.delete_msg(context, &mid, server_folder, msg.server_uid)
-                        .await
-                };
-                match res {
-                    ImapActionResult::AlreadyDone | ImapActionResult::Success => {}
-                    ImapActionResult::RetryLater | ImapActionResult::Failed => {
-                        // If job has failed, for example due to some
-                        // IMAP bug, we postpone it instead of failing
-                        // immediately. This will prevent adding it
-                        // immediately again if user has enabled
-                        // automatic message deletion. Without this,
-                        // we might waste a lot of traffic constantly
-                        // retrying message deletion.
-                        return Status::RetryLater;
-                    }
-                }
-            }
-            if msg.chat_id.is_trash() || msg.hidden {
-                // Messages are stored in trash chat only to keep
-                // their server UID and Message-ID. Once message is
-                // deleted from the server, database record can be
-                // removed as well.
-                //
-                // Hidden messages are similar to trashed, but are
-                // related to some chat. We also delete their
-                // database records.
-                job_try!(msg.id.delete_from_db(context).await)
-            } else {
-                // Remove server UID from the database record.
-                //
-                // We have either just removed the message from the
-                // server, in which case UID is not valid anymore, or
-                // we have more refernces to the same server UID, so
-                // we remove UID to reduce the number of messages
-                // pointing to the corresponding UID. Once the counter
-                // reaches zero, we will remove the message.
-                job_try!(msg.id.unlink(context).await);
-            }
-            Status::Finished(Ok(()))
-        } else {
-            /* eg. device messages have no Message-ID */
-            Status::Finished(Ok(()))
-        }
-    }
-
     /// Read the recipients from old emails sent by the user and add them as contacts.
     /// This way, we can already offer them some email addresses they can write to.
     ///
@@ -709,7 +570,7 @@ impl Job {
                 Config::ConfiguredSentboxFolder,
             ] {
                 if let Some(folder) = job_try!(context.get_config(*config).await) {
-                    if let Err(e) = imap.fetch_new_messages(context, folder, true).await {
+                    if let Err(e) = imap.fetch_new_messages(context, &folder, true).await {
                         // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
                         warn!(context, "Could not fetch messages, retrying: {:#}", e);
                         return Status::RetryLater;
@@ -762,72 +623,75 @@ impl Job {
         }
 
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-
-        let folder = msg.server_folder.as_ref().unwrap();
-
-        let result = if msg.server_uid == 0 {
-            // The message is moved or deleted by us.
-            //
-            // Do not call set_seen with zero UID, as it will return
-            // ImapActionResult::RetryLater, but we do not want to
-            // retry. If the message was moved, we will create another
-            // job to mark the message as seen later. If it was
-            // deleted, there is nothing to do.
-            info!(context, "Can't mark message as seen: No UID");
-            ImapActionResult::Failed
-        } else {
-            imap.set_seen(context, folder, msg.server_uid).await
-        };
-
-        match result {
-            ImapActionResult::RetryLater => Status::RetryLater,
-            ImapActionResult::AlreadyDone => Status::Finished(Ok(())),
-            ImapActionResult::Success | ImapActionResult::Failed => {
-                // XXX the message might just have been moved
-                // we want to send out an MDN anyway
-                // The job will not be retried so locally
-                // there is no risk of double-sending MDNs.
-                //
-                // Read receipts for system messages are never
-                // sent. These messages have no place to display
-                // received read receipt anyway.  And since their text
-                // is locally generated, quoting them is dangerous as
-                // it may contain contact names. E.g., for original
-                // message "Group left by me", a read receipt will
-                // quote "Group left by <name>", and the name can be a
-                // display name stored in address book rather than
-                // the name sent in the From field by the user.
-                if msg.param.get_bool(Param::WantsMdn).unwrap_or_default()
-                    && !msg.is_system_message()
-                {
-                    let mdns_enabled = job_try!(context.get_config_bool(Config::MdnsEnabled).await);
-                    if mdns_enabled {
-                        if let Err(err) = send_mdn(context, &msg).await {
-                            warn!(context, "could not send out mdn for {}: {}", msg.id, err);
-                            return Status::Finished(Err(err));
-                        }
+        let row = job_try!(
+            context
+                .sql
+                .query_row_optional(
+                    "SELECT uid, folder FROM imap
+                    WHERE rfc724_mid=? AND folder=target
+                    ORDER BY uid ASC
+                    LIMIT 1",
+                    paramsv![msg.rfc724_mid],
+                    |row| {
+                        let uid: u32 = row.get(0)?;
+                        let folder: String = row.get(1)?;
+                        Ok((uid, folder))
                     }
+                )
+                .await
+        );
+        if let Some((server_uid, server_folder)) = row {
+            let result = imap.set_seen(context, &server_folder, server_uid).await;
+            match result {
+                ImapActionResult::RetryLater => return Status::RetryLater,
+                ImapActionResult::Success | ImapActionResult::Failed => {}
+            }
+        } else {
+            info!(
+                context,
+                "Can't mark the message {} as seen on IMAP because there is no known UID",
+                msg.rfc724_mid
+            );
+        }
+
+        // XXX we send MDN even in case of failure to mark the messages as seen, e.g. if it was
+        // already deleted on the server by another device. The job will not be retried so locally
+        // there is no risk of double-sending MDNs.
+        //
+        // Read receipts for system messages are never sent. These messages have no place to
+        // display received read receipt anyway.  And since their text is locally generated,
+        // quoting them is dangerous as it may contain contact names. E.g., for original message
+        // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
+        // be a display name stored in address book rather than the name sent in the From field by
+        // the user.
+        if msg.param.get_bool(Param::WantsMdn).unwrap_or_default() && !msg.is_system_message() {
+            let mdns_enabled = job_try!(context.get_config_bool(Config::MdnsEnabled).await);
+            if mdns_enabled {
+                if let Err(err) = send_mdn(context, &msg).await {
+                    warn!(context, "could not send out mdn for {}: {}", msg.id, err);
+                    return Status::Finished(Err(err));
                 }
-                Status::Finished(Ok(()))
             }
         }
+
+        Status::Finished(Ok(()))
     }
 }
 
 /// Delete all pending jobs with the given action.
-pub async fn kill_action(context: &Context, action: Action) -> bool {
+pub async fn kill_action(context: &Context, action: Action) -> Result<()> {
     context
         .sql
         .execute("DELETE FROM jobs WHERE action=?;", paramsv![action])
-        .await
-        .is_ok()
+        .await?;
+    Ok(())
 }
 
 /// Remove jobs with specified IDs.
 async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
     let q = format!(
         "DELETE FROM jobs WHERE id IN({})",
-        job_ids.iter().map(|_| "?").join(",")
+        job_ids.iter().map(|_| "?").collect::<Vec<&str>>().join(",")
     );
     context
         .sql
@@ -836,15 +700,15 @@ async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
     Ok(())
 }
 
-pub async fn action_exists(context: &Context, action: Action) -> bool {
-    context
+pub async fn action_exists(context: &Context, action: Action) -> Result<bool> {
+    let exists = context
         .sql
         .exists(
             "SELECT COUNT(*) FROM jobs WHERE action=?;",
             paramsv![action],
         )
-        .await
-        .unwrap_or_default()
+        .await?;
+    Ok(exists)
 }
 
 async fn set_delivered(context: &Context, msg_id: MsgId) -> Result<()> {
@@ -876,7 +740,7 @@ async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, fold
                 let display_name_normalized = contact
                     .display_name
                     .as_ref()
-                    .map(normalize_name)
+                    .map(|s| normalize_name(s))
                     .unwrap_or_default();
 
                 match Contact::add_or_lookup(
@@ -978,7 +842,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     }
 
     if rendered_msg.is_gossiped {
-        chat::set_gossiped_timestamp(context, msg.chat_id, time()).await?;
+        msg.chat_id.set_gossiped_timestamp(context, time()).await?;
     }
 
     if 0 != rendered_msg.last_added_location_id {
@@ -995,6 +859,12 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
         }
     }
 
+    if let Some(sync_ids) = rendered_msg.sync_ids_to_delete {
+        if let Err(err) = context.delete_sync_ids(sync_ids).await {
+            error!(context, "Failed to delete sync ids: {:?}", err);
+        }
+    }
+
     if attach_selfavatar {
         if let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, time()).await {
             error!(context, "Failed to set selfavatar timestamp: {:?}", err);
@@ -1028,13 +898,6 @@ pub(crate) enum Connection<'a> {
     Smtp(&'a mut Smtp),
 }
 
-pub(crate) async fn load_imap_deletion_job(context: &Context) -> Result<Option<Job>> {
-    let res = load_imap_deletion_msgid(context)
-        .await?
-        .map(|msg_id| Job::new(Action::DeleteMsgOnImap, msg_id.to_u32(), Params::new(), 0));
-    Ok(res)
-}
-
 impl<'a> fmt::Display for Connection<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
@@ -1078,7 +941,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
                     "{} thread increases job {} tries to {}", &connection, job, tries
                 );
                 job.tries = tries;
-                let time_offset = get_backoff_time_offset(tries);
+                let time_offset = get_backoff_time_offset(tries, job.action);
                 job.desired_timestamp = time() + time_offset;
                 info!(
                     context,
@@ -1143,16 +1006,18 @@ async fn perform_job_action(
         Action::MaybeSendLocationsEnded => {
             location::job_maybe_send_locations_ended(context, job).await
         }
-        Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
-        Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
         Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::Housekeeping => {
             sql::housekeeping(context).await.ok_or_log(context);
             Status::Finished(Ok(()))
         }
-        Action::UpdateRecentQuota => context.update_recent_quota(connection.inbox()).await,
+        Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
+            Ok(status) => status,
+            Err(err) => Status::Finished(Err(err)),
+        },
+        Action::DownloadMsg => job.download_msg(context, connection.inbox()).await,
     };
 
     info!(context, "Finished immediate try {} of job {}", tries, job);
@@ -1160,33 +1025,43 @@ async fn perform_job_action(
     try_res
 }
 
-fn get_backoff_time_offset(tries: u32) -> i64 {
-    let n = 2_i32.pow(tries - 1) * 60;
-    let mut rng = thread_rng();
-    let r: i32 = rng.gen();
-    let mut seconds = r % (n + 1);
-    if seconds < 1 {
-        seconds = 1;
+fn get_backoff_time_offset(tries: u32, action: Action) -> i64 {
+    match action {
+        // Just try every 10s to update the quota
+        // If all retries are exhausted, a new job will be created when the quota information is needed
+        Action::UpdateRecentQuota => 10,
+
+        _ => {
+            // Exponential backoff
+            let n = 2_i32.pow(tries - 1) * 60;
+            let mut rng = thread_rng();
+            let r: i32 = rng.gen();
+            let mut seconds = r % (n + 1);
+            if seconds < 1 {
+                seconds = 1;
+            }
+            seconds as i64
+        }
     }
-    seconds as i64
 }
 
 async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     let mut param = Params::new();
     param.set(Param::MsgId, msg.id.to_u32().to_string());
 
-    add(context, Job::new(Action::SendMdn, msg.from_id, param, 0)).await;
+    add(context, Job::new(Action::SendMdn, msg.from_id, param, 0)).await?;
 
     Ok(())
 }
 
-pub(crate) async fn schedule_resync(context: &Context) {
-    kill_action(context, Action::ResyncFolders).await;
+pub(crate) async fn schedule_resync(context: &Context) -> Result<()> {
+    kill_action(context, Action::ResyncFolders).await?;
     add(
         context,
         Job::new(Action::ResyncFolders, 0, Params::new(), 0),
     )
-    .await;
+    .await?;
+    Ok(())
 }
 
 /// Creates a job.
@@ -1200,56 +1075,44 @@ pub fn create(action: Action, foreign_id: u32, param: Params, delay_seconds: i64
 }
 
 /// Adds a job to the database, scheduling it.
-pub async fn add(context: &Context, job: Job) {
+pub async fn add(context: &Context, job: Job) -> Result<()> {
     let action = job.action;
     let delay_seconds = job.delay_seconds();
-    job.save(context).await.unwrap_or_else(|err| {
-        error!(context, "failed to save job: {}", err);
-    });
+    job.save(context).await.context("failed to save job")?;
 
     if delay_seconds == 0 {
         match action {
             Action::Unknown => unreachable!(),
             Action::Housekeeping
-            | Action::DeleteMsgOnImap
             | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
             | Action::FetchExistingMsgs
-            | Action::MoveMsg
-            | Action::UpdateRecentQuota => {
+            | Action::UpdateRecentQuota
+            | Action::DownloadMsg => {
                 info!(context, "interrupt: imap");
-                context
-                    .interrupt_inbox(InterruptInfo::new(false, None))
-                    .await;
+                context.interrupt_inbox(InterruptInfo::new(false)).await;
             }
             Action::MaybeSendLocations
             | Action::MaybeSendLocationsEnded
             | Action::SendMdn
             | Action::SendMsgToSmtp => {
                 info!(context, "interrupt: smtp");
-                context
-                    .interrupt_smtp(InterruptInfo::new(false, None))
-                    .await;
+                context.interrupt_smtp(InterruptInfo::new(false)).await;
             }
         }
     }
+    Ok(())
 }
 
-async fn load_housekeeping_job(context: &Context) -> Option<Job> {
-    let last_time = match context.get_config_i64(Config::LastHousekeeping).await {
-        Ok(last_time) => last_time,
-        Err(err) => {
-            warn!(context, "failed to load housekeeping config: {:?}", err);
-            return None;
-        }
-    };
+async fn load_housekeeping_job(context: &Context) -> Result<Option<Job>> {
+    let last_time = context.get_config_i64(Config::LastHousekeeping).await?;
 
     let next_time = last_time + (60 * 60 * 24);
     if next_time <= time() {
-        kill_action(context, Action::Housekeeping).await;
-        Some(Job::new(Action::Housekeeping, 0, Params::new(), 0))
+        kill_action(context, Action::Housekeeping).await?;
+        Ok(Some(Job::new(Action::Housekeeping, 0, Params::new(), 0)))
     } else {
-        None
+        Ok(None)
     }
 }
 
@@ -1263,37 +1126,15 @@ pub(crate) async fn load_next(
     context: &Context,
     thread: Thread,
     info: &InterruptInfo,
-) -> Option<Job> {
+) -> Result<Option<Job>> {
     info!(context, "loading job for {}-thread", thread);
 
-    while !context.sql.is_open().await {
-        // The db is closed, which means that this thread should not be running.
-        // Wait until the db is re-opened (if we returned None, this thread might do further damage)
-        warn!(
-            context,
-            "{}: load_next() was called but the db was not opened, THIS SHOULD NOT HAPPEN. Waiting...",
-            thread
-        );
-        sleep(Duration::from_millis(500)).await;
-    }
-
     let query;
     let params;
     let t = time();
-    let m;
     let thread_i = thread as i64;
 
-    if let Some(msg_id) = info.msg_id {
-        query = r#"
-SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
-FROM jobs
-WHERE thread=? AND foreign_id=?
-ORDER BY action DESC, added_timestamp
-LIMIT 1;
-"#;
-        m = msg_id;
-        params = paramsv![thread_i, m];
-    } else if !info.probe_network {
+    if !info.probe_network {
         // processing for first-try and after backoff-timeouts:
         // process jobs in the order they were added.
         query = r#"
@@ -1343,51 +1184,32 @@ LIMIT 1;
                 info!(context, "cleaning up job, because of {}", err);
 
                 // TODO: improve by only doing a single query
-                match context
+                let id = context
                     .sql
                     .query_row(query, params.clone(), |row| row.get::<_, i32>(0))
                     .await
-                {
-                    Ok(id) => {
-                        if let Err(err) = context
-                            .sql
-                            .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
-                            .await
-                        {
-                            warn!(context, "failed to delete job {}: {:?}", id, err);
-                        }
-                    }
-                    Err(err) => {
-                        error!(context, "failed to retrieve invalid job from DB: {}", err);
-                        break None;
-                    }
-                }
+                    .context("Failed to retrieve invalid job ID from the database")?;
+                context
+                    .sql
+                    .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
+                    .await
+                    .with_context(|| format!("Failed to delete invalid job {}", id))?;
             }
         }
     };
 
     match thread {
         Thread::Unknown => {
-            error!(context, "unknown thread for job");
-            None
+            bail!("unknown thread for job")
         }
         Thread::Imap => {
             if let Some(job) = job {
-                if job.action < Action::DeleteMsgOnImap {
-                    load_imap_deletion_job(context)
-                        .await
-                        .unwrap_or_default()
-                        .or(Some(job))
-                } else {
-                    Some(job)
-                }
-            } else if let Some(job) = load_imap_deletion_job(context).await.unwrap_or_default() {
-                Some(job)
+                Ok(Some(job))
             } else {
-                load_housekeeping_job(context).await
+                Ok(load_housekeeping_job(context).await?)
             }
         }
-        Thread::Smtp => job,
+        Thread::Smtp => Ok(job),
     }
 }
 
@@ -1407,8 +1229,12 @@ mod tests {
                  VALUES (?, ?, ?, ?, ?, ?);",
                 paramsv![
                     now,
-                    Thread::from(Action::MoveMsg),
-                    if valid { Action::MoveMsg as i32 } else { -1 },
+                    Thread::from(Action::DownloadMsg),
+                    if valid {
+                        Action::DownloadMsg as i32
+                    } else {
+                        -1
+                    },
                     foreign_id,
                     Params::new().to_string(),
                     now
@@ -1419,7 +1245,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_load_next_job_two() {
+    async fn test_load_next_job_two() -> Result<()> {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
         // all jobs.
@@ -1427,35 +1253,37 @@ mod tests {
         insert_job(&t, 1, false).await; // This can not be loaded into Job struct.
         let jobs = load_next(
             &t,
-            Thread::from(Action::MoveMsg),
-            &InterruptInfo::new(false, None),
+            Thread::from(Action::DownloadMsg),
+            &InterruptInfo::new(false),
         )
-        .await;
+        .await?;
         // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
         assert_eq!(jobs.unwrap().action, Action::Housekeeping);
 
         insert_job(&t, 1, true).await;
         let jobs = load_next(
             &t,
-            Thread::from(Action::MoveMsg),
-            &InterruptInfo::new(false, None),
+            Thread::from(Action::DownloadMsg),
+            &InterruptInfo::new(false),
         )
-        .await;
+        .await?;
         assert!(jobs.is_some());
+        Ok(())
     }
 
     #[async_std::test]
-    async fn test_load_next_job_one() {
+    async fn test_load_next_job_one() -> Result<()> {
         let t = TestContext::new().await;
 
         insert_job(&t, 1, true).await;
 
         let jobs = load_next(
             &t,
-            Thread::from(Action::MoveMsg),
-            &InterruptInfo::new(false, None),
+            Thread::from(Action::DownloadMsg),
+            &InterruptInfo::new(false),
         )
-        .await;
+        .await?;
         assert!(jobs.is_some());
+        Ok(())
     }
 }

src/key.rs

@@ -1,4 +1,4 @@
-//! Cryptographic key module
+//! Cryptographic key module.
 
 use std::collections::BTreeMap;
 use std::fmt;
@@ -510,8 +510,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
-        let t = TestContext::new().await;
-        t.configure_alice().await;
+        let t = TestContext::new_alice().await;
         let pubkey = SignedPublicKey::load_self(&t).await.unwrap();
         assert_eq!(alice.public, pubkey);
         let seckey = SignedSecretKey::load_self(&t).await.unwrap();
@@ -521,7 +520,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_generate_public() {
         let t = TestContext::new().await;
-        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
             .await
             .unwrap();
         let key = SignedPublicKey::load_self(&t).await;
@@ -531,7 +530,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_generate_secret() {
         let t = TestContext::new().await;
-        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
             .await
             .unwrap();
         let key = SignedSecretKey::load_self(&t).await;
@@ -543,7 +542,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         use std::thread;
 
         let t = TestContext::new().await;
-        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
             .await
             .unwrap();
         let thr0 = {
@@ -589,27 +588,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(nrows().await, 1);
     }
 
-    // Convenient way to create a new key if you need one, run with
-    // `cargo test key::tests::gen_key`.
-    // #[test]
-    // fn gen_key() {
-    //     let name = "fiona";
-    //     let keypair = crate::pgp::create_keypair(
-    //         EmailAddress::new(&format!("{}@example.net", name)).unwrap(),
-    //     )
-    //     .unwrap();
-    //     std::fs::write(
-    //         format!("test-data/key/{}-public.asc", name),
-    //         keypair.public.to_base64(),
-    //     )
-    //     .unwrap();
-    //     std::fs::write(
-    //         format!("test-data/key/{}-secret.asc", name),
-    //         keypair.secret.to_base64(),
-    //     )
-    //     .unwrap();
-    // }
-
     #[test]
     fn test_fingerprint_from_str() {
         let res = Fingerprint::new(vec![

src/keyring.rs

@@ -79,8 +79,7 @@ mod tests {
     #[async_std::test]
     async fn test_keyring_load_self() {
         // new_self() implies load_self()
-        let t = TestContext::new().await;
-        t.configure_alice().await;
+        let t = TestContext::new_alice().await;
         let alice = alice_keypair();
 
         let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t).await.unwrap();

src/lib.rs

@@ -1,3 +1,5 @@
+//! # Delta Chat Core Library.
+
 #![forbid(unsafe_code)]
 #![deny(
     clippy::correctness,
@@ -10,7 +12,8 @@
 #![allow(
     clippy::match_bool,
     clippy::eval_order_dependence,
-    clippy::bool_assert_comparison
+    clippy::bool_assert_comparison,
+    clippy::manual_split_once
 )]
 
 #[macro_use]
@@ -53,19 +56,19 @@ mod configure;
 pub mod constants;
 pub mod contact;
 pub mod context;
+pub mod download;
 mod e2ee;
 pub mod ephemeral;
 mod imap;
 pub mod imex;
 mod scheduler;
 #[macro_use]
-pub mod job;
+mod job;
 mod format_flowed;
 pub mod key;
 mod keyring;
 pub mod location;
 mod login_param;
-pub mod lot;
 pub mod message;
 mod mimefactory;
 pub mod mimeparser;
@@ -75,17 +78,21 @@ pub mod peerstate;
 pub mod pgp;
 pub mod provider;
 pub mod qr;
+pub mod qr_code_generator;
 pub mod quota;
 pub mod securejoin;
 mod simplify;
 mod smtp;
 pub mod stock_str;
+mod sync;
 mod token;
+mod update_helper;
 #[macro_use]
 mod dehtml;
 mod color;
 pub mod html;
 pub mod plaintext;
+pub mod summary;
 
 pub mod dc_receive_imf;
 pub mod dc_tools;

src/location.rs

@@ -1,7 +1,7 @@
-//! Location handling
+//! Location handling.
 use std::convert::TryFrom;
 
-use anyhow::{ensure, Error};
+use anyhow::{ensure, Result};
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
@@ -63,7 +63,7 @@ impl Kml {
         Default::default()
     }
 
-    pub fn parse(context: &Context, to_parse: &[u8]) -> Result<Self, Error> {
+    pub fn parse(context: &Context, to_parse: &[u8]) -> Result<Self> {
         ensure!(to_parse.len() <= 1024 * 1024, "kml-file is too large");
 
         let mut reader = quick_xml::Reader::from_reader(to_parse);
@@ -191,89 +191,97 @@ impl Kml {
 }
 
 /// Enables location streaming in chat identified by `chat_id` for `seconds` seconds.
-pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
+pub async fn send_locations_to_chat(
+    context: &Context,
+    chat_id: ChatId,
+    seconds: i64,
+) -> Result<()> {
+    ensure!(seconds >= 0);
+    ensure!(!chat_id.is_special());
     let now = time();
-    if !(seconds < 0 || chat_id.is_special()) {
-        let is_sending_locations_before =
-            is_sending_locations_to_chat(context, Some(chat_id)).await;
-        if context
-            .sql
-            .execute(
-                "UPDATE chats    \
-             SET locations_send_begin=?,        \
-             locations_send_until=?  \
-             WHERE id=?",
-                paramsv![
-                    if 0 != seconds { now } else { 0 },
-                    if 0 != seconds { now + seconds } else { 0 },
-                    chat_id,
-                ],
-            )
+    let is_sending_locations_before = is_sending_locations_to_chat(context, Some(chat_id)).await?;
+    context
+        .sql
+        .execute(
+            "UPDATE chats    \
+         SET locations_send_begin=?,        \
+         locations_send_until=?  \
+         WHERE id=?",
+            paramsv![
+                if 0 != seconds { now } else { 0 },
+                if 0 != seconds { now + seconds } else { 0 },
+                chat_id,
+            ],
+        )
+        .await?;
+    if 0 != seconds && !is_sending_locations_before {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(stock_str::msg_location_enabled(context).await);
+        msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
+        chat::send_msg(context, chat_id, &mut msg)
             .await
-            .is_ok()
-        {
-            if 0 != seconds && !is_sending_locations_before {
-                let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some(stock_str::msg_location_enabled(context).await);
-                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
-                chat::send_msg(context, chat_id, &mut msg)
-                    .await
-                    .unwrap_or_default();
-            } else if 0 == seconds && is_sending_locations_before {
-                let stock_str = stock_str::msg_location_disabled(context).await;
-                chat::add_info_msg(context, chat_id, stock_str, now).await;
-            }
-            context.emit_event(EventType::ChatModified(chat_id));
-            if 0 != seconds {
-                schedule_maybe_send_locations(context, false).await;
-                job::add(
-                    context,
-                    job::Job::new(
-                        job::Action::MaybeSendLocationsEnded,
-                        chat_id.to_u32(),
-                        Params::new(),
-                        seconds + 1,
-                    ),
-                )
-                .await;
-            }
-        }
+            .unwrap_or_default();
+    } else if 0 == seconds && is_sending_locations_before {
+        let stock_str = stock_str::msg_location_disabled(context).await;
+        chat::add_info_msg(context, chat_id, &stock_str, now).await?;
     }
+    context.emit_event(EventType::ChatModified(chat_id));
+    if 0 != seconds {
+        schedule_maybe_send_locations(context, false).await?;
+        job::add(
+            context,
+            job::Job::new(
+                job::Action::MaybeSendLocationsEnded,
+                chat_id.to_u32(),
+                Params::new(),
+                seconds + 1,
+            ),
+        )
+        .await?;
+    }
+    Ok(())
 }
 
-async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) {
-    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await {
+async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) -> Result<()> {
+    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await? {
         job::add(
             context,
             job::Job::new(job::Action::MaybeSendLocations, 0, Params::new(), 60),
         )
-        .await;
+        .await?;
     };
+    Ok(())
 }
 
 /// Returns whether `chat_id` or any chat is sending locations.
 ///
 /// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
 /// is sending locations.
-pub async fn is_sending_locations_to_chat(context: &Context, chat_id: Option<ChatId>) -> bool {
-    match chat_id {
-        Some(chat_id) => context
-            .sql
-            .exists(
-                "SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;",
-                paramsv![chat_id, time()],
-            )
-            .await
-            .unwrap_or_default(),
-        None => context
-            .sql
-            .exists(
-                "SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;",
-                paramsv![time()],
-            )
-            .await
-            .unwrap_or_default(),
-    }
+pub async fn is_sending_locations_to_chat(
+    context: &Context,
+    chat_id: Option<ChatId>,
+) -> Result<bool> {
+    let exists = match chat_id {
+        Some(chat_id) => {
+            context
+                .sql
+                .exists(
+                    "SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;",
+                    paramsv![chat_id, time()],
+                )
+                .await?
+        }
+        None => {
+            context
+                .sql
+                .exists(
+                    "SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;",
+                    paramsv![time()],
+                )
+                .await?
+        }
+    };
+    Ok(exists)
 }
 
 pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
@@ -288,7 +296,11 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
             "SELECT id FROM chats WHERE locations_send_until>?;",
             paramsv![time()],
             |row| row.get::<_, i32>(0),
-            |chats| chats.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+            |chats| {
+                chats
+                    .collect::<std::result::Result<Vec<_>, _>>()
+                    .map_err(Into::into)
+            },
         )
         .await
     {
@@ -313,7 +325,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
         if continue_streaming {
             context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
-        schedule_maybe_send_locations(context, false).await;
+        schedule_maybe_send_locations(context, false).await.ok();
     }
 
     continue_streaming
@@ -325,7 +337,7 @@ pub async fn get_range(
     contact_id: Option<u32>,
     timestamp_from: i64,
     mut timestamp_to: i64,
-) -> Result<Vec<Location>, Error> {
+) -> Result<Vec<Location>> {
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
@@ -400,7 +412,7 @@ fn is_marker(txt: &str) -> bool {
 }
 
 /// Deletes all locations from the database.
-pub async fn delete_all(context: &Context) -> Result<(), Error> {
+pub async fn delete_all(context: &Context) -> Result<()> {
     context
         .sql
         .execute("DELETE FROM locations;", paramsv![])
@@ -409,7 +421,7 @@ pub async fn delete_all(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Error> {
+pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)> {
     let mut last_added_location_id = 0;
 
     let self_addr = context
@@ -517,7 +529,7 @@ pub async fn set_kml_sent_timestamp(
     context: &Context,
     chat_id: ChatId,
     timestamp: i64,
-) -> Result<(), Error> {
+) -> Result<()> {
     context
         .sql
         .execute(
@@ -528,11 +540,7 @@ pub async fn set_kml_sent_timestamp(
     Ok(())
 }
 
-pub async fn set_msg_location_id(
-    context: &Context,
-    msg_id: MsgId,
-    location_id: u32,
-) -> Result<(), Error> {
+pub async fn set_msg_location_id(context: &Context, msg_id: MsgId, location_id: u32) -> Result<()> {
     context
         .sql
         .execute(
@@ -544,17 +552,20 @@ pub async fn set_msg_location_id(
     Ok(())
 }
 
-pub async fn save(
+/// Saves given locations to the database.
+///
+/// Returns the database row ID of the location with the highest timestamp.
+pub(crate) async fn save(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
     locations: &[Location],
     independent: bool,
-) -> Result<u32, Error> {
+) -> Result<Option<u32>> {
     ensure!(!chat_id.is_special(), "Invalid chat id");
 
     let mut newest_timestamp = 0;
-    let mut newest_location_id = 0;
+    let mut newest_location_id = None;
 
     let stmt_insert = "INSERT INTO locations\
              (timestamp, from_id, chat_id, latitude, longitude, accuracy, independent) \
@@ -592,12 +603,12 @@ pub async fn save(
                 drop(stmt_test);
                 drop(stmt_insert);
                 newest_timestamp = timestamp;
-                newest_location_id = conn.last_insert_rowid();
+                newest_location_id = Some(u32::try_from(conn.last_insert_rowid())?);
             }
         }
     }
 
-    Ok(u32::try_from(newest_location_id)?)
+    Ok(newest_location_id)
 }
 
 pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> job::Status {
@@ -630,7 +641,7 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
             },
             |rows| {
                 rows.filter_map(|v| v.transpose())
-                    .collect::<Result<Vec<_>, _>>()
+                    .collect::<std::result::Result<Vec<_>, _>>()
                     .map_err(Into::into)
             },
         )
@@ -690,7 +701,7 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
     }
 
     if continue_streaming {
-        schedule_maybe_send_locations(context, true).await;
+        job_try!(schedule_maybe_send_locations(context, true).await);
     }
     job::Status::Finished(Ok(()))
 }
@@ -736,7 +747,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
             );
 
             let stock_str = stock_str::msg_location_disabled(context).await;
-            chat::add_info_msg(context, chat_id, stock_str, now).await;
+            job_try!(chat::add_info_msg(context, chat_id, &stock_str, now).await);
             context.emit_event(EventType::ChatModified(chat_id));
         }
     }

src/log.rs

@@ -1,5 +1,7 @@
-//! # Logging
+//! # Logging.
+
 use crate::context::Context;
+use async_std::task::block_on;
 
 #[macro_export]
 macro_rules! info {
@@ -12,7 +14,7 @@ macro_rules! info {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::EventType::Info(full));
+        $ctx.emit_event($crate::EventType::Info(full));
     }};
 }
 
@@ -27,7 +29,7 @@ macro_rules! warn {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::EventType::Warning(full));
+        $ctx.emit_event($crate::EventType::Warning(full));
     }};
 }
 
@@ -38,15 +40,26 @@ macro_rules! error {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        emit_event!($ctx, $crate::EventType::Error(formatted));
+        $ctx.set_last_error(&formatted);
+        $ctx.emit_event($crate::EventType::Error(formatted));
     }};
 }
 
-#[macro_export]
-macro_rules! emit_event {
-    ($ctx:expr, $event:expr) => {
-        $ctx.emit_event($event);
-    };
+impl Context {
+    /// Set last error string.
+    /// Implemented as blocking as used from macros in different, not always async blocks.
+    pub fn set_last_error(&self, error: &str) {
+        block_on(async move {
+            let mut last_error = self.last_error.write().await;
+            *last_error = error.to_string();
+        });
+    }
+
+    /// Get last error string.
+    pub async fn get_last_error(&self) -> String {
+        let last_error = &*self.last_error.read().await;
+        last_error.clone()
+    }
 }
 
 pub trait LogExt<T, E>
@@ -135,8 +148,36 @@ impl<T, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
             );
             // We can't use the warn!() macro here as the file!() and line!() macros
             // don't work with #[track_caller]
-            emit_event!(context, crate::EventType::Warning(full));
+            context.emit_event(crate::EventType::Warning(full));
         };
         self
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::test_utils::TestContext;
+    use anyhow::Result;
+
+    #[async_std::test]
+    async fn test_get_last_error() -> Result<()> {
+        let t = TestContext::new().await;
+
+        assert_eq!(t.get_last_error().await, "");
+
+        error!(t, "foo-error");
+        assert_eq!(t.get_last_error().await, "foo-error");
+
+        warn!(t, "foo-warning");
+        assert_eq!(t.get_last_error().await, "foo-error");
+
+        info!(t, "foo-info");
+        assert_eq!(t.get_last_error().await, "foo-error");
+
+        error!(t, "bar-error");
+        error!(t, "baz-error");
+        assert_eq!(t.get_last_error().await, "baz-error");
+
+        Ok(())
+    }
+}

src/login_param.rs

@@ -1,12 +1,21 @@
-//! # Login parameters
+//! # Login parameters.
 
 use std::borrow::Cow;
 use std::fmt;
+use std::time::Duration;
 
 use crate::provider::{get_provider_by_id, Provider};
 use crate::{context::Context, provider::Socket};
 use anyhow::Result;
 
+use async_std::io;
+use async_std::net::TcpStream;
+
+use async_native_tls::Certificate;
+pub use async_smtp::ServerAddress;
+use fast_socks5::client::Socks5Stream;
+use once_cell::sync::Lazy;
+
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(u32)]
 #[strum(serialize_all = "snake_case")]
@@ -44,6 +53,81 @@ pub struct ServerLoginParam {
     pub certificate_checks: CertificateChecks,
 }
 
+#[derive(Default, Debug, Clone, PartialEq)]
+pub struct Socks5Config {
+    pub host: String,
+    pub port: u16,
+    pub user_password: Option<(String, String)>,
+}
+
+impl Socks5Config {
+    /// Reads SOCKS5 configuration from the database.
+    pub async fn from_database(context: &Context) -> Result<Option<Self>> {
+        let sql = &context.sql;
+
+        let enabled = sql.get_raw_config_bool("socks5_enabled").await?;
+        if enabled {
+            let host = sql.get_raw_config("socks5_host").await?.unwrap_or_default();
+            let port: u16 = sql
+                .get_raw_config_int("socks5_port")
+                .await?
+                .unwrap_or_default() as u16;
+            let user = sql.get_raw_config("socks5_user").await?.unwrap_or_default();
+            let password = sql
+                .get_raw_config("socks5_password")
+                .await?
+                .unwrap_or_default();
+
+            let socks5_config = Self {
+                host,
+                port,
+                user_password: if !user.is_empty() {
+                    Some((user, password))
+                } else {
+                    None
+                },
+            };
+            Ok(Some(socks5_config))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub async fn connect(
+        &self,
+        target_addr: &ServerAddress,
+        timeout: Option<Duration>,
+    ) -> io::Result<Socks5Stream<TcpStream>> {
+        self.to_async_smtp_socks5_config()
+            .connect(target_addr, timeout)
+            .await
+    }
+
+    pub fn to_async_smtp_socks5_config(&self) -> async_smtp::smtp::Socks5Config {
+        async_smtp::smtp::Socks5Config {
+            host: self.host.clone(),
+            port: self.port,
+            user_password: self.user_password.clone(),
+        }
+    }
+}
+
+impl fmt::Display for Socks5Config {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "host:{},port:{},user_password:{}",
+            self.host,
+            self.port,
+            if let Some(user_password) = self.user_password.clone() {
+                format!("user: {}, password: ***", user_password.0)
+            } else {
+                "user: None".to_string()
+            }
+        )
+    }
+}
+
 #[derive(Default, Debug, Clone, PartialEq)]
 pub struct LoginParam {
     pub addr: String,
@@ -51,6 +135,7 @@ pub struct LoginParam {
     pub smtp: ServerLoginParam,
     pub server_flags: i32,
     pub provider: Option<&'static Provider>,
+    pub socks5_config: Option<Socks5Config>,
 }
 
 impl LoginParam {
@@ -130,6 +215,8 @@ impl LoginParam {
             .await?
             .and_then(|provider_id| get_provider_by_id(&provider_id));
 
+        let socks5_config = Socks5Config::from_database(context).await?;
+
         Ok(LoginParam {
             addr,
             imap: ServerLoginParam {
@@ -150,6 +237,7 @@ impl LoginParam {
             },
             provider,
             server_flags,
+            socks5_config,
         })
     }
 
@@ -282,8 +370,18 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
+// this certificate is missing on older android devices (eg. lg with android6 from 2017)
+// certificate downloaded from https://letsencrypt.org/certificates/
+static LETSENCRYPT_ROOT: Lazy<Certificate> = Lazy::new(|| {
+    Certificate::from_der(include_bytes!(
+        "../assets/root-certificates/letsencrypt/isrgrootx1.der"
+    ))
+    .unwrap()
+});
+
 pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
-    let tls_builder = async_native_tls::TlsConnector::new();
+    let tls_builder =
+        async_native_tls::TlsConnector::new().add_root_certificate(LETSENCRYPT_ROOT.clone());
 
     if strict_tls {
         tls_builder
@@ -315,7 +413,7 @@ mod tests {
         let t = TestContext::new().await;
 
         let param = LoginParam {
-            addr: "alice@example.com".to_string(),
+            addr: "alice@example.org".to_string(),
             imap: ServerLoginParam {
                 server: "imap.example.com".to_string(),
                 user: "alice".to_string(),
@@ -326,7 +424,7 @@ mod tests {
             },
             smtp: ServerLoginParam {
                 server: "smtp.example.com".to_string(),
-                user: "alice@example.com".to_string(),
+                user: "alice@example.org".to_string(),
                 password: "bar".to_string(),
                 port: 456,
                 security: Socket::Ssl,
@@ -334,6 +432,8 @@ mod tests {
             },
             server_flags: 0,
             provider: get_provider_by_id("example.com"),
+            // socks5_config is not saved by `save_to_database`, using default value
+            socks5_config: None,
         };
 
         param.save_to_database(&t, "foobar_").await?;
@@ -342,4 +442,13 @@ mod tests {
         assert_eq!(param, loaded);
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_build_tls() -> Result<()> {
+        // we are using some additional root certificates.
+        // make sure, they do not break construction of TlsConnector
+        let _ = dc_build_tls(true);
+        let _ = dc_build_tls(false);
+        Ok(())
+    }
 }

src/lot.rs

@@ -1,139 +0,0 @@
-use deltachat_derive::{FromSql, ToSql};
-
-use crate::key::Fingerprint;
-
-/// An object containing a set of values.
-/// The meaning of the values is defined by the function returning the object.
-/// Lot objects are created
-/// eg. by chatlist.get_summary() or dc_msg_get_summary().
-///
-/// *Lot* is used in the meaning *heap* here.
-#[derive(Default, Debug, Clone)]
-pub struct Lot {
-    pub(crate) text1_meaning: Meaning,
-    pub(crate) text1: Option<String>,
-    pub(crate) text2: Option<String>,
-    pub(crate) timestamp: i64,
-    pub(crate) state: LotState,
-    pub(crate) id: u32,
-    pub(crate) fingerprint: Option<Fingerprint>,
-    pub(crate) invitenumber: Option<String>,
-    pub(crate) auth: Option<String>,
-}
-
-#[repr(u8)]
-#[derive(
-    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
-)]
-pub enum Meaning {
-    None = 0,
-    Text1Draft = 1,
-    Text1Username = 2,
-    Text1Self = 3,
-}
-
-impl Default for Meaning {
-    fn default() -> Self {
-        Meaning::None
-    }
-}
-
-impl Lot {
-    pub fn new() -> Self {
-        Default::default()
-    }
-
-    pub fn get_text1(&self) -> Option<&str> {
-        self.text1.as_deref()
-    }
-
-    pub fn get_text2(&self) -> Option<&str> {
-        self.text2.as_deref()
-    }
-
-    pub fn get_text1_meaning(&self) -> Meaning {
-        self.text1_meaning
-    }
-
-    pub fn get_state(&self) -> LotState {
-        self.state
-    }
-
-    pub fn get_id(&self) -> u32 {
-        self.id
-    }
-
-    pub fn get_timestamp(&self) -> i64 {
-        self.timestamp
-    }
-}
-
-#[repr(u32)]
-#[derive(
-    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
-)]
-pub enum LotState {
-    // Default
-    Undefined = 0,
-
-    // Qr States
-    /// id=contact
-    QrAskVerifyContact = 200,
-
-    /// text1=groupname
-    QrAskVerifyGroup = 202,
-
-    /// id=contact
-    QrFprOk = 210,
-
-    /// id=contact
-    QrFprMismatch = 220,
-
-    /// test1=formatted fingerprint
-    QrFprWithoutAddr = 230,
-
-    /// text1=domain
-    QrAccount = 250,
-
-    /// text1=domain, text2=instance pattern
-    QrWebrtcInstance = 260,
-
-    /// id=contact
-    QrAddr = 320,
-
-    /// text1=text
-    QrText = 330,
-
-    /// text1=URL
-    QrUrl = 332,
-
-    /// text1=error string
-    QrError = 400,
-
-    QrWithdrawVerifyContact = 500,
-
-    /// text1=groupname
-    QrWithdrawVerifyGroup = 502,
-
-    QrReviveVerifyContact = 510,
-
-    /// text1=groupname
-    QrReviveVerifyGroup = 512,
-
-    // Message States
-    MsgInFresh = 10,
-    MsgInNoticed = 13,
-    MsgInSeen = 16,
-    MsgOutPreparing = 18,
-    MsgOutDraft = 19,
-    MsgOutPending = 20,
-    MsgOutFailed = 24,
-    MsgOutDelivered = 26,
-    MsgOutMdnRcvd = 28,
-}
-
-impl Default for LotState {
-    fn default() -> Self {
-        LotState::Undefined
-    }
-}

src/mimefactory.rs

@@ -1,6 +1,8 @@
+//! # MIME message production.
+
 use std::convert::TryInto;
 
-use anyhow::{bail, ensure, format_err, Result};
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
 
@@ -66,6 +68,13 @@ pub struct MimeFactory<'a> {
     references: String,
     req_mdn: bool,
     last_added_location_id: u32,
+
+    /// If the created mime-structure contains sync-items,
+    /// the IDs of these items are listed here.
+    /// The IDs are returned via `RenderedEmail`
+    /// and must be deleted if the message is actually queued for sending.
+    sync_ids_to_delete: Option<String>,
+
     attach_selfavatar: bool,
 }
 
@@ -78,6 +87,12 @@ pub struct RenderedEmail {
     pub is_gossiped: bool,
     pub last_added_location_id: u32,
 
+    /// A comma-separated string of sync-IDs that are used by the rendered email
+    /// and must be deleted once the message is actually queued for sending
+    /// (deletion must be done by `delete_sync_ids()`).
+    /// If the rendered email is not queued for sending, the IDs must not be deleted.
+    pub sync_ids_to_delete: Option<String>,
+
     /// Message ID (Message in the sense of Email)
     pub rfc724_mid: String,
     pub subject: String,
@@ -139,6 +154,12 @@ impl<'a> MimeFactory<'a> {
 
         if chat.is_self_talk() {
             recipients.push((from_displayname.to_string(), from_addr.to_string()));
+        } else if chat.is_mailing_list() {
+            let list_post = chat
+                .param
+                .get(Param::ListPost)
+                .context("Can't write to mailinglist without ListPost param")?;
+            recipients.push(("".to_string(), list_post.to_string()));
         } else {
             context
                 .sql
@@ -203,6 +224,7 @@ impl<'a> MimeFactory<'a> {
             references,
             req_mdn,
             last_added_location_id: 0,
+            sync_ids_to_delete: None,
             attach_selfavatar,
         };
         Ok(factory)
@@ -247,6 +269,7 @@ impl<'a> MimeFactory<'a> {
             references: String::default(),
             req_mdn: false,
             last_added_location_id: 0,
+            sync_ids_to_delete: None,
             attach_selfavatar: false,
         };
 
@@ -314,6 +337,10 @@ impl<'a> MimeFactory<'a> {
             Loaded::Message { chat } => {
                 if chat.is_protected() {
                     false
+                } else if chat.typ == Chattype::Broadcast {
+                    // encryption may disclose recipients;
+                    // this is probably a worse issue than not opportunistically (!) encrypting
+                    true
                 } else {
                     self.msg
                         .param
@@ -340,7 +367,7 @@ impl<'a> MimeFactory<'a> {
         match &self.loaded {
             Loaded::Message { chat } => {
                 // beside key- and member-changes, force re-gossip every 48 hours
-                let gossiped_timestamp = chat.get_gossiped_timestamp(context).await?;
+                let gossiped_timestamp = chat.id.get_gossiped_timestamp(context).await?;
                 if time() > gossiped_timestamp + (2 * 24 * 60 * 60) {
                     Ok(true)
                 } else {
@@ -386,17 +413,11 @@ impl<'a> MimeFactory<'a> {
 
         let subject = match self.loaded {
             Loaded::Message { ref chat } => {
-                if self.msg.param.get_cmd() == SystemMessage::AutocryptSetupMessage {
-                    return Ok(stock_str::ac_setup_msg_subject(context).await);
-                }
-
                 if !self.msg.subject.is_empty() {
                     return Ok(self.msg.subject.clone());
                 }
 
                 if chat.typ == Chattype::Group && quoted_msg_subject.is_none_or_empty() {
-                    // If we have a `quoted_msg_subject`, we use the subject of the quoted message
-                    // instead of the group name
                     let re = if self.in_reply_to.is_empty() {
                         ""
                     } else {
@@ -405,22 +426,22 @@ impl<'a> MimeFactory<'a> {
                     return Ok(format!("{}{}", re, chat.name));
                 }
 
-                let parent_subject = if quoted_msg_subject.is_none_or_empty() {
-                    chat.param.get(Param::LastSubject)
-                } else {
-                    quoted_msg_subject.as_deref()
-                };
-
-                if let Some(last_subject) = parent_subject {
-                    format!("Re: {}", remove_subject_prefix(last_subject))
-                } else {
-                    let self_name = match context.get_config(Config::Displayname).await? {
-                        Some(name) => name,
-                        None => context.get_config(Config::Addr).await?.unwrap_or_default(),
+                if chat.typ != Chattype::Broadcast {
+                    let parent_subject = if quoted_msg_subject.is_none_or_empty() {
+                        chat.param.get(Param::LastSubject)
+                    } else {
+                        quoted_msg_subject.as_deref()
                     };
-
-                    stock_str::subject_for_new_contact(context, self_name).await
+                    if let Some(last_subject) = parent_subject {
+                        return Ok(format!("Re: {}", remove_subject_prefix(last_subject)));
+                    }
                 }
+
+                let self_name = match context.get_config(Config::Displayname).await? {
+                    Some(name) => name,
+                    None => context.get_config(Config::Addr).await?.unwrap_or_default(),
+                };
+                stock_str::subject_for_new_contact(context, self_name).await
             }
             Loaded::Mdn { .. } => stock_str::read_rcpt(context).await,
         };
@@ -443,20 +464,45 @@ impl<'a> MimeFactory<'a> {
             self.from_addr.clone(),
         );
 
-        let mut to = Vec::new();
-        for (name, addr) in self.recipients.iter() {
-            if name.is_empty() {
-                to.push(Address::new_mailbox(addr.clone()));
-            } else {
-                to.push(Address::new_mailbox_with_name(
-                    name.to_string(),
-                    addr.clone(),
-                ));
-            }
-        }
+        let undisclosed_recipients = match &self.loaded {
+            Loaded::Message { chat } => chat.typ == Chattype::Broadcast,
+            Loaded::Mdn { .. } => false,
+        };
 
-        if to.is_empty() {
-            to.push(from.clone());
+        let mut to = Vec::new();
+        if undisclosed_recipients {
+            to.push(Address::new_group(
+                "hidden-recipients".to_string(),
+                Vec::new(),
+            ));
+        } else {
+            let email_to_remove =
+                if self.msg.param.get_cmd() == SystemMessage::MemberRemovedFromGroup {
+                    self.msg.param.get(Param::Arg)
+                } else {
+                    None
+                };
+
+            for (name, addr) in self.recipients.iter() {
+                if let Some(email_to_remove) = email_to_remove {
+                    if email_to_remove == addr {
+                        continue;
+                    }
+                }
+
+                if name.is_empty() {
+                    to.push(Address::new_mailbox(addr.clone()));
+                } else {
+                    to.push(Address::new_mailbox_with_name(
+                        name.to_string(),
+                        addr.clone(),
+                    ));
+                }
+            }
+
+            if to.is_empty() {
+                to.push(from.clone());
+            }
         }
 
         headers
@@ -560,6 +606,7 @@ impl<'a> MimeFactory<'a> {
         headers
             .unprotected
             .push(Header::new_with_value("To".into(), to).unwrap());
+
         headers
             .unprotected
             .push(Header::new_with_value("From".into(), vec![from]).unwrap());
@@ -591,12 +638,20 @@ impl<'a> MimeFactory<'a> {
             main_part
         } else {
             // Multiple parts, render as multipart.
-            parts.into_iter().fold(
-                PartBuilder::new()
-                    .message_type(MimeMultipartType::Mixed)
-                    .child(main_part.build()),
-                |message, part| message.child(part.build()),
-            )
+            let part_holder = if self.msg.param.get_cmd() == SystemMessage::MultiDeviceSync {
+                PartBuilder::new().header((
+                    "Content-Type".to_string(),
+                    "multipart/report; report-type=multi-device-sync".to_string(),
+                ))
+            } else {
+                PartBuilder::new().message_type(MimeMultipartType::Mixed)
+            };
+
+            parts
+                .into_iter()
+                .fold(part_holder.child(main_part.build()), |message, part| {
+                    message.child(part.build())
+                })
         };
 
         let outer_message = if is_encrypted {
@@ -717,6 +772,7 @@ impl<'a> MimeFactory<'a> {
             is_encrypted,
             is_gossiped,
             last_added_location_id,
+            sync_ids_to_delete: self.sync_ids_to_delete,
             rfc724_mid,
             subject: subject_str,
         })
@@ -861,7 +917,7 @@ impl<'a> MimeFactory<'a> {
                     "ephemeral-timer-changed".to_string(),
                 ));
             }
-            SystemMessage::LocationOnly => {
+            SystemMessage::LocationOnly | SystemMessage::MultiDeviceSync => {
                 // This should prevent automatic replies,
                 // such as non-delivery reports.
                 //
@@ -1082,7 +1138,7 @@ impl<'a> MimeFactory<'a> {
             parts.push(msg_kml_part);
         }
 
-        if location::is_sending_locations_to_chat(context, Some(self.msg.chat_id)).await {
+        if location::is_sending_locations_to_chat(context, Some(self.msg.chat_id)).await? {
             match self.get_location_kml_part(context).await {
                 Ok(part) => parts.push(part),
                 Err(err) => {
@@ -1091,6 +1147,15 @@ impl<'a> MimeFactory<'a> {
             }
         }
 
+        // we do not piggyback sync-files to other self-sent-messages
+        // to not risk files becoming too larger and being skipped by download-on-demand.
+        if command == SystemMessage::MultiDeviceSync && self.is_e2ee_guaranteed() {
+            let json = self.msg.param.get(Param::Arg).unwrap_or_default();
+            let ids = self.msg.param.get(Param::Arg2).unwrap_or_default();
+            parts.push(context.build_sync_part(json.to_string()).await);
+            self.sync_ids_to_delete = Some(ids.to_string());
+        }
+
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar).await? {
                 Some(path) => match build_selfavatar_file(context, &path) {
@@ -1145,7 +1210,11 @@ impl<'a> MimeFactory<'a> {
         {
             stock_str::encrypted_msg(context).await
         } else {
-            self.msg.get_summarytext(context, 32).await
+            self.msg
+                .get_summary(context, None)
+                .await?
+                .truncated_text(32)
+                .to_string()
         };
         let p2 = stock_str::read_rcpt_mail_body(context, p1).await;
         let message_text = format!("{}\r\n", format_flowed(&p2));
@@ -1356,6 +1425,10 @@ mod tests {
     use async_std::prelude::*;
 
     use crate::chat::ChatId;
+    use crate::chat::{
+        add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg,
+        ProtectionStatus,
+    };
     use crate::chatlist::Chatlist;
     use crate::contact::Origin;
     use crate::dc_receive_imf::dc_receive_imf;
@@ -1363,7 +1436,7 @@ mod tests {
     use crate::test_utils::{get_chat_msg, TestContext};
 
     use async_std::fs::File;
-    use pretty_assertions::assert_eq;
+    use mailparse::{addrparse_header, MailHeaderMap};
 
     #[test]
     fn test_render_email_address() {
@@ -1461,7 +1534,7 @@ mod tests {
             msg_to_subject_str(
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: Bob <bob@example.com>\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Antw: Chat: hello\n\
                 Message-ID: <2222@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
@@ -1476,7 +1549,7 @@ mod tests {
             msg_to_subject_str(
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: Bob <bob@example.com>\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Infos: 42\n\
                 Message-ID: <2222@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
@@ -1495,7 +1568,7 @@ mod tests {
             msg_to_subject_str(
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: bob@example.com\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
                 Message-ID: <2223@example.com>\n\
@@ -1513,7 +1586,7 @@ mod tests {
         // 3. Send the first message to a new contact
         let t = TestContext::new_alice().await;
 
-        assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.org");
 
         let t = TestContext::new_alice().await;
         t.set_config(Config::Displayname, Some("Alice"))
@@ -1528,7 +1601,7 @@ mod tests {
         msg_to_subject_str(
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
             From: bob@example.com\n\
-            To: alice@example.com\n\
+            To: alice@example.org\n\
             Subject: äääää\n\
             Chat-Version: 1.0\n\
             Message-ID: <2893@example.com>\n\
@@ -1542,7 +1615,7 @@ mod tests {
         msg_to_subject_str(
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
             From: bob@example.com\n\
-            To: alice@example.com\n\
+            To: alice@example.org\n\
             Subject: aäääää\n\
             Chat-Version: 1.0\n\
             Message-ID: <2893@example.com>\n\
@@ -1561,7 +1634,7 @@ mod tests {
         dc_receive_imf(
             &t,
             b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-            From: alice@example.com\n\
+            From: alice@example.org\n\
             To: bob@example.com\n\
             Subject: Hello, Bob\n\
             Chat-Version: 1.0\n\
@@ -1570,7 +1643,6 @@ mod tests {
             \n\
             hello\n",
             "INBOX",
-            1,
             false,
         )
         .await
@@ -1578,7 +1650,7 @@ mod tests {
         let new_msg = incoming_msg_to_reply_msg(
             b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                  From: bob@example.com\n\
-                 To: alice@example.com\n\
+                 To: alice@example.org\n\
                  Subject: message opened\n\
                  Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
                  Chat-Version: 1.0\n\
@@ -1607,27 +1679,30 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_subject_in_group() {
+    async fn test_subject_in_group() -> Result<()> {
         async fn send_msg_get_subject(
             t: &TestContext,
             group_id: ChatId,
             quote: Option<&Message>,
-        ) -> String {
+        ) -> Result<String> {
             let mut new_msg = Message::new(Viewtype::Text);
             new_msg.set_text(Some("Hi".to_string()));
             if let Some(q) = quote {
-                new_msg.set_quote(t, q).await.unwrap();
+                new_msg.set_quote(t, q).await?;
             }
             let sent = t.send_msg(group_id, &mut new_msg).await;
             get_subject(t, sent).await
         }
-        async fn get_subject(t: &TestContext, sent: crate::test_utils::SentMessage) -> String {
+        async fn get_subject(
+            t: &TestContext,
+            sent: crate::test_utils::SentMessage,
+        ) -> Result<String> {
             let parsed_subject = t.parse_msg(&sent).await.get_subject().unwrap();
 
-            let sent_msg = Message::load_from_db(t, sent.sender_msg_id).await.unwrap();
+            let sent_msg = Message::load_from_db(t, sent.sender_msg_id).await?;
             assert_eq!(parsed_subject, sent_msg.subject);
 
-            parsed_subject
+            Ok(parsed_subject)
         }
 
         // 6. Test that in a group, replies also take the quoted message's subject, while non-replies use the group title as subject
@@ -1636,13 +1711,13 @@ mod tests {
             chat::create_group_chat(&t, chat::ProtectionStatus::Unprotected, "groupname") // TODO encodings, ä
                 .await
                 .unwrap();
-        let bob = Contact::create(&t, "", "bob@example.org").await.unwrap();
-        chat::add_contact_to_chat(&t, group_id, bob).await;
+        let bob = Contact::create(&t, "", "bob@example.org").await?;
+        chat::add_contact_to_chat(&t, group_id, bob).await?;
 
-        let subject = send_msg_get_subject(&t, group_id, None).await;
+        let subject = send_msg_get_subject(&t, group_id, None).await?;
         assert_eq!(subject, "groupname");
 
-        let subject = send_msg_get_subject(&t, group_id, None).await;
+        let subject = send_msg_get_subject(&t, group_id, None).await?;
         assert_eq!(subject, "Re: groupname");
 
         dc_receive_imf(
@@ -1650,7 +1725,7 @@ mod tests {
             format!(
                 "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: bob@example.com\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Different subject\n\
                 In-Reply-To: {}\n\
                 Message-ID: <2893@example.com>\n\
@@ -1661,31 +1736,28 @@ mod tests {
             )
             .as_bytes(),
             "INBOX",
-            5,
             false,
         )
-        .await
-        .unwrap();
+        .await?;
         let message_from_bob = t.get_last_msg().await;
 
-        let subject = send_msg_get_subject(&t, group_id, None).await;
+        let subject = send_msg_get_subject(&t, group_id, None).await?;
         assert_eq!(subject, "Re: groupname");
 
-        let subject = send_msg_get_subject(&t, group_id, Some(&message_from_bob)).await;
+        let subject = send_msg_get_subject(&t, group_id, Some(&message_from_bob)).await?;
         let outgoing_quoting_msg = t.get_last_msg().await;
         assert_eq!(subject, "Re: Different subject");
 
-        let subject = send_msg_get_subject(&t, group_id, None).await;
+        let subject = send_msg_get_subject(&t, group_id, None).await?;
         assert_eq!(subject, "Re: groupname");
 
-        let subject = send_msg_get_subject(&t, group_id, Some(&outgoing_quoting_msg)).await;
+        let subject = send_msg_get_subject(&t, group_id, Some(&outgoing_quoting_msg)).await?;
         assert_eq!(subject, "Re: Different subject");
 
-        chat::forward_msgs(&t, &[message_from_bob.id], group_id)
-            .await
-            .unwrap();
-        let subject = get_subject(&t, t.pop_sent_msg().await).await;
-        assert_eq!(subject, "Fwd: Different subject");
+        chat::forward_msgs(&t, &[message_from_bob.id], group_id).await?;
+        let subject = get_subject(&t, t.pop_sent_msg().await).await?;
+        assert_eq!(subject, "Re: groupname");
+        Ok(())
     }
 
     async fn first_subject_str(t: TestContext) -> String {
@@ -1707,7 +1779,7 @@ mod tests {
         mf.subject_str(&t).await.unwrap()
     }
 
-    // In `imf_raw`, From has to be bob@example.com, To has to be alice@example.com
+    // In `imf_raw`, From has to be bob@example.com, To has to be alice@example.org
     async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
         let subject_str = msg_to_subject_str_inner(imf_raw, false, false, false).await;
 
@@ -1768,14 +1840,13 @@ mod tests {
                 &t,
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                     From: Bob <bob@example.com>\n\
-                    To: alice@example.com\n\
+                    To: alice@example.org\n\
                     Subject: Some other, completely unrelated subject\n\
                     Message-ID: <3cl4@example.com>\n\
                     Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                     \n\
                     Some other, completely unrelated content\n",
                 "INBOX",
-                2,
                 false,
             )
             .await
@@ -1800,7 +1871,7 @@ mod tests {
             .await
             .unwrap();
 
-        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
+        dc_receive_imf(context, imf_raw, "INBOX", false)
             .await
             .unwrap();
 
@@ -1828,7 +1899,7 @@ mod tests {
         let msg = incoming_msg_to_reply_msg(
             b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: Charlie <charlie@example.com>\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
                 Message-ID: <2223@example.com>\n\
@@ -1967,4 +2038,35 @@ mod tests {
 
         Ok(())
     }
+
+    /// Test that removed member address does not go into the `To:` field.
+    #[async_std::test]
+    async fn test_remove_member_bcc() -> Result<()> {
+        // Alice creates a group with Bob and Claire and then removes Bob.
+        let alice = TestContext::new_alice().await;
+
+        let bob_id = Contact::create(&alice, "Bob", "bob@example.net").await?;
+        let claire_id = Contact::create(&alice, "Claire", "claire@foo.de").await?;
+
+        let alice_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "foo").await?;
+        add_contact_to_chat(&alice, alice_chat_id, bob_id).await?;
+        add_contact_to_chat(&alice, alice_chat_id, claire_id).await?;
+        send_text_msg(&alice, alice_chat_id, "Creating a group".to_string()).await?;
+
+        remove_contact_from_chat(&alice, alice_chat_id, claire_id).await?;
+        let remove = alice.pop_sent_msg().await;
+        let remove_payload = remove.payload();
+        let parsed = mailparse::parse_mail(remove_payload.as_bytes())?;
+        let to = parsed
+            .headers
+            .get_first_header("To")
+            .ok_or_else(|| format_err!("No To: header parsed"))?;
+        let to = addrparse_header(to)?;
+        let mailbox = to
+            .extract_single_info()
+            .ok_or_else(|| format_err!("To: field does not contain exactly one address"))?;
+        assert_eq!(mailbox.addr, "bob@example.net");
+
+        Ok(())
+    }
 }

src/mimeparser.rs

@@ -1,3 +1,5 @@
+//! # MIME message parsing module.
+
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
 use std::pin::Pin;
@@ -10,10 +12,10 @@ use once_cell::sync::Lazy;
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
-use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSE};
+use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
 use crate::contact::addr_normalize;
 use crate::context::Context;
-use crate::dc_tools::{dc_get_filemeta, dc_truncate};
+use crate::dc_tools::{dc_get_filemeta, dc_truncate, parse_receive_headers};
 use crate::dehtml::dehtml;
 use crate::e2ee;
 use crate::events::EventType;
@@ -26,6 +28,7 @@ use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::simplify::simplify;
 use crate::stock_str;
+use crate::sync::SyncItems;
 
 /// A parsed MIME message.
 ///
@@ -44,6 +47,7 @@ pub struct MimeMessage {
     /// Addresses are normalized and lowercased:
     pub recipients: Vec<SingleInfo>,
     pub from: Vec<SingleInfo>,
+    pub list_post: Option<String>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
 
@@ -54,11 +58,14 @@ pub struct MimeMessage {
     /// this set is empty.
     pub signatures: HashSet<Fingerprint>,
 
-    pub gossipped_addr: HashSet<String>,
+    /// The set of mail recipient addresses for which gossip headers were applied, regardless of
+    /// whether they modified any peerstates.
+    pub gossiped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
     pub location_kml: Option<location::Kml>,
     pub message_kml: Option<location::Kml>,
+    pub(crate) sync_items: Option<SyncItems>,
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
     pub(crate) mdn_reports: Vec<Report>,
@@ -76,6 +83,8 @@ pub struct MimeMessage {
     /// This is non-empty only if the message was actually encrypted.  It is used
     /// for e.g. late-parsing HTML.
     pub decoded_data: Vec<u8>,
+
+    pub(crate) hop_info: String,
 }
 
 #[derive(Debug, PartialEq)]
@@ -122,6 +131,10 @@ pub enum SystemMessage {
     // Chat protection state changed
     ChatProtectionEnabled = 11,
     ChatProtectionDisabled = 12,
+
+    /// Self-sent-message that contains only json used for multi-device-sync;
+    /// if possible, we attach that to other messages as for locations.
+    MultiDeviceSync = 20,
 }
 
 impl Default for SystemMessage {
@@ -134,6 +147,18 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
 
 impl MimeMessage {
     pub async fn from_bytes(context: &Context, body: &[u8]) -> Result<Self> {
+        MimeMessage::from_bytes_with_partial(context, body, None).await
+    }
+
+    /// Parse a mime message.
+    ///
+    /// If `partial` is set, it contains the full message size in bytes
+    /// and `body` contains the header only.
+    pub async fn from_bytes_with_partial(
+        context: &Context,
+        body: &[u8],
+        partial: Option<u32>,
+    ) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
 
         let message_time = mail
@@ -141,10 +166,12 @@ impl MimeMessage {
             .get_header_value(HeaderDef::Date)
             .and_then(|v| mailparse::dateparse(&v).ok())
             .unwrap_or_default();
+        let hop_info = parse_receive_headers(&mail.get_headers());
 
         let mut headers = Default::default();
         let mut recipients = Default::default();
         let mut from = Default::default();
+        let mut list_post = Default::default();
         let mut chat_disposition_notification_to = None;
 
         // Parse IMF headers.
@@ -153,6 +180,7 @@ impl MimeMessage {
             &mut headers,
             &mut recipients,
             &mut from,
+            &mut list_post,
             &mut chat_disposition_notification_to,
             &mail.headers,
         );
@@ -178,7 +206,7 @@ impl MimeMessage {
 
         // Memory location for a possible decrypted message.
         let mut mail_raw = Vec::new();
-        let mut gossipped_addr = Default::default();
+        let mut gossiped_addr = Default::default();
 
         let (mail, signatures, warn_empty_signature) =
             match e2ee::try_decrypt(context, &mail, message_time).await {
@@ -201,7 +229,7 @@ impl MimeMessage {
                         if !signatures.is_empty() {
                             let gossip_headers =
                                 decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
-                            gossipped_addr = update_gossip_peerstates(
+                            gossiped_addr = update_gossip_peerstates(
                                 context,
                                 message_time,
                                 &mail,
@@ -226,6 +254,7 @@ impl MimeMessage {
                             &mut headers,
                             &mut recipients,
                             &mut throwaway_from,
+                            &mut list_post,
                             &mut chat_disposition_notification_to,
                             &decrypted_mail.headers,
                         );
@@ -253,26 +282,40 @@ impl MimeMessage {
             parts: Vec::new(),
             header: headers,
             recipients,
+            list_post,
             from,
             chat_disposition_notification_to,
             decrypting_failed: false,
 
             // only non-empty if it was a valid autocrypt message
             signatures,
-            gossipped_addr,
+            gossiped_addr,
             is_forwarded: false,
             mdn_reports: Vec::new(),
             is_system_message: SystemMessage::Unknown,
             location_kml: None,
             message_kml: None,
+            sync_items: None,
             user_avatar: None,
             group_avatar: None,
             failure_report: None,
             footer: None,
             is_mime_modified: false,
             decoded_data: Vec::new(),
+            hop_info,
         };
-        parser.parse_mime_recursive(context, &mail, false).await?;
+
+        match partial {
+            Some(org_bytes) => {
+                parser
+                    .create_stub_from_partial_download(context, org_bytes)
+                    .await?;
+            }
+            None => {
+                parser.parse_mime_recursive(context, &mail, false).await?;
+            }
+        };
+
         parser.maybe_remove_bad_parts();
         parser.maybe_remove_inline_mailinglist_footer();
         parser.heuristically_parse_ndn(context).await;
@@ -293,7 +336,7 @@ impl MimeMessage {
 
     /// Parses system messages.
     fn parse_system_message_headers(&mut self, context: &Context) {
-        if self.get(HeaderDef::AutocryptSetupMessage).is_some() {
+        if self.get_header(HeaderDef::AutocryptSetupMessage).is_some() {
             self.parts = self
                 .parts
                 .iter()
@@ -309,7 +352,7 @@ impl MimeMessage {
             } else {
                 warn!(context, "could not determine ASM mime-part");
             }
-        } else if let Some(value) = self.get(HeaderDef::ChatContent) {
+        } else if let Some(value) = self.get_header(HeaderDef::ChatContent) {
             if value == "location-streaming-enabled" {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
             } else if value == "ephemeral-timer-changed" {
@@ -324,19 +367,19 @@ impl MimeMessage {
 
     /// Parses avatar action headers.
     async fn parse_avatar_headers(&mut self, context: &Context) {
-        if let Some(header_value) = self.get(HeaderDef::ChatGroupAvatar).cloned() {
+        if let Some(header_value) = self.get_header(HeaderDef::ChatGroupAvatar).cloned() {
             self.group_avatar = self.avatar_action_from_header(context, header_value).await;
         }
 
-        if let Some(header_value) = self.get(HeaderDef::ChatUserAvatar).cloned() {
+        if let Some(header_value) = self.get_header(HeaderDef::ChatUserAvatar).cloned() {
             self.user_avatar = self.avatar_action_from_header(context, header_value).await;
         }
     }
 
     fn parse_videochat_headers(&mut self) {
-        if let Some(value) = self.get(HeaderDef::ChatContent).cloned() {
+        if let Some(value) = self.get_header(HeaderDef::ChatContent).cloned() {
             if value == "videochat-invitation" {
-                let instance = self.get(HeaderDef::ChatWebrtcRoom).cloned();
+                let instance = self.get_header(HeaderDef::ChatWebrtcRoom).cloned();
                 if let Some(part) = self.parts.first_mut() {
                     part.typ = Viewtype::VideochatInvitation;
                     part.param
@@ -393,11 +436,12 @@ impl MimeMessage {
         }
 
         if let Some(mut part) = self.parts.pop() {
-            if part.typ == Viewtype::Audio && self.get(HeaderDef::ChatVoiceMessage).is_some() {
+            if part.typ == Viewtype::Audio && self.get_header(HeaderDef::ChatVoiceMessage).is_some()
+            {
                 part.typ = Viewtype::Voice;
             }
             if part.typ == Viewtype::Image || part.typ == Viewtype::Gif {
-                if let Some(value) = self.get(HeaderDef::ChatContent) {
+                if let Some(value) = self.get_header(HeaderDef::ChatContent) {
                     if value == "sticker" {
                         part.typ = Viewtype::Sticker;
                     }
@@ -407,7 +451,7 @@ impl MimeMessage {
                 || part.typ == Viewtype::Voice
                 || part.typ == Viewtype::Video
             {
-                if let Some(field_0) = self.get(HeaderDef::ChatDuration) {
+                if let Some(field_0) = self.get_header(HeaderDef::ChatDuration) {
                     let duration_ms = field_0.parse().unwrap_or_default();
                     if duration_ms > 0 && duration_ms < 24 * 60 * 60 * 1000 {
                         part.param.set_int(Param::Duration, duration_ms);
@@ -589,12 +633,12 @@ impl MimeMessage {
     }
 
     pub(crate) fn get_subject(&self) -> Option<String> {
-        self.get(HeaderDef::Subject)
+        self.get_header(HeaderDef::Subject)
             .filter(|s| !s.is_empty())
             .map(|s| s.to_string())
     }
 
-    pub fn get(&self, headerdef: HeaderDef) -> Option<&String> {
+    pub fn get_header(&self, headerdef: HeaderDef) -> Option<&String> {
         self.header.get(headerdef.get_headername())
     }
 
@@ -787,6 +831,12 @@ impl MimeMessage {
                                 }
                             }
                         }
+                        Some("multi-device-sync") => {
+                            if let Some(second) = mail.subparts.get(1) {
+                                self.add_single_part_if_known(context, second, is_related)
+                                    .await?;
+                            }
+                        }
                         Some(_) => {
                             if let Some(first) = mail.subparts.get(0) {
                                 any_part_added = self
@@ -903,13 +953,14 @@ impl MimeMessage {
                             (simplified_txt, top_quote)
                         };
 
-                        let simplified_txt =
-                            if simplified_txt.len() > DC_DESIRED_TEXT_LEN + DC_ELLIPSE.len() {
-                                self.is_mime_modified = true;
-                                dc_truncate(&*simplified_txt, DC_DESIRED_TEXT_LEN).to_string()
-                            } else {
-                                simplified_txt
-                            };
+                        let simplified_txt = if simplified_txt.chars().count()
+                            > DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len()
+                        {
+                            self.is_mime_modified = true;
+                            dc_truncate(&*simplified_txt, DC_DESIRED_TEXT_LEN).to_string()
+                        } else {
+                            simplified_txt
+                        };
 
                         if !simplified_txt.is_empty() || simplified_quote.is_some() {
                             let mut part = Part {
@@ -972,7 +1023,20 @@ impl MimeMessage {
                 }
                 return;
             }
+        } else if filename == "multi-device-sync.json" {
+            let serialized = String::from_utf8_lossy(decoded_data)
+                .parse()
+                .unwrap_or_default();
+            self.sync_items = context
+                .parse_sync_items(serialized)
+                .await
+                .map_err(|err| {
+                    warn!(context, "failed to parse sync data: {}", err);
+                })
+                .ok();
+            return;
         }
+
         /* we have a regular file attachment,
         write decoded data to new blob object */
 
@@ -1016,12 +1080,12 @@ impl MimeMessage {
     }
 
     pub(crate) fn get_mailinglist_type(&self) -> MailinglistType {
-        if self.get(HeaderDef::ListId).is_some() {
+        if self.get_header(HeaderDef::ListId).is_some() {
             return MailinglistType::ListIdBased;
-        } else if self.get(HeaderDef::Sender).is_some() {
+        } else if self.get_header(HeaderDef::Sender).is_some() {
             // the `Sender:`-header alone is no indicator for mailing list
             // as also used for bot-impersonation via `set_override_sender_name()`
-            if let Some(precedence) = self.get(HeaderDef::Precedence) {
+            if let Some(precedence) = self.get_header(HeaderDef::Precedence) {
                 if precedence == "list" || precedence == "bulk" {
                     return MailinglistType::SenderBased;
                 }
@@ -1037,18 +1101,18 @@ impl MimeMessage {
         }
     }
 
-    pub fn repl_msg_by_error(&mut self, error_msg: impl AsRef<str>) {
+    pub fn repl_msg_by_error(&mut self, error_msg: &str) {
         self.is_system_message = SystemMessage::Unknown;
         if let Some(part) = self.parts.first_mut() {
             part.typ = Viewtype::Text;
-            part.msg = format!("[{}]", error_msg.as_ref());
+            part.msg = format!("[{}]", error_msg);
             self.parts.truncate(1);
         }
     }
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
-        self.get(HeaderDef::XMicrosoftOriginalMessageId)
-            .or_else(|| self.get(HeaderDef::MessageId))
+        self.get_header(HeaderDef::XMicrosoftOriginalMessageId)
+            .or_else(|| self.get_header(HeaderDef::MessageId))
             .and_then(|msgid| parse_message_id(msgid).ok())
     }
 
@@ -1057,6 +1121,7 @@ impl MimeMessage {
         headers: &mut HashMap<String, String>,
         recipients: &mut Vec<SingleInfo>,
         from: &mut Vec<SingleInfo>,
+        list_post: &mut Option<String>,
         chat_disposition_notification_to: &mut Option<SingleInfo>,
         fields: &[mailparse::MailHeader<'_>],
     ) {
@@ -1087,6 +1152,10 @@ impl MimeMessage {
         if !from_new.is_empty() {
             *from = from_new;
         }
+        let list_post_new = get_list_post(fields);
+        if list_post_new.is_some() {
+            *list_post = list_post_new;
+        }
     }
 
     fn process_report(
@@ -1104,23 +1173,21 @@ impl MimeMessage {
 
         // must be present
         if let Some(_disposition) = report_fields.get_header_value(HeaderDef::Disposition) {
-            if let Some(original_message_id) = report_fields
+            let original_message_id = report_fields
                 .get_header_value(HeaderDef::OriginalMessageId)
-                .and_then(|v| parse_message_id(&v).ok())
-            {
-                let additional_message_ids = report_fields
-                    .get_header_value(HeaderDef::AdditionalMessageIds)
-                    .map_or_else(Vec::new, |v| {
-                        v.split(' ')
-                            .filter_map(|s| parse_message_id(s).ok())
-                            .collect()
-                    });
+                .and_then(|v| parse_message_id(&v).ok());
+            let additional_message_ids = report_fields
+                .get_header_value(HeaderDef::AdditionalMessageIds)
+                .map_or_else(Vec::new, |v| {
+                    v.split(' ')
+                        .filter_map(|s| parse_message_id(s).ok())
+                        .collect()
+                });
 
-                return Ok(Some(Report {
-                    original_message_id,
-                    additional_message_ids,
-                }));
-            }
+            return Ok(Some(Report {
+                original_message_id,
+                additional_message_ids,
+            }));
         }
         warn!(
             context,
@@ -1137,11 +1204,11 @@ impl MimeMessage {
         report: &mailparse::ParsedMail<'_>,
     ) -> Result<Option<FailureReport>> {
         // parse as mailheaders
-        if let Some(original_msg) = report
-            .subparts
-            .iter()
-            .find(|p| p.ctype.mimetype.contains("rfc822") || p.ctype.mimetype == "message/global")
-        {
+        if let Some(original_msg) = report.subparts.iter().find(|p| {
+            p.ctype.mimetype.contains("rfc822")
+                || p.ctype.mimetype == "message/global"
+                || p.ctype.mimetype == "message/global-headers"
+        }) {
             let report_body = original_msg.get_body_raw()?;
             let (report_fields, _) = mailparse::parse_headers(&report_body)?;
 
@@ -1235,7 +1302,7 @@ impl MimeMessage {
     /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
     #[allow(clippy::indexing_slicing)]
     async fn heuristically_parse_ndn(&mut self, context: &Context) {
-        let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
+        let maybe_ndn = if let Some(from) = self.get_header(HeaderDef::From_) {
             let from = from.to_ascii_lowercase();
             from.contains("mailer-daemon") || from.contains("mail-daemon")
         } else {
@@ -1276,8 +1343,10 @@ impl MimeMessage {
         parts: &[Part],
     ) {
         for report in &self.mdn_reports {
-            for original_message_id in
-                std::iter::once(&report.original_message_id).chain(&report.additional_message_ids)
+            for original_message_id in report
+                .original_message_id
+                .iter()
+                .chain(&report.additional_message_ids)
             {
                 match message::handle_mdn(context, from_id, original_message_id, sent_timestamp)
                     .await
@@ -1310,7 +1379,7 @@ impl MimeMessage {
     /// database, returns None.
     pub async fn get_parent_timestamp(&self, context: &Context) -> Result<Option<i64>> {
         let parent_timestamp = if let Some(field) = self
-            .get(HeaderDef::InReplyTo)
+            .get_header(HeaderDef::InReplyTo)
             .and_then(|msgid| parse_message_id(msgid).ok())
         {
             context
@@ -1327,6 +1396,9 @@ impl MimeMessage {
     }
 }
 
+/// Parses `Autocrypt-Gossip` headers from the email and applies them to peerstates.
+///
+/// Returns the set of mail recipient addresses for which valid gossip headers were found.
 async fn update_gossip_peerstates(
     context: &Context,
     message_time: i64,
@@ -1334,48 +1406,55 @@ async fn update_gossip_peerstates(
     gossip_headers: Vec<String>,
 ) -> Result<HashSet<String>> {
     // XXX split the parsing from the modification part
-    let mut gossipped_addr: HashSet<String> = Default::default();
+    let mut gossiped_addr: HashSet<String> = Default::default();
 
     for value in &gossip_headers {
-        let gossip_header = value.parse::<Aheader>();
-
-        if let Ok(ref header) = gossip_header {
-            if get_recipients(&mail.headers)
-                .iter()
-                .any(|info| info.addr == header.addr.to_lowercase())
-            {
-                let mut peerstate = Peerstate::from_addr(context, &header.addr).await?;
-                if let Some(ref mut peerstate) = peerstate {
-                    peerstate.apply_gossip(header, message_time);
-                    peerstate.save_to_db(&context.sql, false).await?;
-                } else {
-                    let p = Peerstate::from_gossip(header, message_time);
-                    p.save_to_db(&context.sql, true).await?;
-                    peerstate = Some(p);
-                }
-                if let Some(peerstate) = peerstate {
-                    peerstate
-                        .handle_fingerprint_change(context, message_time)
-                        .await?;
-                }
-
-                gossipped_addr.insert(header.addr.clone());
-            } else {
-                warn!(
-                    context,
-                    "Ignoring gossipped \"{}\" as the address is not in To/Cc list.", &header.addr,
-                );
+        let header = match value.parse::<Aheader>() {
+            Ok(header) => header,
+            Err(err) => {
+                warn!(context, "Failed parsing Autocrypt-Gossip header: {}", err);
+                continue;
             }
+        };
+
+        if !get_recipients(&mail.headers)
+            .iter()
+            .any(|info| info.addr == header.addr.to_lowercase())
+        {
+            warn!(
+                context,
+                "Ignoring gossiped \"{}\" as the address is not in To/Cc list.", &header.addr,
+            );
+            continue;
         }
+
+        let peerstate;
+        if let Some(mut p) = Peerstate::from_addr(context, &header.addr).await? {
+            p.apply_gossip(&header, message_time);
+            p.save_to_db(&context.sql, false).await?;
+            peerstate = p;
+        } else {
+            let p = Peerstate::from_gossip(&header, message_time);
+            p.save_to_db(&context.sql, true).await?;
+            peerstate = p;
+        };
+        peerstate
+            .handle_fingerprint_change(context, message_time)
+            .await?;
+
+        gossiped_addr.insert(header.addr.clone());
     }
 
-    Ok(gossipped_addr)
+    Ok(gossiped_addr)
 }
 
 #[derive(Debug)]
 pub(crate) struct Report {
     /// Original-Message-ID header
-    original_message_id: String,
+    ///
+    /// It MUST be present if the original message has a Message-ID according to RFC 8098, but MS
+    /// Exchange does not add it nevertheless, in which case it is `None`.
+    original_message_id: Option<String>,
     /// Additional-Message-IDs
     additional_message_ids: Vec<String>,
 }
@@ -1439,9 +1518,9 @@ pub struct Part {
     pub msg_raw: Option<String>,
     pub bytes: usize,
     pub param: Params,
-    org_filename: Option<String>,
+    pub(crate) org_filename: Option<String>,
     pub error: Option<String>,
-    dehtml_failed: bool,
+    pub(crate) dehtml_failed: bool,
 
     /// the part is a child or a descendant of multipart/related.
     /// typically, these are images that are referenced from text/html part
@@ -1449,7 +1528,7 @@ pub struct Part {
     ///
     /// note that multipart/related may contain further multipart nestings
     /// and all of them needs to be marked with `is_related`.
-    is_related: bool,
+    pub(crate) is_related: bool,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1565,6 +1644,14 @@ pub(crate) fn get_from(headers: &[MailHeader]) -> Vec<SingleInfo> {
     get_all_addresses_from_header(headers, |header_key| header_key == "from")
 }
 
+/// Returned addresses are normalized and lowercased.
+pub(crate) fn get_list_post(headers: &[MailHeader]) -> Option<String> {
+    get_all_addresses_from_header(headers, |header_key| header_key == "list-post")
+        .into_iter()
+        .next()
+        .map(|s| s.addr)
+}
+
 fn get_all_addresses_from_header<F>(headers: &[MailHeader], pred: F) -> Vec<SingleInfo>
 where
     F: Fn(String) -> bool,
@@ -1604,7 +1691,6 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
-    use crate::constants::DC_MAX_GET_TEXT_LEN;
     use crate::{
         chatlist::Chatlist,
         config::Config,
@@ -1980,11 +2066,11 @@ mod tests {
             .unwrap();
 
         // non-overwritten headers do not bubble up
-        let of = mimeparser.get(HeaderDef::SecureJoinGroup).unwrap();
+        let of = mimeparser.get_header(HeaderDef::SecureJoinGroup).unwrap();
         assert_eq!(of, "no");
 
         // unknown headers do not bubble upwards
-        let of = mimeparser.get(HeaderDef::_TestHeader).unwrap();
+        let of = mimeparser.get_header(HeaderDef::_TestHeader).unwrap();
         assert_eq!(of, "Bar");
 
         // the following fields would bubble up
@@ -1993,13 +2079,15 @@ mod tests {
         // for Chat-Version, also the case-insensivity is tested.
         assert_eq!(mimeparser.get_subject(), Some("outer-subject".into()));
 
-        let of = mimeparser.get(HeaderDef::ChatVersion).unwrap();
+        let of = mimeparser.get_header(HeaderDef::ChatVersion).unwrap();
         assert_eq!(of, "0.0");
         assert_eq!(mimeparser.parts.len(), 1);
 
         // make sure, headers that are only allowed in the encrypted part
         // cannot be set from the outer part
-        assert!(mimeparser.get(HeaderDef::SecureJoinFingerprint).is_none());
+        assert!(mimeparser
+            .get_header(HeaderDef::SecureJoinFingerprint)
+            .is_none());
     }
 
     #[async_std::test]
@@ -2283,7 +2371,7 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
         assert_eq!(message.mdn_reports.len(), 1);
         assert_eq!(
             message.mdn_reports[0].original_message_id,
-            "foo@example.org"
+            Some("foo@example.org".to_string())
         );
         assert_eq!(
             &message.mdn_reports[0].additional_message_ids,
@@ -2785,7 +2873,6 @@ On 2020-10-25, Bob wrote:
             &t.ctx,
             include_bytes!("../test-data/message/subj_with_multimedia_msg.eml"),
             "INBOX",
-            1,
             false,
         )
         .await
@@ -2880,21 +2967,20 @@ On 2020-10-25, Bob wrote:
         let t = TestContext::new().await;
 
         static REPEAT_TXT: &str = "this text with 42 chars is just repeated.\n";
-        static REPEAT_CNT: usize = 2000; // results in a text of 84k, should be more than DC_MAX_GET_TEXT_LEN
+        static REPEAT_CNT: usize = 2000; // results in a text of 84k, should be more than DC_DESIRED_TEXT_LEN
         let long_txt = format!("From: alice@c.de\n\n{}", REPEAT_TXT.repeat(REPEAT_CNT));
-        assert!(DC_DESIRED_TEXT_LEN + DC_ELLIPSE.len() < DC_MAX_GET_TEXT_LEN);
 
         let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref())
             .await
             .unwrap();
         assert_eq!(long_txt.matches("just repeated").count(), REPEAT_CNT);
-        assert!(long_txt.len() > DC_MAX_GET_TEXT_LEN);
+        assert!(long_txt.len() > DC_DESIRED_TEXT_LEN);
         assert!(mimemsg.is_mime_modified);
         assert!(
             mimemsg.parts[0].msg.matches("just repeated").count()
-                < DC_MAX_GET_TEXT_LEN / REPEAT_TXT.len()
+                <= DC_DESIRED_TEXT_LEN / REPEAT_TXT.len()
         );
-        assert!(mimemsg.parts[0].msg.len() <= DC_MAX_GET_TEXT_LEN);
+        assert!(mimemsg.parts[0].msg.len() <= DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len());
     }
 
     #[async_std::test]
@@ -2935,7 +3021,7 @@ Subject: ...
 
 Some quote.
 "###;
-        dc_receive_imf(&t, raw, "INBOX", 1, false).await?;
+        dc_receive_imf(&t, raw, "INBOX", false).await?;
 
         // Delta Chat generates In-Reply-To with a starting tab when Message-ID is too long.
         let raw = br###"In-Reply-To:
@@ -2952,7 +3038,7 @@ Subject: ...
 Some reply
 "###;
 
-        dc_receive_imf(&t, raw, "INBOX", 2, false).await?;
+        dc_receive_imf(&t, raw, "INBOX", false).await?;
 
         let msg = t.get_last_msg().await;
         assert_eq!(msg.get_text().unwrap(), "Some reply");
@@ -2972,25 +3058,107 @@ Some reply
 Chat-Version: 1.0\n\
 Message-ID: <foobarbaz@example.org>
 To: Bob <bob@example.org>
-From: Alice <alice@example.com>
+From: Alice <alice@example.org>
 Subject: subject
-Chat-Disposition-Notification-To: alice@example.com
+Chat-Disposition-Notification-To: alice@example.org
 
 Message.
 "###;
 
         // Bob receives message.
-        dc_receive_imf(&bob, raw, "INBOX", 1, false).await?;
+        dc_receive_imf(&bob, raw, "INBOX", false).await?;
         let msg = bob.get_last_msg().await;
         // Message is incoming.
         assert!(msg.param.get_bool(Param::WantsMdn).unwrap());
 
         // Alice receives copy-to-self.
-        dc_receive_imf(&alice, raw, "INBOX", 1, false).await?;
+        dc_receive_imf(&alice, raw, "INBOX", false).await?;
         let msg = alice.get_last_msg().await;
         // Message is outgoing, don't send read receipt to self.
         assert!(msg.param.get_bool(Param::WantsMdn).is_none());
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_ignore_read_receipt_to_self() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        // Alice receives BCC-self copy of a message sent to Bob.
+        dc_receive_imf(
+            &alice,
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                 From: alice@example.org\n\
+                 To: bob@example.net\n\
+                 Subject: foo\n\
+                 Message-ID: first@example.com\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Disposition-Notification-To: alice@example.org\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n"
+                .as_bytes(),
+            "INBOX",
+            false,
+        )
+        .await?;
+        let msg = alice.get_last_msg().await;
+        assert_eq!(msg.state, MessageState::OutDelivered);
+
+        // Due to a bug in the old version running on the other device, Alice receives a read
+        // receipt from self.
+        dc_receive_imf(
+            &alice,
+                "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                 From: alice@example.org\n\
+                 To: alice@example.org\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: second@example.com\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Original-Recipient: rfc822;bob@example.com\n\
+                 Final-Recipient: rfc822;bob@example.com\n\
+                 Original-Message-ID: <first@example.com>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n\
+                 \n\
+                 --SNIPP--"
+            .as_bytes(),
+            "INBOX",
+            false,
+        )
+        .await?;
+
+        // Check that the state has not changed to `MessageState::OutMdnRcvd`.
+        let msg = Message::load_from_db(&alice, msg.id).await?;
+        assert_eq!(msg.state, MessageState::OutDelivered);
+
+        Ok(())
+    }
+
+    /// Test parsing of MDN sent by MS Exchange.
+    ///
+    /// It does not have required Original-Message-ID field, so it is useless, but we want to
+    /// recognize it as MDN nevertheless to avoid displaying it in the chat as normal message.
+    #[async_std::test]
+    async fn test_ms_exchange_mdn() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let raw =
+            include_bytes!("../test-data/message/ms_exchange_report_disposition_notification.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await?;
+        assert!(!mimeparser.mdn_reports.is_empty());
+        Ok(())
+    }
 }

src/oauth2.rs

@@ -1,4 +1,4 @@
-//! OAuth 2 module
+//! OAuth 2 module.
 
 use std::collections::HashMap;
 
@@ -6,6 +6,7 @@ use anyhow::Result;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
+use crate::config::Config;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::provider;
@@ -40,6 +41,7 @@ struct Oauth2 {
 
 /// OAuth 2 Access Token Response
 #[derive(Debug, Deserialize)]
+#[allow(dead_code)]
 struct Response {
     // Should always be there according to: <https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/>
     // but previous code handled its abscense.
@@ -55,22 +57,19 @@ pub async fn dc_get_oauth2_url(
     context: &Context,
     addr: &str,
     redirect_uri: &str,
-) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr).await {
-        if context
+) -> Result<Option<String>> {
+    let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
+    if let Some(oauth2) = Oauth2::from_address(context, addr, socks5_enabled).await {
+        context
             .sql
             .set_raw_config("oauth2_pending_redirect_uri", Some(redirect_uri))
-            .await
-            .is_err()
-        {
-            return None;
-        }
+            .await?;
         let oauth2_url = replace_in_uri(oauth2.get_code, "$CLIENT_ID", oauth2.client_id);
         let oauth2_url = replace_in_uri(&oauth2_url, "$REDIRECT_URI", redirect_uri);
 
-        Some(oauth2_url)
+        Ok(Some(oauth2_url))
     } else {
-        None
+        Ok(None)
     }
 }
 
@@ -80,7 +79,8 @@ pub async fn dc_get_oauth2_access_token(
     code: &str,
     regenerate: bool,
 ) -> Result<Option<String>> {
-    if let Some(oauth2) = Oauth2::from_address(addr).await {
+    let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
+    if let Some(oauth2) = Oauth2::from_address(context, addr, socks5_enabled).await {
         let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
@@ -225,7 +225,8 @@ pub async fn dc_get_oauth2_addr(
     addr: &str,
     code: &str,
 ) -> Result<Option<String>> {
-    let oauth2 = match Oauth2::from_address(addr).await {
+    let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
+    let oauth2 = match Oauth2::from_address(context, addr, socks5_enabled).await {
         Some(o) => o,
         None => return Ok(None),
     };
@@ -253,13 +254,13 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    async fn from_address(addr: &str) -> Option<Self> {
+    async fn from_address(context: &Context, addr: &str, skip_mx: bool) -> Option<Self> {
         let addr_normalized = normalize_addr(addr);
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            if let Some(oauth2_authorizer) = provider::get_provider_info(domain)
+            if let Some(oauth2_authorizer) = provider::get_provider_info(context, domain, skip_mx)
                 .await
                 .and_then(|provider| provider.oauth2_authorizer.as_ref())
             {
@@ -356,32 +357,39 @@ mod tests {
 
     #[async_std::test]
     async fn test_oauth_from_address() {
+        let t = TestContext::new().await;
         assert_eq!(
-            Oauth2::from_address("hello@gmail.com").await,
+            Oauth2::from_address(&t, "hello@gmail.com", false).await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com").await,
+            Oauth2::from_address(&t, "hello@googlemail.com", false).await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com").await,
+            Oauth2::from_address(&t, "hello@yandex.com", false).await,
             Some(OAUTH2_YANDEX)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.ru").await,
+            Oauth2::from_address(&t, "hello@yandex.ru", false).await,
             Some(OAUTH2_YANDEX)
         );
-
-        assert_eq!(Oauth2::from_address("hello@web.de").await, None);
+        assert_eq!(Oauth2::from_address(&t, "hello@web.de", false).await, None);
     }
 
     #[async_std::test]
     async fn test_oauth_from_mx() {
+        // youtube staff seems to use "google workspace with oauth2", figures this out by MX lookup
+        let t = TestContext::new().await;
         assert_eq!(
-            Oauth2::from_address("hello@google.com").await,
+            Oauth2::from_address(&t, "hello@youtube.com", false).await,
             Some(OAUTH2_GMAIL)
         );
+        // without MX lookup, we would not know as youtube.com is not in our provider-db
+        assert_eq!(
+            Oauth2::from_address(&t, "hello@youtube.com", true).await,
+            None
+        );
     }
 
     #[async_std::test]
@@ -399,7 +407,9 @@ mod tests {
         let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let redirect_uri = "chat.delta:/com.b44t.messenger";
-        let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri).await;
+        let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri)
+            .await
+            .unwrap();
 
         assert_eq!(res, Some("https://accounts.google.com/o/oauth2/auth?client_id=959970109878%2D4mvtgf6feshskf7695nfln6002mom908%2Eapps%2Egoogleusercontent%2Ecom&redirect_uri=chat%2Edelta%3A%2Fcom%2Eb44t%2Emessenger&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline".into()));
     }

src/param.rs

@@ -4,7 +4,6 @@ use std::str;
 
 use anyhow::{bail, Error};
 use async_std::path::PathBuf;
-use itertools::Itertools;
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 
@@ -114,6 +113,9 @@ pub enum Param {
     /// For Jobs: space-separated list of message recipients
     Recipients = b'R',
 
+    /// For MDN-sending job
+    MsgId = b'I',
+
     /// For Groups
     ///
     /// An unpromoted group has not had any messages sent to it and thus only exists on the
@@ -137,8 +139,38 @@ pub enum Param {
     /// For Chats
     Devicetalk = b'D',
 
-    /// For MDN-sending job
-    MsgId = b'I',
+    /// For Chats: If this is a mailing list chat, contains the List-Post address.
+    /// None if there simply is no `List-Post` header in the mailing list.
+    /// Some("") if the mailing list is using multiple different List-Post headers.
+    ///
+    /// The List-Post address is the email address where the user can write to in order to
+    /// post something to the mailing list.
+    ListPost = b'p',
+
+    /// For Contacts: If this is the List-Post address of a mailing list, contains
+    /// the List-Id of the mailing list (which is also used as the group id of the chat).
+    ListId = b's',
+
+    /// For Contacts: timestamp of status (aka signature or footer) update.
+    StatusTimestamp = b'j',
+
+    /// For Contacts and Chats: timestamp of avatar update.
+    AvatarTimestamp = b'J',
+
+    /// For Chats: timestamp of status/signature/footer update.
+    EphemeralSettingsTimestamp = b'B',
+
+    /// For Chats: timestamp of subject update.
+    SubjectTimestamp = b'C',
+
+    /// For Chats: timestamp of group name update.
+    GroupNameTimestamp = b'g',
+
+    /// For Chats: timestamp of group name update.
+    MemberListTimestamp = b'k',
+
+    /// For Chats: timestamp of protection settings update.
+    ProtectionSettingsTimestamp = b'L',
 }
 
 /// An object for handling key=value parameter lists.
@@ -161,7 +193,7 @@ impl fmt::Display for Params {
                 f,
                 "{}={}",
                 *key as u8 as char,
-                value.split('\n').join("\n\n")
+                value.split('\n').collect::<Vec<&str>>().join("\n\n")
             )?;
         }
         Ok(())
@@ -171,6 +203,11 @@ impl fmt::Display for Params {
 impl str::FromStr for Params {
     type Err = Error;
 
+    /// Parse a raw string to Param.
+    ///
+    /// Silently ignore unknown keys:
+    /// they may come from a downgrade (when a shortly new version adds a key)
+    /// or from an upgrade (when a key is dropped but was used in the past)
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let mut inner = BTreeMap::new();
         let mut lines = s.lines().peekable();
@@ -190,8 +227,6 @@ impl str::FromStr for Params {
 
                 if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
                     inner.insert(key, value);
-                } else {
-                    bail!("Unknown key: {}", key);
                 }
             } else {
                 bail!("Not a key-value pair: {:?}", line);
@@ -245,6 +280,11 @@ impl Params {
         self.get(key).and_then(|s| s.parse().ok())
     }
 
+    /// Get the given parameter and parse as `i64`.
+    pub fn get_i64(&self, key: Param) -> Option<i64> {
+        self.get(key).and_then(|s| s.parse().ok())
+    }
+
     /// Get the given parameter and parse as `bool`.
     pub fn get_bool(&self, key: Param) -> Option<bool> {
         self.get_int(key).map(|v| v != 0)
@@ -346,6 +386,12 @@ impl Params {
         self
     }
 
+    /// Set the given paramter to the passed in `i64`.
+    pub fn set_i64(&mut self, key: Param, value: i64) -> &mut Self {
+        self.set(key, value.to_string());
+        self
+    }
+
     /// Set the given parameter to the passed in `f64` .
     pub fn set_float(&mut self, key: Param, value: f64) -> &mut Self {
         self.set(key, format!("{}", value));
@@ -383,10 +429,12 @@ impl<'a> ParamsFile<'a> {
 mod tests {
     use super::*;
 
+    use anyhow::Result;
     use async_std::fs;
     use async_std::path::Path;
 
     use crate::test_utils::TestContext;
+    use std::str::FromStr;
 
     #[test]
     fn test_dc_param() {
@@ -489,4 +537,14 @@ mod tests {
         assert!(p.get_path(Param::File, &t).unwrap().is_none());
         assert!(p.get_blob(Param::File, &t, false).await.unwrap().is_none());
     }
+
+    #[async_std::test]
+    async fn test_params_unknown_key() -> Result<()> {
+        // 'Z' is used as a key that is known to be unused; these keys should be ignored silently by definition.
+        let p = Params::from_str("w=12\nZ=13\nh=14")?;
+        assert_eq!(p.len(), 2);
+        assert_eq!(p.get(Param::Width), Some("12"));
+        assert_eq!(p.get(Param::Height), Some("14"));
+        Ok(())
+    }
 }

src/peerstate.rs

@@ -1,4 +1,4 @@
-//! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
+//! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module.
 
 use std::collections::HashSet;
 use std::fmt;
@@ -277,8 +277,8 @@ impl Peerstate {
 
                 let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
 
-                chat::add_info_msg(context, chat_id, msg, timestamp).await;
-                emit_event!(context, EventType::ChatModified(chat_id));
+                chat::add_info_msg(context, chat_id, &msg, timestamp).await?;
+                context.emit_event(EventType::ChatModified(chat_id));
             } else {
                 bail!("contact with peerstate.addr {:?} not found", &self.addr);
             }
@@ -496,11 +496,34 @@ impl Peerstate {
     }
 }
 
+/// Removes duplicate peerstates from `acpeerstates` database table.
+///
+/// Normally there should be no more than one peerstate per address.
+/// However, the database does not enforce this condition.
+///
+/// Previously there were bugs that caused creation of additional
+/// peerstates when existing peerstate could not be read due to a
+/// temporary database error or a failure to parse stored data.  This
+/// procedure fixes the problem by removing duplicate records.
+pub(crate) async fn deduplicate_peerstates(sql: &Sql) -> Result<()> {
+    sql.execute(
+        "DELETE FROM acpeerstates
+         WHERE id NOT IN (
+         SELECT MIN(id)
+         FROM acpeerstates
+         GROUP BY addr
+         )",
+        paramsv![],
+    )
+    .await?;
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test_utils::alice_keypair;
-    use pretty_assertions::assert_eq;
 
     #[async_std::test]
     async fn test_peerstate_save_to_db() {

src/pgp.rs

@@ -1,4 +1,4 @@
-//! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp)
+//! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp).
 
 use std::collections::{BTreeMap, HashSet};
 use std::io;
@@ -8,7 +8,7 @@ use anyhow::{bail, ensure, format_err, Result};
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
-    SignedPublicSubKey, SignedSecretKey, SubkeyParamsBuilder,
+    SignedPublicSubKey, SignedSecretKey, StandaloneSignature, SubkeyParamsBuilder,
 };
 use pgp::crypto::{HashAlgorithm, SymmetricKeyAlgorithm};
 use pgp::types::{
@@ -248,7 +248,7 @@ pub async fn pk_encrypt(
         let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
             .keys()
             .iter()
-            .filter_map(|key| select_pk_for_encryption(key))
+            .filter_map(select_pk_for_encryption)
             .collect();
         let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
@@ -277,16 +277,17 @@ pub async fn pk_encrypt(
 /// Receiver private keys are provided in
 /// `private_keys_for_decryption`.
 ///
-/// If `ret_signature_fingerprints` is not `None`, stores fingerprints
+/// Returns decrypted message and fingerprints
 /// of all keys from the `public_keys_for_validation` keyring that
 /// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
 pub async fn pk_decrypt(
     ctext: Vec<u8>,
     private_keys_for_decryption: Keyring<SignedSecretKey>,
-    public_keys_for_validation: Keyring<SignedPublicKey>,
-    ret_signature_fingerprints: Option<&mut HashSet<Fingerprint>>,
-) -> Result<Vec<u8>> {
+    public_keys_for_validation: &Keyring<SignedPublicKey>,
+) -> Result<(Vec<u8>, HashSet<Fingerprint>)> {
+    let mut ret_signature_fingerprints: HashSet<Fingerprint> = Default::default();
+
     let msgs = async_std::task::spawn_blocking(move || {
         let cursor = Cursor::new(ctext);
         let (msg, _) = Message::from_armor_single(cursor)?;
@@ -308,33 +309,54 @@ pub async fn pk_decrypt(
             None => bail!("The decrypted message is empty"),
         };
 
-        if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
-            if !public_keys_for_validation.is_empty() {
-                let fingerprints = async_std::task::spawn_blocking(move || {
-                    let pkeys = public_keys_for_validation.keys();
+        if !public_keys_for_validation.is_empty() {
+            let pkeys = public_keys_for_validation.keys();
 
-                    let mut fingerprints: Vec<Fingerprint> = Vec::new();
-                    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
-                        for pkey in pkeys {
-                            if signed_msg.verify(&pkey.primary_key).is_ok() {
-                                let fp = DcKey::fingerprint(pkey);
-                                fingerprints.push(fp);
-                            }
-                        }
+            let mut fingerprints: Vec<Fingerprint> = Vec::new();
+            if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+                for pkey in pkeys {
+                    if signed_msg.verify(&pkey.primary_key).is_ok() {
+                        let fp = DcKey::fingerprint(pkey);
+                        fingerprints.push(fp);
                     }
-                    fingerprints
-                })
-                .await;
-
-                ret_signature_fingerprints.extend(fingerprints);
+                }
             }
+
+            ret_signature_fingerprints.extend(fingerprints);
         }
-        Ok(content)
+        Ok((content, ret_signature_fingerprints))
     } else {
         bail!("No valid messages found");
     }
 }
 
+/// Validates detached signature.
+pub async fn pk_validate(
+    content: &[u8],
+    signature: &[u8],
+    public_keys_for_validation: &Keyring<SignedPublicKey>,
+) -> Result<HashSet<Fingerprint>> {
+    let mut ret: HashSet<Fingerprint> = Default::default();
+
+    let standalone_signature = StandaloneSignature::from_armor_single(Cursor::new(signature))?.0;
+    let pkeys = public_keys_for_validation.keys();
+
+    // Remove trailing CRLF before the delimiter.
+    // According to RFC 3156 it is considered to be part of the MIME delimiter for the purpose of
+    // OpenPGP signature calculation.
+    let content = content
+        .get(..content.len().saturating_sub(2))
+        .ok_or_else(|| format_err!("index is out of range"))?;
+
+    for pkey in pkeys {
+        if standalone_signature.verify(pkey, content).is_ok() {
+            let fp = DcKey::fingerprint(pkey);
+            ret.insert(fp);
+        }
+    }
+    Ok(ret)
+}
+
 /// Symmetric encryption.
 pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
@@ -492,12 +514,10 @@ mod tests {
         decrypt_keyring.add(KEYS.alice_secret.clone());
         let mut sig_check_keyring: Keyring<SignedPublicKey> = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
+        let (plain, valid_signatures) = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            sig_check_keyring,
-            Some(&mut valid_signatures),
+            &sig_check_keyring,
         )
         .await
         .map_err(|err| println!("{:?}", err))
@@ -510,12 +530,10 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
+        let (plain, valid_signatures) = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            sig_check_keyring,
-            Some(&mut valid_signatures),
+            &sig_check_keyring,
         )
         .await
         .map_err(|err| println!("{:?}", err))
@@ -529,15 +547,10 @@ mod tests {
         let mut keyring = Keyring::new();
         keyring.add(KEYS.alice_secret.clone());
         let empty_keyring = Keyring::new();
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes().to_vec(),
-            keyring,
-            empty_keyring,
-            Some(&mut valid_signatures),
-        )
-        .await
-        .unwrap();
+        let (plain, valid_signatures) =
+            pk_decrypt(CTEXT_SIGNED.as_bytes().to_vec(), keyring, &empty_keyring)
+                .await
+                .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
@@ -549,12 +562,10 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.bob_public.clone());
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
+        let (plain, valid_signatures) = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            sig_check_keyring,
-            Some(&mut valid_signatures),
+            &sig_check_keyring,
         )
         .await
         .unwrap();
@@ -567,34 +578,14 @@ mod tests {
         let mut decrypt_keyring = Keyring::new();
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let sig_check_keyring = Keyring::new();
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
+        let (plain, valid_signatures) = pk_decrypt(
             CTEXT_UNSIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            sig_check_keyring,
-            Some(&mut valid_signatures),
+            &sig_check_keyring,
         )
         .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
-
-    #[async_std::test]
-    async fn test_decrypt_signed_no_sigret() {
-        // Check decrypting signed cyphertext without providing the HashSet for signatures.
-        let mut decrypt_keyring = Keyring::new();
-        decrypt_keyring.add(KEYS.bob_secret.clone());
-        let mut sig_check_keyring = Keyring::new();
-        sig_check_keyring.add(KEYS.alice_public.clone());
-        let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes().to_vec(),
-            decrypt_keyring,
-            sig_check_keyring,
-            None,
-        )
-        .await
-        .unwrap();
-        assert_eq!(plain, CLEARTEXT);
-    }
 }

src/plaintext.rs

@@ -1,4 +1,5 @@
-///! Handle plain text together with some attributes.
+//! Handle plain text together with some attributes.
+
 use crate::simplify::split_lines;
 use once_cell::sync::Lazy;
 

src/provider.rs

@@ -1,10 +1,12 @@
-//! [Provider database](https://providers.delta.chat/) module
+//! [Provider database](https://providers.delta.chat/) module.
 
 mod data;
 
 use crate::config::Config;
+use crate::context::Context;
 use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED};
-use async_std_resolver::{config, resolver};
+use anyhow::Result;
+use async_std_resolver::{config, resolver, resolver_from_system_conf, AsyncStdResolver};
 use chrono::{NaiveDateTime, NaiveTime};
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
@@ -81,6 +83,23 @@ pub struct Provider {
     pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
+/// Get resolver to query MX records.
+///
+/// We first try resolver_from_system_conf() which reads the system's resolver from `/etc/resolv.conf`.
+/// This does not work at least on some Androids, therefore we use use ResolverConfig::default()
+/// which default eg. to google's 8.8.8.8 or 8.8.4.4 as a fallback.
+async fn get_resolver() -> Result<AsyncStdResolver> {
+    if let Ok(resolver) = resolver_from_system_conf().await {
+        return Ok(resolver);
+    }
+    let resolver = resolver(
+        config::ResolverConfig::default(),
+        config::ResolverOpts::default(),
+    )
+    .await?;
+    Ok(resolver)
+}
+
 /// Returns provider for the given domain.
 ///
 /// This function looks up domain in offline database first. If not
@@ -89,15 +108,21 @@ pub struct Provider {
 ///
 /// For compatibility, email address can be passed to this function
 /// instead of the domain.
-pub async fn get_provider_info(domain: &str) -> Option<&'static Provider> {
+pub async fn get_provider_info(
+    context: &Context,
+    domain: &str,
+    skip_mx: bool,
+) -> Option<&'static Provider> {
     let domain = domain.rsplitn(2, '@').next()?;
 
     if let Some(provider) = get_provider_by_domain(domain) {
         return Some(provider);
     }
 
-    if let Some(provider) = get_provider_by_mx(domain).await {
-        return Some(provider);
+    if !skip_mx {
+        if let Some(provider) = get_provider_by_mx(context, domain).await {
+            return Some(provider);
+        }
     }
 
     None
@@ -115,13 +140,8 @@ pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
 /// Finds a provider based on MX record for the given domain.
 ///
 /// For security reasons, only Gmail can be configured this way.
-pub async fn get_provider_by_mx(domain: &str) -> Option<&'static Provider> {
-    if let Ok(resolver) = resolver(
-        config::ResolverConfig::default(),
-        config::ResolverOpts::default(),
-    )
-    .await
-    {
+pub async fn get_provider_by_mx(context: &Context, domain: &str) -> Option<&'static Provider> {
+    if let Ok(resolver) = get_resolver().await {
         let mut fqdn: String = domain.to_string();
         if !fqdn.ends_with('.') {
             fqdn.push('.');
@@ -146,6 +166,8 @@ pub async fn get_provider_by_mx(domain: &str) -> Option<&'static Provider> {
                 }
             }
         }
+    } else {
+        warn!(context, "cannot get a resolver to check MX records.");
     }
 
     None
@@ -172,6 +194,7 @@ mod tests {
 
     use super::*;
     use crate::dc_tools::time;
+    use crate::test_utils::TestContext;
     use chrono::NaiveDate;
 
     #[test]
@@ -221,11 +244,18 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_provider_info() {
-        assert!(get_provider_info("").await.is_none());
-        assert!(get_provider_info("google.com").await.unwrap().id == "gmail");
+        let t = TestContext::new().await;
+        assert!(get_provider_info(&t, "", false).await.is_none());
+        assert!(get_provider_info(&t, "google.com", false).await.unwrap().id == "gmail");
 
         // get_provider_info() accepts email addresses for backwards compatibility
-        assert!(get_provider_info("example@google.com").await.unwrap().id == "gmail");
+        assert!(
+            get_provider_info(&t, "example@google.com", false)
+                .await
+                .unwrap()
+                .id
+                == "gmail"
+        );
     }
 
     #[test]
@@ -239,4 +269,10 @@ mod tests {
         assert!(get_provider_update_timestamp() <= time());
         assert!(get_provider_update_timestamp() > timestamp_past);
     }
+
+    #[async_std::test]
+    async fn test_get_resolver() -> Result<()> {
+        assert!(get_resolver().await.is_ok());
+        Ok(())
+    }
 }

src/provider/data.rs

@@ -131,6 +131,35 @@ static P_AUTISTICI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
+// blindzeln.org.md: delta.blinzeln.de, delta.blindzeln.org
+static P_BLINDZELN_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "blindzeln.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/blindzeln-org",
+    server: vec![
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "webbox222.server-home.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "webbox222.server-home.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    config_defaults: None,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
+    oauth2_authorizer: None,
+});
+
 // bluewin.ch.md: bluewin.ch
 static P_BLUEWIN_CH: Lazy<Provider> = Lazy::new(|| Provider {
     id: "bluewin.ch",
@@ -299,10 +328,6 @@ static P_DUBBY_ORG: Lazy<Provider> = Lazy::new(|| Provider {
             key: Config::SentboxWatch,
             value: "0",
         },
-        ConfigDefault {
-            key: Config::MvboxWatch,
-            value: "0",
-        },
         ConfigDefault {
             key: Config::MvboxMove,
             value: "0",
@@ -446,10 +471,6 @@ static P_FIVE_CHAT: Lazy<Provider> = Lazy::new(|| Provider {
             key: Config::SentboxWatch,
             value: "0",
         },
-        ConfigDefault {
-            key: Config::MvboxWatch,
-            value: "0",
-        },
         ConfigDefault {
             key: Config::MvboxMove,
             value: "0",
@@ -794,10 +815,6 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| Provider {
             key: Config::SentboxWatch,
             value: "0",
         },
-        ConfigDefault {
-            key: Config::MvboxWatch,
-            value: "0",
-        },
         ConfigDefault {
             key: Config::MvboxMove,
             value: "0",
@@ -1072,10 +1089,6 @@ static P_TESTRUN: Lazy<Provider> = Lazy::new(|| Provider {
             key: Config::SentboxWatch,
             value: "0",
         },
-        ConfigDefault {
-            key: Config::MvboxWatch,
-            value: "0",
-        },
         ConfigDefault {
             key: Config::MvboxMove,
             value: "0",
@@ -1363,7 +1376,7 @@ static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// zoho.md: zohomail.eu, zoho.com
+// zoho.md: zohomail.eu, zohomail.com, zoho.com
 static P_ZOHO: Lazy<Provider> = Lazy::new(|| Provider {
     id: "zoho",
     status: Status::Preparation,
@@ -1399,6 +1412,8 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("aol.com", &*P_AOL),
         ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
+        ("delta.blinzeln.de", &*P_BLINDZELN_ORG),
+        ("delta.blindzeln.org", &*P_BLINDZELN_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
         ("buzon.uy", &*P_BUZON_UY),
         ("chello.at", &*P_CHELLO_AT),
@@ -1580,6 +1595,7 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("yggmail", &*P_YGGMAIL),
         ("ziggo.nl", &*P_ZIGGO_NL),
         ("zohomail.eu", &*P_ZOHO),
+        ("zohomail.com", &*P_ZOHO),
         ("zoho.com", &*P_ZOHO),
     ]
     .iter()
@@ -1594,6 +1610,7 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("aol", &*P_AOL),
         ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
+        ("blindzeln.org", &*P_BLINDZELN_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
         ("buzon.uy", &*P_BUZON_UY),
         ("chello.at", &*P_CHELLO_AT),
@@ -1653,4 +1670,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 8, 17));
+    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 9, 29));

src/qr_code_generator.rs

@@ -0,0 +1,265 @@
+use anyhow::Result;
+use qrcodegen::{QrCode, QrCodeEcc};
+
+use crate::{
+    blob::BlobObject,
+    chat::{Chat, ChatId},
+    color::color_int_to_hex_string,
+    config::Config,
+    constants::DC_CONTACT_ID_SELF,
+    contact::Contact,
+    context::Context,
+    securejoin, stock_str,
+};
+
+pub async fn get_securejoin_qr_svg(context: &Context, chat_id: Option<ChatId>) -> Result<String> {
+    if let Some(chat_id) = chat_id {
+        generate_join_group_qr_code(context, chat_id).await
+    } else {
+        generate_verification_qr(context).await
+    }
+}
+
+async fn generate_join_group_qr_code(context: &Context, chat_id: ChatId) -> Result<String> {
+    let chat = Chat::load_from_db(context, chat_id).await?;
+
+    let avatar = match chat.get_profile_image(context).await? {
+        Some(path) => {
+            let avatar_blob = BlobObject::from_path(context, &path)?;
+            Some(std::fs::read(avatar_blob.to_abs_path())?)
+        }
+        None => None,
+    };
+
+    inner_generate_secure_join_qr_code(
+        &stock_str::secure_join_group_qr_description(context, &chat).await,
+        &securejoin::dc_get_securejoin_qr(context, Some(chat_id)).await?,
+        &color_int_to_hex_string(chat.get_color(context).await?),
+        avatar,
+        chat.get_name().chars().next().unwrap_or('#'),
+    )
+}
+
+async fn generate_verification_qr(context: &Context) -> Result<String> {
+    let contact = Contact::get_by_id(context, DC_CONTACT_ID_SELF).await?;
+
+    let avatar = match contact.get_profile_image(context).await? {
+        Some(path) => {
+            let avatar_blob = BlobObject::from_path(context, &path)?;
+            Some(std::fs::read(avatar_blob.to_abs_path())?)
+        }
+        None => None,
+    };
+
+    let displayname = match context.get_config(Config::Displayname).await? {
+        Some(name) => name,
+        None => contact.get_addr().to_owned(),
+    };
+
+    inner_generate_secure_join_qr_code(
+        &stock_str::setup_contact_qr_description(context, &displayname, contact.get_addr()).await,
+        &securejoin::dc_get_securejoin_qr(context, None).await?,
+        &color_int_to_hex_string(contact.get_color()),
+        avatar,
+        displayname.chars().next().unwrap_or('#'),
+    )
+}
+
+fn inner_generate_secure_join_qr_code(
+    raw_qrcode_description: &str,
+    qrcode_content: &str,
+    color: &str,
+    avatar: Option<Vec<u8>>,
+    avatar_letter: char,
+) -> Result<String> {
+    let qrcode_description = &escaper::encode_minimal(raw_qrcode_description);
+    // config
+    let width = 515.0;
+    let height = 630.0;
+    let logo_offset = 28.0;
+    let qr_code_size = 400.0;
+    let qr_translate_up = 40.0;
+    let text_y_pos = ((height - qr_code_size) / 2.0) + qr_code_size;
+    let avatar_border_size = 9.0;
+    let card_border_size = 2.0;
+    let card_roundness = 40.0;
+
+    let qr = QrCode::encode_text(qrcode_content, QrCodeEcc::Medium)?;
+    let mut svg = String::with_capacity(28000);
+    let mut w = tagger::new(&mut svg);
+
+    w.elem("svg", |d| {
+        d.attr("xmlns", "http://www.w3.org/2000/svg")?;
+        d.attr("viewBox", format_args!("0 0 {} {}", width, height))?;
+        Ok(())
+    })?
+    .build(|w| {
+        // White Background apears like a card
+        w.single("rect", |d| {
+            d.attr("x", card_border_size)?;
+            d.attr("y", card_border_size)?;
+            d.attr("rx", card_roundness)?;
+            d.attr("stroke", "#c6c6c6")?;
+            d.attr("stroke-width", card_border_size)?;
+            d.attr("width", width - (card_border_size * 2.0))?;
+            d.attr("height", height - (card_border_size * 2.0))?;
+            d.attr("style", "fill:#f2f2f2")?;
+            Ok(())
+        })?;
+        // Qrcode
+        w.elem("g", |d| {
+            d.attr(
+                "transform",
+                format!(
+                    "translate({},{})",
+                    (width - qr_code_size) / 2.0,
+                    ((height - qr_code_size) / 2.0) - qr_translate_up
+                ),
+            )
+            // If the qr code should be in the wrong place,
+            // we could also translate and scale the points in the path already,
+            // but that would make the resulting svg way bigger in size and might bring up rounding issues,
+            // so better avoid doing it manually if possible
+        })?
+        .build(|w| {
+            w.single("path", |d| {
+                let mut path_data = String::with_capacity(0);
+                let scale = qr_code_size / qr.size() as f32;
+
+                for y in 0..qr.size() {
+                    for x in 0..qr.size() {
+                        if qr.get_module(x, y) {
+                            path_data += &format!("M{},{}h1v1h-1z", x, y);
+                        }
+                    }
+                }
+
+                d.attr("style", "fill:#000000")?;
+                d.attr("d", path_data)?;
+                d.attr("transform", format!("scale({})", scale))
+            })
+        })?;
+
+        // Text
+        const BIG_TEXT_CHARS_PER_LINE: usize = 32;
+        const SMALL_TEXT_CHARS_PER_LINE: usize = 38;
+        let chars_per_line = if qrcode_description.len() > SMALL_TEXT_CHARS_PER_LINE * 2 {
+            SMALL_TEXT_CHARS_PER_LINE
+        } else {
+            BIG_TEXT_CHARS_PER_LINE
+        };
+        let lines = textwrap::fill(qrcode_description, chars_per_line);
+        let (text_font_size, text_y_shift) = if lines.split('\n').count() <= 2 {
+            (27.0, 0.0)
+        } else {
+            (19.0, -10.0)
+        };
+        for (count, line) in lines.split('\n').enumerate() {
+            w.elem("text", |d| {
+                d.attr(
+                    "y",
+                    (count as f32 * (text_font_size * 1.2)) + text_y_pos + text_y_shift,
+                )?;
+                d.attr("x", width / 2.0)?;
+                d.attr("text-anchor", "middle")?;
+                d.attr(
+                    "style",
+                    format!(
+                        "font-family:sans-serif;\
+                        font-weight:bold;\
+                        font-size:{}px;\
+                        fill:#000000;\
+                        stroke:none",
+                        text_font_size
+                    ),
+                )
+            })?
+            .build(|w| w.put_raw(line))?;
+        }
+        // contact avatar in middle of qrcode
+        const LOGO_SIZE: f32 = 94.4;
+        const HALF_LOGO_SIZE: f32 = LOGO_SIZE / 2.0;
+        let logo_position_in_qr = (qr_code_size / 2.0) - HALF_LOGO_SIZE;
+        let logo_position_x = ((width - qr_code_size) / 2.0) + logo_position_in_qr;
+        let logo_position_y =
+            ((height - qr_code_size) / 2.0) - qr_translate_up + logo_position_in_qr;
+
+        w.single("circle", |d| {
+            d.attr("cx", logo_position_x + HALF_LOGO_SIZE)?;
+            d.attr("cy", logo_position_y + HALF_LOGO_SIZE)?;
+            d.attr("r", HALF_LOGO_SIZE + avatar_border_size)?;
+            d.attr("style", "fill:#f2f2f2")
+        })?;
+
+        if let Some(img) = avatar {
+            w.elem("defs", tagger::no_attr())?.build(|w| {
+                w.elem("clipPath", |d| d.attr("id", "avatar-cut"))?
+                    .build(|w| {
+                        w.single("circle", |d| {
+                            d.attr("cx", logo_position_x + HALF_LOGO_SIZE)?;
+                            d.attr("cy", logo_position_y + HALF_LOGO_SIZE)?;
+                            d.attr("r", HALF_LOGO_SIZE)
+                        })
+                    })
+            })?;
+
+            w.single("image", |d| {
+                d.attr("x", logo_position_x)?;
+                d.attr("y", logo_position_y)?;
+                d.attr("width", HALF_LOGO_SIZE * 2.0)?;
+                d.attr("height", HALF_LOGO_SIZE * 2.0)?;
+                d.attr("preserveAspectRatio", "none")?;
+                d.attr("clip-path", "url(#avatar-cut)")?;
+                d.attr(
+                    "href", /*might need xlink:href instead if it doesn't work on older devices?*/
+                    format!("data:image/jpeg;base64,{}", base64::encode(img)),
+                )
+            })?;
+        } else {
+            w.single("circle", |d| {
+                d.attr("cx", logo_position_x + HALF_LOGO_SIZE)?;
+                d.attr("cy", logo_position_y + HALF_LOGO_SIZE)?;
+                d.attr("r", HALF_LOGO_SIZE)?;
+                d.attr("style", format!("fill:{}", &color))
+            })?;
+
+            let avatar_font_size = LOGO_SIZE * 0.65;
+            let font_offset = avatar_font_size * 0.1;
+            w.elem("text", |d| {
+                d.attr("y", logo_position_y + HALF_LOGO_SIZE + font_offset)?;
+                d.attr("x", logo_position_x + HALF_LOGO_SIZE)?;
+                d.attr("text-anchor", "middle")?;
+                d.attr("dominant-baseline", "central")?;
+                d.attr("alignment-baseline", "middle")?;
+                d.attr(
+                    "style",
+                    format!(
+                        "font-family:sans-serif;\
+                            font-weight:400;\
+                            font-size:{}px;\
+                            fill:#ffffff;",
+                        avatar_font_size
+                    ),
+                )
+            })?
+            .build(|w| w.put_raw(avatar_letter.to_uppercase()))?;
+        }
+
+        // Footer logo
+        const FOOTER_HEIGHT: f32 = 35.0;
+        const FOOTER_WIDTH: f32 = 198.0;
+        w.elem("g", |d| {
+            d.attr(
+                "transform",
+                format!(
+                    "translate({},{})",
+                    (width - FOOTER_WIDTH) / 2.0,
+                    height - logo_offset - FOOTER_HEIGHT - text_y_shift
+                ),
+            )
+        })?
+        .build(|w| w.put_raw(include_str!("../assets/qrcode_logo_footer.svg")))
+    })?;
+
+    Ok(svg)
+}

src/quota.rs

@@ -1,22 +1,37 @@
+//! # Support for IMAP QUOTA extension.
+
 use anyhow::{anyhow, Result};
 use async_imap::types::{Quota, QuotaResource};
-use indexmap::IndexMap;
+use std::collections::BTreeMap;
 
+use crate::chat::add_device_msg_with_importance;
+use crate::config::Config;
+use crate::constants::Viewtype;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::imap::scan_folders::get_watched_folders;
 use crate::imap::Imap;
 use crate::job::{Action, Status};
+use crate::message::Message;
 use crate::param::Params;
-use crate::{job, EventType};
+use crate::{job, stock_str, EventType};
 
 /// warn about a nearly full mailbox after this usage percentage is reached.
 /// quota icon is "yellow".
 pub const QUOTA_WARN_THRESHOLD_PERCENTAGE: u64 = 80;
 
-// warning is already issued at QUOTA_WARN_THRESHOLD_PERCENTAGE,
-// this threshold only makes the quota icon "red".
-pub const QUOTA_ERROR_THRESHOLD_PERCENTAGE: u64 = 99;
+// warning again after this usage percentage is reached,
+// quota icon is "red".
+pub const QUOTA_ERROR_THRESHOLD_PERCENTAGE: u64 = 95;
+
+/// if quota is below this value (again),
+/// QuotaExceeding is cleared.
+/// This value should be a bit below QUOTA_WARN_THRESHOLD_PERCENTAGE to
+/// avoid jittering and lots of warnings when quota is exactly at the warning threshold.
+///
+/// We do not repeat warnings on a daily base or so as some provider
+/// providers report bad values and we would then spam the user.
+pub const QUOTA_ALLCLEAR_PERCENTAGE: u64 = 75;
 
 // if recent quota is older,
 // it is re-fetched on dc_get_connectivity_html()
@@ -28,7 +43,7 @@ pub struct QuotaInfo {
     /// set to `Err()` if the provider does not support quota or on other errors,
     /// set to `Ok()` for valid quota information.
     /// Updated by `Action::UpdateRecentQuota`
-    pub(crate) recent: Result<IndexMap<String, Vec<QuotaResource>>>,
+    pub(crate) recent: Result<BTreeMap<String, Vec<QuotaResource>>>,
 
     /// Timestamp when structure was modified.
     pub(crate) modified: i64,
@@ -37,8 +52,8 @@ pub struct QuotaInfo {
 async fn get_unique_quota_roots_and_usage(
     folders: Vec<String>,
     imap: &mut Imap,
-) -> Result<IndexMap<String, Vec<QuotaResource>>> {
-    let mut unique_quota_roots: IndexMap<String, Vec<QuotaResource>> = IndexMap::new();
+) -> Result<BTreeMap<String, Vec<QuotaResource>>> {
+    let mut unique_quota_roots: BTreeMap<String, Vec<QuotaResource>> = BTreeMap::new();
     for folder in folders {
         let (quota_roots, quotas) = &imap.get_quota_roots(&folder).await?;
         // if there are new quota roots found in this imap folder, add them to the list
@@ -54,47 +69,143 @@ async fn get_unique_quota_roots_and_usage(
                 // messages could be recieved and so the usage could have been changed
                 *unique_quota_roots
                     .entry(quota_root_name.clone())
-                    .or_insert(vec![]) = quota.resources;
+                    .or_insert_with(Vec::new) = quota.resources;
             }
         }
     }
     Ok(unique_quota_roots)
 }
 
+fn get_highest_usage<'t>(
+    unique_quota_roots: &'t BTreeMap<String, Vec<QuotaResource>>,
+) -> Result<(u64, &'t String, &QuotaResource)> {
+    let mut highest: Option<(u64, &'t String, &QuotaResource)> = None;
+    for (name, resources) in unique_quota_roots {
+        for r in resources {
+            let usage_percent = r.get_usage_percentage();
+            match highest {
+                None => {
+                    highest = Some((usage_percent, name, r));
+                }
+                Some((up, ..)) => {
+                    if up <= usage_percent {
+                        highest = Some((usage_percent, name, r));
+                    }
+                }
+            };
+        }
+    }
+
+    highest.ok_or_else(|| anyhow!("no quota_resource found, this is unexpected"))
+}
+
+/// Checks if a quota warning is needed.
+pub fn needs_quota_warning(curr_percentage: u64, warned_at_percentage: u64) -> bool {
+    (curr_percentage >= QUOTA_WARN_THRESHOLD_PERCENTAGE
+        && warned_at_percentage < QUOTA_WARN_THRESHOLD_PERCENTAGE)
+        || (curr_percentage >= QUOTA_ERROR_THRESHOLD_PERCENTAGE
+            && warned_at_percentage < QUOTA_ERROR_THRESHOLD_PERCENTAGE)
+}
+
 impl Context {
     // Adds a job to update `quota.recent`
-    pub(crate) async fn schedule_quota_update(&self) {
-        job::kill_action(self, Action::UpdateRecentQuota).await;
-        job::add(
-            self,
-            job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0),
-        )
-        .await;
+    pub(crate) async fn schedule_quota_update(&self) -> Result<()> {
+        if !job::action_exists(self, Action::UpdateRecentQuota).await? {
+            job::add(
+                self,
+                job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0),
+            )
+            .await?;
+        }
+        Ok(())
     }
 
     /// Updates `quota.recent`, sets `quota.modified` to the current time
     /// and emits an event to let the UIs update connectivity view.
     ///
+    /// Moreover, once each time quota gets larger than `QUOTA_WARN_THRESHOLD_PERCENTAGE`,
+    /// a device message is added.
+    /// As the message is added only once, the user is not spammed
+    /// in case for some providers the quota is always at ~100%
+    /// and new space is allocated as needed.
+    ///
     /// Called in response to `Action::UpdateRecentQuota`.
-    pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Status {
+    pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<Status> {
         if let Err(err) = imap.prepare(self).await {
             warn!(self, "could not connect: {:?}", err);
-            return Status::RetryNow;
+            return Ok(Status::RetryNow);
         }
 
         let quota = if imap.can_check_quota() {
-            let folders = get_watched_folders(self).await;
+            let folders = get_watched_folders(self).await?;
             get_unique_quota_roots_and_usage(folders, imap).await
         } else {
-            Err(anyhow!("Quota not supported by your provider."))
+            Err(anyhow!(stock_str::not_supported_by_provider(self).await))
         };
 
+        if let Ok(quota) = &quota {
+            match get_highest_usage(quota) {
+                Ok((highest, _, _)) => {
+                    if needs_quota_warning(
+                        highest,
+                        self.get_config_int(Config::QuotaExceeding).await? as u64,
+                    ) {
+                        self.set_config(Config::QuotaExceeding, Some(&highest.to_string()))
+                            .await?;
+                        let mut msg = Message::new(Viewtype::Text);
+                        msg.text = Some(stock_str::quota_exceeding(self, highest).await);
+                        add_device_msg_with_importance(self, None, Some(&mut msg), true).await?;
+                    } else if highest <= QUOTA_ALLCLEAR_PERCENTAGE {
+                        self.set_config(Config::QuotaExceeding, None).await?;
+                    }
+                }
+                Err(err) => warn!(self, "cannot get highest quota usage: {:?}", err),
+            }
+        }
+
         *self.quota.write().await = Some(QuotaInfo {
             recent: quota,
             modified: time(),
         });
 
         self.emit_event(EventType::ConnectivityChanged);
-        Status::Finished(Ok(()))
+        Ok(Status::Finished(Ok(())))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::quota::{
+        QUOTA_ALLCLEAR_PERCENTAGE, QUOTA_ERROR_THRESHOLD_PERCENTAGE,
+        QUOTA_WARN_THRESHOLD_PERCENTAGE,
+    };
+
+    #[async_std::test]
+    async fn test_needs_quota_warning() -> Result<()> {
+        assert!(!needs_quota_warning(0, 0));
+        assert!(!needs_quota_warning(10, 0));
+        assert!(!needs_quota_warning(70, 0));
+        assert!(!needs_quota_warning(75, 0));
+        assert!(!needs_quota_warning(79, 0));
+        assert!(needs_quota_warning(80, 0));
+        assert!(needs_quota_warning(81, 0));
+        assert!(!needs_quota_warning(85, 80));
+        assert!(!needs_quota_warning(85, 81));
+        assert!(needs_quota_warning(95, 82));
+        assert!(!needs_quota_warning(97, 95));
+        assert!(!needs_quota_warning(97, 96));
+        assert!(!needs_quota_warning(1000, 96));
+        Ok(())
+    }
+
+    #[allow(clippy::assertions_on_constants)]
+    #[async_std::test]
+    async fn test_quota_thresholds() -> anyhow::Result<()> {
+        assert!(QUOTA_ALLCLEAR_PERCENTAGE > 50);
+        assert!(QUOTA_ALLCLEAR_PERCENTAGE < QUOTA_WARN_THRESHOLD_PERCENTAGE);
+        assert!(QUOTA_WARN_THRESHOLD_PERCENTAGE < QUOTA_ERROR_THRESHOLD_PERCENTAGE);
+        assert!(QUOTA_ERROR_THRESHOLD_PERCENTAGE < 100);
+        Ok(())
     }
 }

src/scheduler.rs

@@ -10,7 +10,6 @@ use crate::context::Context;
 use crate::dc_tools::maybe_add_time_based_warnings;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
-use crate::message::MsgId;
 use crate::smtp::Smtp;
 
 use self::connectivity::ConnectivityStore;
@@ -82,7 +81,15 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         let mut jobs_loaded = 0;
         let mut info = InterruptInfo::default();
         loop {
-            match job::load_next(&ctx, Thread::Imap, &info).await {
+            let job = match job::load_next(&ctx, Thread::Imap, &info).await {
+                Err(err) => {
+                    error!(ctx, "Failed loading job from the database: {:#}.", err);
+                    None
+                }
+                Ok(job) => job,
+            };
+
+            match job {
                 Some(job) if jobs_loaded <= 20 => {
                     jobs_loaded += 1;
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
@@ -91,41 +98,15 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                 Some(job) => {
                     // Let the fetch run, but return back to the job afterwards.
                     jobs_loaded = 0;
-                    if ctx
-                        .get_config_bool(Config::InboxWatch)
-                        .await
-                        .unwrap_or_default()
-                    {
-                        info!(ctx, "postponing imap-job {} to run fetch...", job);
-                        fetch(&ctx, &mut connection).await;
-                    }
+                    info!(ctx, "postponing imap-job {} to run fetch...", job);
+                    fetch(&ctx, &mut connection).await;
                 }
                 None => {
                     jobs_loaded = 0;
 
-                    // Expunge folder if needed, e.g. if some jobs have
-                    // deleted messages on the server.
-                    if let Err(err) = connection.maybe_close_folder(&ctx).await {
-                        warn!(ctx, "failed to close folder: {:?}", err);
-                    }
-
                     maybe_add_time_based_warnings(&ctx).await;
 
-                    info = if ctx
-                        .get_config_bool(Config::InboxWatch)
-                        .await
-                        .unwrap_or_default()
-                    {
-                        fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await
-                    } else {
-                        if let Err(err) = connection.scan_folders(&ctx).await {
-                            warn!(ctx, "{}", err);
-                            connection.connectivity.set_err(&ctx, err).await;
-                        } else {
-                            connection.connectivity.set_not_configured(&ctx).await;
-                        }
-                        connection.fake_idle(&ctx, None).await
-                    };
+                    info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await;
                 }
             }
         }
@@ -153,7 +134,7 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
             }
 
             // fetch
-            if let Err(err) = connection.fetch(ctx, &watch_folder).await {
+            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
                 connection.trigger_reconnect(ctx).await;
                 warn!(ctx, "{:#}", err);
             }
@@ -179,13 +160,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 return connection.fake_idle(ctx, Some(watch_folder)).await;
             }
 
-            // fetch
-            if let Err(err) = connection.fetch(ctx, &watch_folder).await {
-                connection.trigger_reconnect(ctx).await;
-                warn!(ctx, "{:#}", err);
-                return InterruptInfo::new(false, None);
-            }
-
+            // Scan other folders before fetching from watched folder. This may result in the
+            // messages being moved into the watched folder, for example from the Spam folder to
+            // the Inbox folder.
             if folder == Config::ConfiguredInboxFolder {
                 // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
                 if let Err(err) = connection.scan_folders(ctx).await {
@@ -195,6 +172,13 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 }
             }
 
+            // fetch
+            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
+                connection.trigger_reconnect(ctx).await;
+                warn!(ctx, "{:#}", err);
+                return InterruptInfo::new(false);
+            }
+
             connection.connectivity.set_connected(ctx).await;
 
             // idle
@@ -204,7 +188,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                     Err(err) => {
                         connection.trigger_reconnect(ctx).await;
                         warn!(ctx, "{}", err);
-                        InterruptInfo::new(false, None)
+                        InterruptInfo::new(false)
                     }
                 }
             } else {
@@ -289,7 +273,11 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
 
         let mut interrupt_info = Default::default();
         loop {
-            match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info)
+                .await
+                .ok()
+                .flatten()
+            {
                 Some(job) => {
                     info!(ctx, "executing smtp job");
                     job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;
@@ -345,7 +333,7 @@ impl Scheduler {
             }))
         };
 
-        if ctx.get_config_bool(Config::MvboxWatch).await? {
+        if ctx.get_config_bool(Config::MvboxMove).await? {
             let ctx = ctx.clone();
             mvbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
@@ -429,10 +417,10 @@ impl Scheduler {
             return;
         }
 
-        self.interrupt_inbox(InterruptInfo::new(true, None))
-            .join(self.interrupt_mvbox(InterruptInfo::new(true, None)))
-            .join(self.interrupt_sentbox(InterruptInfo::new(true, None)))
-            .join(self.interrupt_smtp(InterruptInfo::new(true, None)))
+        self.interrupt_inbox(InterruptInfo::new(true))
+            .join(self.interrupt_mvbox(InterruptInfo::new(true)))
+            .join(self.interrupt_sentbox(InterruptInfo::new(true)))
+            .join(self.interrupt_smtp(InterruptInfo::new(true)))
             .await;
     }
 
@@ -441,10 +429,10 @@ impl Scheduler {
             return;
         }
 
-        self.interrupt_inbox(InterruptInfo::new(false, None))
-            .join(self.interrupt_mvbox(InterruptInfo::new(false, None)))
-            .join(self.interrupt_sentbox(InterruptInfo::new(false, None)))
-            .join(self.interrupt_smtp(InterruptInfo::new(false, None)))
+        self.interrupt_inbox(InterruptInfo::new(false))
+            .join(self.interrupt_mvbox(InterruptInfo::new(false)))
+            .join(self.interrupt_sentbox(InterruptInfo::new(false)))
+            .join(self.interrupt_smtp(InterruptInfo::new(false)))
             .await;
     }
 
@@ -674,14 +662,10 @@ struct ImapConnectionHandlers {
 #[derive(Default, Debug)]
 pub struct InterruptInfo {
     pub probe_network: bool,
-    pub msg_id: Option<MsgId>,
 }
 
 impl InterruptInfo {
-    pub fn new(probe_network: bool, msg_id: Option<MsgId>) -> Self {
-        Self {
-            probe_network,
-            msg_id,
-        }
+    pub fn new(probe_network: bool) -> Self {
+        Self { probe_network }
     }
 }