scheduler: rate limit IMAP loops

Avoid hitting the server with repeated requests or reconnections if some operation, such as FETCH, keeps failing.

.github/mergeable.yml

@@ -1,26 +0,0 @@
-version: 2
-mergeable:
-  - when: pull_request.*
-    name: "Changelog check"
-    validate:
-      - do: or
-        validate:
-        - do: description
-          must_include:
-            regex: '#skip-changelog'
-        - do: and
-          validate:
-            - do: dependent
-              changed:
-                file: 'src/**'
-                required: ['CHANGELOG.md']
-            - do: dependent
-              changed:
-                file: 'deltachat-ffi/**'
-                required: ['CHANGELOG.md']
-    fail:
-      - do: checks
-        status: 'action_required' 
-        payload:
-          title: Changlog might need an update
-          summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -8,9 +8,6 @@ on:
       - staging
       - trying
 
-env:
-  RUSTFLAGS: -Dwarnings
-
 jobs:
 
   fmt:
@@ -24,8 +21,6 @@ jobs:
           toolchain: stable
           override: true
       - run: rustup component add rustfmt
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
       - uses: actions-rs/cargo@v1
         with:
           command: fmt
@@ -40,8 +35,6 @@ jobs:
             toolchain: stable
             components: clippy
             override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -83,13 +76,14 @@ jobs:
             rust: 1.54.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.51.0
+          # Minimum Supported Rust Version = 1.48.0
+          # This is the Debian "bullseye" release version of Rust.
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.51.0
+            rust: 1.48.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:
@@ -101,14 +95,31 @@ jobs:
         toolchain: ${{ matrix.rust }}
         override: true
 
-    - name: Cache rust cargo artifacts
-      uses: swatinem/rust-cache@v1
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
 
     - name: check
       uses: actions-rs/cargo@v1
+      env:
+        RUSTFLAGS: -D warnings
       with:
         command: check
-        args: --all --bins --examples --tests --features repl --benches
+        args: --all --bins --examples --tests --features repl
 
     - name: tests
       uses: actions-rs/cargo@v1
@@ -140,4 +151,4 @@ jobs:
         DCC_RS_TARGET: debug
         DCC_RS_DEV: ${{ github.workspace }}
       working-directory: python
-      run: tox -e lint,mypy,doc,py3
+      run: tox -e lint,doc,py3

.gitignore

@@ -28,4 +28,3 @@ deltachat-ffi/xml
 .rsynclist
 
 coverage/
-.DS_Store

CHANGELOG.md

@@ -3,215 +3,14 @@
 ## Unreleased
 
 ### Changes
-- don't watch Sent folder by default #3025
-- use webxdc app name in chatlist/quotes/replies etc. #3027
-- refactorings #3023
-- remove direct dependency on `byteorder` crate #3031
-- make it possible to cancel message sending by removing the message #3034,
-  this was previosuly removed in 1.71.0 #2939
-- always skip Seen flag synchronization when there are no updates #3039
-
-### Fixes
-- fix splitting off text from webxdc messages #3032
-- call slow `delete_expired_imap_messages()` less often #3037
-- make synchronization of Seen status more robust in case unsolicited FETCH
-  result without UID is returned #3022
-
-
-## 1.72.0
-
-### Fixes
-- run migrations on backup import #3006
-
-
-## 1.71.0
-
-### API Changes
-- added APIs to handle database passwords: `dc_context_new_closed()`, `dc_context_open()`,
-  `dc_context_is_open()` and `dc_accounts_add_closed_account()` #2956 #2972
-- use second parameter of `dc_imex` to provide backup passphrase #2980
-- added `DC_MSG_WEBXDC`, `dc_send_webxdc_status_update()`,
-  `dc_get_webxdc_status_updates()`, `dc_msg_get_webxdc_blob()`, `dc_msg_get_webxdc_info()`
-  and `DC_EVENT_WEBXDC_STATUS_UPDATE` #2826 #2971 #2975 #2977 #2979 #2993 #2994 #2998 #3001 #3003
-- added `dc_msg_get_parent()` #2984
-- added `dc_msg_force_plaintext()` API for bots #2847
-- allow removing quotes on drafts `dc_msg_set_quote(msg, NULL)` #2950
-- removed `mvbox_watch` option; watching is enabled when `mvbox_move` is enabled #2906
-- removed `inbox_watch` option #2922
-- deprecated `os_name` in `dc_context_new()`, pass `NULL` or an empty string #2956
-
-### Changes
-- start making it possible to write to mailing lists #2736
-- add `hop_info` to `dc_get_info()` #2751 #2914 #2923
-- add information about whether the database is encrypted or not to `dc_get_info()` #3000
-- selfstatus now defaults to empty #2951 #2960
-- validate detached cryptographic signatures as used eg. by Thunderbird #2865
-- do not change the draft's `msg_id` on updates and sending #2887
-- add `imap` table to keep track of message UIDs #2909 #2938
-- replace `SendMsgToSmtp` jobs which stored outgoing messages in blobdir with `smtp` SQL table #2939 #2996
-- sql: enable `auto_vacuum=INCREMENTAL` #2931
-- sql: build rusqlite with sqlcipher #2934
-- synchronize Seen status across devices #2942
-- `dc_preconfigure_keypair` now takes ascii armored keys instead of base64 #2862
-- put removed member in Bcc instead of To in the message about removal #2864
-- improve group updates #2889
-- re-write the blob filename creation loop #2981
-- update provider database (11 Jan 2022) #2959
-- python: allow timeout for internal configure tracker API #2967
-- python: remove API deprecated in Python 3.10 #2907
-- refactorings #2932 #2957 #2947
-- improve tests #2863 #2866 #2881 #2908 #2918 #2901 #2973
-- improve documentation #2880 #2886 #2895
-- improve ci #2919 #2926 #2969 #2999
-
-### Fixes
-- fix leaving groups #2929
-- fix unread count #2861
-- make `add_parts()` not early-exit #2879
-- recognize MS Exchange read receipts as read receipts #2890
-- create parent directory if creating a new file fails #2978
-- save "configured" flag later #2974
-- improve log #2928
-- `dc_receive_imf`: don't fail on invalid address in the To field #2940
-
-
-## 1.70.0
-
-### Fixes
-- fix: do not abort Param parsing on unknown keys #2856
-- fix: execute `Chat-Group-Member-Removed:` even when arriving disordered #2857
-
-
-## 1.69.0
-
-### Fixes
-- fix group-related system messages in multi-device setups #2848
-- fix "Google Workspace" (former "G Suite") issues related to bad resolvers #2852
-
-
-## 1.68.0
-
-### Fixes
-- fix chat assignment when forwarding #2843
-- fix layout issues with the generated QR code svg #2842
-
-
-## 1.67.0
-
-### API changes
-- `dc_get_securejoin_qr_svg(chat_id)` added #2815
-- added stock-strings `DC_STR_SETUP_CONTACT_QR_DESC` and `DC_STR_SECURE_JOIN_GROUP_QR_DESC`
-
-
-## 1.66.0
-
-### API changes
-- `dc_contact_get_last_seen()` added #2823
-- python: `Contact.last_seen` added #2823
-- removed `DC_STR_NEWGROUPDRAFT`, we don't set draft after creating group anymore #2805
-
-### Changes
-- python: add cutil.from_optional_dc_charpointer() #2824
-- refactorings #2807 #2822 #2825
-
-
-## 1.65.0
-
-### Changes
-- python: add mypy support and some type hints #2809
-
-### Fixes
-- do not disable ephemeral timer when downloading a message partially #2811
-- apply existing ephemeral timer also to partially downloaded messages;
-  after full download, the ephemeral timer starts over #2811
-- replace user-visible error on verification failure with warning;
-  the error is logged to the corresponding chat anyway #2808
-
-
-## 1.64.0
-
-### Fixes
-- add 'waiting for being added to the group' only for group-joins,
-  not for setup-contact #2797
-- prioritize In-Reply-To: and References: headers over group IDs when assigning
-  messages to chats to fix incorrect assignment of Delta Chat replies to
-  classic email threads #2795
-
-
-## 1.63.0
-
-### API changes
-- `dc_get_last_error()` added #2788
-
-### Changes
-- Optimize Autocrypt gossip #2743
-
-### Fixes
-- fix permanently hiding of one-to-one chats after secure-join #2791
-
-
-## 1.62.0
-
-### API Changes
-- `dc_join_securejoin()` now always returns immediately;
-  the returned chat may not allow sending (`dc_chat_can_send()` returns false)
-  which may change as usual on `DC_EVENT_CHAT_MODIFIED` #2508 #2767
-- introduce multi-device-sync-messages;
-  as older cores display them as files in self-chat,
-  they are currently only sent if config option `send_sync_msgs` is set #2669
-- add `DC_EVENT_SELFAVATAR_CHANGED` #2742
-
-### Changes
-- use system DNS instead of google for MX queries #2780
-- improve error logging #2758
-- improve tests #2764 #2781
-- improve ci #2770
-- refactorings #2677 #2728 #2740 #2729 #2766 #2778
-
-### Fixes
-- add Let's Encrypt certificate to core as it may be missing older devices #2752
-- prioritize certificate setting from user over the one from provider-db #2749
-- fix "QR process failed" error #2725
-- do not update quota in endless loop #2726
-
-
-## 1.61.0
-
-### API Changes
-- download-on-demand added: `dc_msg_get_download_state()`, `dc_download_full_msg()`
-  and `download_limit` config option #2631 #2696
-- `dc_create_broadcast_list()` and chat type `DC_CHAT_TYPE_BROADCAST` added #2707 #2722
-- allow ui-specific configs using `ui.`-prefix in key (`dc_set_config(context, "ui.*", value)`) #2672
-- new strings from `DC_STR_PARTIAL_DOWNLOAD_MSG_BODY`
-  to `DC_STR_PART_OF_TOTAL_USED` #2631 #2694 #2707 #2723
-- emit warnings and errors from account manager with account-id 0 #2712
-
-### Changes
-- notify about incoming contact requests #2690
-- messages are marked as read on first read receipt #2699
-- quota warning reappears after import, rewarning at 95% #2702
-- lock strict TLS if certificate checks are automatic #2711
-- always check certificates strictly when connecting over SOCKS5 in Automatic mode #2657
 - `Accounts` is not cloneable anymore #2654 #2658
+- always check certificates strictly when connecting over SOCKS5 in Automatic mode #2657
 - update chat/contact data only when there was no newer update #2642
-- better detection of mailing list names #2665 #2685
-- log all decisions when applying ephemeral timer to chats #2679
-- connectivity view now translatable #2694 #2723
-- improve Doxygen documentation #2647 #2668 #2684 #2688 #2705
-- refactorings #2656 #2659 #2677 #2673 #2678 #2675 #2663 #2692 #2706
-- update provider database #2618
+- improve Doxygen documentation style #2647
 
 ### Fixes
-- ephemeral timer rollback protection #2693 #2709
-- recreate configured folders if they are deleted #2691
 - ignore MDNs sent to self #2674
-- recognize NDNs that put headers into "message/global-headers" part #2598
-- avoid `dc_get_contacts()` returning duplicate contact ids #2591
-- do not leak group names on forwarding messages #2719
-- in case of smtp-errors, iterate over all addresses to fix ipv6/v4 problems #2720
 - fix pkg-config file #2660
-- fix "QR process failed" error #2725
-
 
 ## 1.60.0
 
@@ -226,7 +25,6 @@
 ### Fixes
 - keep event emitter from closing when there are no accounts #2636
 
-
 ## 1.59.0
 
 ### Added
@@ -251,6 +49,7 @@
 ## 1.58.0
 
 ### Fixes
+
 - move WAL file together with database
   and avoid using data if the database was not closed correctly before #2583
 

Cargo.toml

@@ -1,10 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.72.0"
+version = "1.60.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
-resolver = "2"
 
 [profile.dev]
 debug = 0
@@ -12,84 +11,81 @@ debug = 0
 [profile.release]
 lto = true
 
-[patch.crates-io]
-rusqlite = { git = "https://github.com/rusqlite/rusqlite", branch="master" }
-
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
 ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1"
+anyhow = "1.0.43"
 async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3" }
+async-native-tls = { version = "0.3.3" }
 async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
-async-std-resolver = "0.20"
-async-std = { version = "1", features = ["unstable"] }
-async-tar = { version = "0.4", default-features=false }
-async-trait = "0.1"
-backtrace = "0.3"
+async-std-resolver = "0.20.3"
+async-std = { version = "~1.10.0", features = ["unstable"] }
+async-tar = "0.3.0"
+async-trait = "0.1.51"
+backtrace = "0.3.59"
 base64 = "0.13"
-bitflags = "1.3"
-chrono = "0.4"
-dirs = { version = "4", optional=true }
+bitflags = "1.3.2"
+byteorder = "1.3.1"
+chrono = "0.4.6"
+dirs = { version = "3.0.2", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1"
-futures = "0.3"
+escaper = "0.1.1"
+futures = "0.3.17"
+governor = "0.3.2"
 hex = "0.4.0"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-imap-proto = "0.14.3"
+indexmap = "1.7.0"
+itertools = "0.10.1"
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2"
+libc = "0.2.101"
 log = {version = "0.4.8", optional = true }
-mailparse = "0.13"
-native-tls = "0.2"
-num_cpus = "1.13"
-num-derive = "0.3"
-num-traits = "0.2"
-once_cell = "1.9.0"
+mailparse = "0.13.6"
+native-tls = "0.2.8"
+num_cpus = "1.13.0"
+num-derive = "0.3.0"
+num-traits = "0.2.6"
+once_cell = "1.8.0"
 percent-encoding = "2.0"
-pgp = { version = "0.7", default-features = false }
-pretty_env_logger = { version = "0.4", optional = true }
-quick-xml = "0.22"
-r2d2 = "0.8"
-r2d2_sqlite = "0.19"
-rand = "0.7"
-regex = "1.5"
-rusqlite = { version = "0.26", features = ["sqlcipher"] }
-rust-hsluv = "0.1"
-rustyline = { version = "9", optional = true }
-sanitize-filename = "0.3"
+pgp = { version = "0.7.2", default-features = false }
+pretty_env_logger = { version = "0.4.0", optional = true }
+quick-xml = "0.22.0"
+r2d2 = "0.8.9"
+r2d2_sqlite = "0.18.0"
+rand = "0.7.0"
+regex = "1.4.6"
+rusqlite = "0.25"
+rust-hsluv = "0.1.4"
+rustyline = { version = "8.2.0", optional = true }
+sanitize-filename = "0.3.0"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.10"
-sha2 = "0.10"
-smallvec = "1"
-stop-token = "0.7"
-strum = "0.23"
-strum_macros = "0.23"
-surf = { version = "2.3", default-features = false, features = ["h1-client"] }
-thiserror = "1"
-toml = "0.5"
-url = "2"
+sha-1 = "0.9.8"
+sha2 = "0.9.8"
+smallvec = "1.0.0"
+stop-token = "0.2.0"
+strum = "0.21.0"
+strum_macros = "0.21.1"
+surf = { version = "2.3.1", default-features = false, features = ["h1-client"] }
+thiserror = "1.0.29"
+toml = "0.5.6"
+url = "2.2.2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
-fast-socks5 = "0.4"
-humansize = "1"
-qrcodegen = "1.7.0"
-tagger = "4.2.1"
-textwrap = "0.14.2"
-zip = { version = "0.5.13", default-features = false, features = ["deflate"] }
+fast-socks5 = "0.4.3"
+humansize = "1.1.1"
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1", features = ["unstable", "attributes"] }
+async-std = { version = "1.10.0", features = ["unstable", "attributes"] }
 criterion = "0.3"
-futures-lite = "1.12"
-log = "0.4"
-pretty_env_logger = "0.4"
-proptest = { version = "1", default-features = false, features = ["std"] }
-tempfile = "3"
+futures-lite = "1.12.0"
+log = "0.4.11"
+pretty_assertions = "0.7.2"
+pretty_env_logger = "0.4.0"
+proptest = "1.0"
+tempfile = "3.0"
 
 [workspace]
 members = [
@@ -124,5 +120,5 @@ harness = false
 default = ["vendored"]
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
 nightly = ["pgp/nightly"]

README.md

@@ -125,11 +125,11 @@ $ cargo test -- --ignored
 
 Language bindings are available for:
 
-- **C** \[[📂 source](./deltachat-ffi) | [📚 docs](https://c.delta.chat)\]
-- **Node.js** \[[📂 source](https://github.com/deltachat/deltachat-node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
-- **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
-- **Go** \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
-- **Free Pascal** \[[📂 source](https://github.com/deltachat/deltachat-fp/)\]
+- [C](https://c.delta.chat)
+- [Node.js](https://www.npmjs.com/package/deltachat-node)
+- [Python](https://py.delta.chat)
+- [Go](https://github.com/deltachat/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library

assets/icon-broadcast.svg

@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:xlink="http://www.w3.org/1999/xlink"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   enable-background="new 0 0 128 128"
-   viewBox="0 0 60 60"
-   version="1.1"
-   id="svg878"
-   sodipodi:docname="icon-broadcast.svg"
-   width="60"
-   height="60"
-   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
-   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-broadcast.png"
-   inkscape:export-xdpi="409.60001"
-   inkscape:export-ydpi="409.60001">
-  <metadata
-     id="metadata884">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title />
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <defs
-     id="defs882" />
-  <sodipodi:namedview
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="1329"
-     inkscape:window-height="847"
-     id="namedview880"
-     showgrid="false"
-     inkscape:zoom="5.21875"
-     inkscape:cx="36.598802"
-     inkscape:cy="32.191617"
-     inkscape:window-x="111"
-     inkscape:window-y="205"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg878"
-     inkscape:document-rotation="0" />
-  <radialGradient
-     id="c"
-     cx="65.25"
-     cy="89"
-     r="26.440001"
-     gradientTransform="matrix(0.77611266,0.11996647,-0.18999676,1.2286617,-11.305867,-60.065999)"
-     gradientUnits="userSpaceOnUse">
-    <stop
-       stop-color="#FFC107"
-       offset="0"
-       id="stop833" />
-    <stop
-       stop-color="#FFBD06"
-       offset=".3502"
-       id="stop835" />
-    <stop
-       stop-color="#FFB104"
-       offset=".6938"
-       id="stop837" />
-    <stop
-       stop-color="#FFA000"
-       offset="1"
-       id="stop839" />
-  </radialGradient>
-  <radialGradient
-     id="b"
-     cx="52.5"
-     cy="19.75"
-     r="92.975998"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)">
-    <stop
-       stop-color="#EF5350"
-       offset="0"
-       id="stop848" />
-    <stop
-       stop-color="#EB4F4C"
-       offset=".246"
-       id="stop850" />
-    <stop
-       stop-color="#E04341"
-       offset=".4878"
-       id="stop852" />
-    <stop
-       stop-color="#CD302F"
-       offset=".7272"
-       id="stop854" />
-    <stop
-       stop-color="#C62828"
-       offset=".8004"
-       id="stop856" />
-    <stop
-       stop-color="#C62828"
-       offset="1"
-       id="stop858" />
-  </radialGradient>
-  <radialGradient
-     id="a"
-     cx="16.979"
-     cy="92"
-     r="24.165001"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)"
-     xlink:href="#b">
-    <stop
-       stop-color="#E0E0E0"
-       offset="0"
-       id="stop863" />
-    <stop
-       stop-color="#CFCFCF"
-       offset=".3112"
-       id="stop865" />
-    <stop
-       stop-color="#A4A4A4"
-       offset=".9228"
-       id="stop867" />
-    <stop
-       stop-color="#9E9E9E"
-       offset="1"
-       id="stop869" />
-  </radialGradient>
-  <rect
-     y="0"
-     x="0"
-     height="60"
-     width="60"
-     id="rect1420"
-     style="fill:#7cc0bc;fill-opacity:1;stroke:none;stroke-width:1.29077" />
-  <path
-     id="path872"
-     style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.336872;stroke-opacity:1"
-     d="m 8.6780027,35.573064 0.032831,-11.910176 c 0.00138,-0.476406 0.4881282,-0.794259 0.9235226,-0.604877 l 4.1144877,2.345752 -0.02386,8.656315 -4.1268029,2.122946 C 9.1617452,36.370003 8.6766889,36.049472 8.6780027,35.573064 Z m 5.0469633,-1.508222 0.02386,-8.656314 31.145424,-9.537653 c 0.841472,-0.219211 1.65915,0.41667 1.656755,1.283728 l -0.06929,25.139995 c -0.0024,0.867062 -0.825942,1.500799 -1.663803,1.274581 z m 3.8042,6.892234 C 16.681121,40.104348 16.315444,38.819414 16.69043,37.591308 l 2.252234,-7.347193 c 0.2644,-0.861571 0.845185,-1.567441 1.641953,-1.989251 0.796769,-0.421808 1.706956,-0.509819 2.568531,-0.245419 l 7.263888,2.225804 c 1.775518,0.543235 2.780299,2.432591 2.232297,4.208094 L 30.3971,41.790532 c -0.545627,1.777887 -2.432591,2.780297 -4.208095,2.232298 l -7.263891,-2.225804 c -0.545033,-0.165864 -1.01825,-0.460162 -1.395948,-0.83995 z m 12.377693,-7.976728 c -0.07601,-0.07642 -0.17114,-0.133864 -0.280621,-0.167516 l -7.263891,-2.225803 c -0.233244,-0.07209 -0.421626,0.0013 -0.512275,0.04861 -0.09064,0.0474 -0.25772,0.166033 -0.327435,0.396899 l -2.252234,7.347191 c -0.108166,0.354628 0.09088,0.731541 0.447888,0.842099 l 7.263891,2.225802 c 0.354626,0.108174 0.731539,-0.09088 0.842099,-0.447888 l 2.249845,-7.344814 c 0.07453,-0.245145 0.0014,-0.504991 -0.167267,-0.67458 z" />
-</svg>

assets/icon-webxdc.svg

@@ -1,181 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   width="80mm"
-   height="297mm"
-   viewBox="0 0 80 297"
-   version="1.1"
-   id="svg71"
-   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
-   sodipodi:docname="icon-webxdc.svg"
-   inkscape:export-filename="C:\Users\user\OneDrive - BFW-Leipzig\Documents\LogoDC\finalohnerand.png"
-   inkscape:export-xdpi="96"
-   inkscape:export-ydpi="96">
-  <metadata
-     id="metadata856">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title></dc:title>
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <sodipodi:namedview
-     id="namedview73"
-     pagecolor="#767676"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0"
-     inkscape:pagecheckerboard="true"
-     inkscape:document-units="mm"
-     showgrid="false"
-     showborder="false"
-     inkscape:snap-bbox="true"
-     inkscape:bbox-paths="true"
-     inkscape:bbox-nodes="true"
-     inkscape:snap-bbox-edge-midpoints="true"
-     inkscape:snap-bbox-midpoints="true"
-     inkscape:object-paths="true"
-     inkscape:snap-intersection-paths="true"
-     inkscape:zoom="1.4142136"
-     inkscape:cx="-90.271136"
-     inkscape:cy="-1233.1209"
-     inkscape:window-width="1864"
-     inkscape:window-height="1027"
-     inkscape:window-x="56"
-     inkscape:window-y="25"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="layer1"
-     inkscape:snap-global="false"
-     showguides="false"
-     inkscape:guide-bbox="true"
-     inkscape:document-rotation="0"
-     units="px">
-    <sodipodi:guide
-       position="-154.76097,641.11689"
-       orientation="0,-1"
-       id="guide21118" />
-    <sodipodi:guide
-       position="-60.286487,633.36619"
-       orientation="0,-1"
-       id="guide21120" />
-  </sodipodi:namedview>
-  <defs
-     id="defs68">
-    <linearGradient
-       id="linearGradient4375">
-      <stop
-         style="stop-color:#364e59;stop-opacity:1;"
-         offset="0"
-         id="stop4377" />
-      <stop
-         style="stop-color:#364e59;stop-opacity:0;"
-         offset="1"
-         id="stop4379" />
-    </linearGradient>
-  </defs>
-  <g
-     inkscape:label="Ebene 1"
-     inkscape:groupmode="layer"
-     id="layer1">
-    <rect
-       style="fill:#1a1a1a;stroke:#000000;stroke-width:0.167903"
-       id="rect880"
-       width="79.8321"
-       height="79.8321"
-       x="-64.03286"
-       y="-375.9097"
-       ry="0" />
-    <path
-       inkscape:connector-curvature="0"
-       id="path3799-2"
-       d="m -24.089585,-372.59579 c -19.986026,0.24336 -36.196903,16.666 -36.196903,36.67011 0,20.00409 16.210877,36.03233 36.196903,35.78912 19.0024236,-0.076 14.5340713,-10.6146 35.538854,-0.85693 -11.50627538,-17.97454 0.390097,-20.36737 0.658079,-35.81316 0,-20.00411 -16.2108788,-36.03235 -36.196911,-35.78914 z"
-       style="fill:#364e59;fill-opacity:1;stroke:none;stroke-width:1.93355"
-       sodipodi:nodetypes="sscccs" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -54.193871,-325.26419 Z"
-       id="path3846" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -49.397951,-326.67773 Z"
-       id="path3848" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="m -49.397951,-326.67773 v 0 0"
-       id="path3850" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.01;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
-       d="m -51.35133,-325.0334 -7.964067,5.98895 z"
-       id="path3965" />
-    <path
-       inkscape:connector-curvature="0"
-       id="path11037"
-       d="m -24.089585,-372.19891 c -19.986026,0.24156 -36.196903,16.54352 -36.196903,36.40062 0,7.86524 2.543315,15.1113 6.857155,20.97971 6.577146,8.94734 11.123515,9.77363 11.123515,9.77363 1.343237,1.78324 10.270932,4.3223 10.270932,4.3223 l 16.791727,-70.86654 -0.468369,-0.33457 c 0.458597,0.26445 0.428277,-0.27515 -8.378035,-0.27515 z"
-       style="fill:#7cc5cc;fill-opacity:1;stroke:none;stroke-width:1.92643"
-       sodipodi:nodetypes="sssccccss" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -49.944239,-310.69957 Z"
-       id="path13674" />
-    <g
-       id="g15178"
-       transform="matrix(0.79975737,0,0,0.79975737,53.088959,-63.716396)">
-      <rect
-         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-         id="rect15072"
-         width="29.897917"
-         height="6.8791666"
-         x="-334.4964"
-         y="-154.51025"
-         transform="rotate(45)" />
-      <rect
-         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-         id="rect15072-5"
-         width="29.897917"
-         height="6.8791666"
-         x="147.63107"
-         y="-334.4964"
-         transform="rotate(-45)"
-         inkscape:transform-center-x="-0.74835017"
-         inkscape:transform-center-y="0.37417525" />
-    </g>
-    <g
-       id="g22468"
-       transform="translate(3.3033974)">
-      <g
-         id="g15178-0"
-         transform="matrix(-0.79975737,0,0,0.79975737,-103.11028,-63.716404)"
-         style="fill:#7cc5cc;fill-opacity:1">
-        <rect
-           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-           id="rect15072-2"
-           width="29.897917"
-           height="6.8791666"
-           x="-334.4964"
-           y="-154.51025"
-           transform="rotate(45)" />
-        <rect
-           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-           id="rect15072-5-5"
-           width="29.897917"
-           height="6.8791666"
-           x="147.63107"
-           y="-334.4964"
-           transform="rotate(-45)"
-           inkscape:transform-center-x="-0.74835017"
-           inkscape:transform-center-y="0.37417525" />
-      </g>
-    </g>
-  </g>
-</svg>

assets/qrcode_logo_footer.svg

@@ -1,10 +0,0 @@
-<text
-     xml:space="preserve"
-     style="font-weight:bold;font-size:24.4118px;line-height:1.25;font-family:sans-serif;fill:#aaaaaa;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     x="42.325161"
-     y="23.32255"
-     id="text72398">get.delta.chat</text>
-  <path
-     id="path84310"
-     style="opacity:0.25;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     d="M 17.13769,0.00129321 C 7.6753075,0.11650893 0,7.8915283 0,17.362467 c 0,9.47094 7.6753075,17.059745 17.13769,16.944599 8.99669,-0.03598 6.880074,-5.025654 16.824785,-0.405885 -5.447648,-8.510047 0.184241,-9.642482 0.311117,-16.955289 0,-9.4709395 -7.673512,-17.0597453 -17.135895,-16.94459879 z M 17.0769,4.9986797 c 1.84214,0 3.447355,0.253959 4.815003,0.7616693 1.381603,0.5076411 2.072253,1.207862 2.072253,2.0990711 0,0.4286855 -0.167495,0.7836052 -0.50242,1.0656242 -0.334921,0.2819844 -0.724544,0.4237724 -1.171121,0.4237724 -0.641952,0 -1.396532,-0.3909376 -2.261778,-1.169353 C 19.14963,7.3898036 18.402555,6.83791 17.788507,6.5220182 17.188416,6.1950547 16.484552,6.0321266 15.675129,6.0321266 c -1.032717,0 -1.883352,0.1854523 -2.553215,0.5578447 -0.655913,0.372254 -0.98517,0.8460916 -0.98517,1.4214436 0,0.5414792 0.272815,1.0495355 0.817093,1.5233385 0.544275,0.4738026 1.946291,1.3367446 4.207097,2.5889976 2.414319,1.342419 4.117377,2.390985 5.108232,3.146807 1.004795,0.755857 1.821505,1.675853 2.449514,2.758846 0.628002,1.082993 0.942253,2.227607 0.942253,3.434674 0,2.120834 -0.929555,3.993314 -2.785656,5.617786 -1.84214,1.613228 -3.99694,2.41915 -6.467082,2.41915 -2.246845,0 -4.145607,-0.647976 -5.694677,-1.945312 -1.5490699,-1.297336 -2.3225722,-3.028063 -2.3225722,-5.194049 0,-2.087031 0.8506345,-3.83094 2.5532182,-5.229825 1.716541,-1.398884 3.824203,-2.245599 6.322256,-2.538897 -0.697774,-0.631749 -1.668763,-1.387225 -2.910816,-2.267155 -1.367648,-0.970199 -2.287914,-1.73045 -2.762402,-2.283243 -0.474491,-0.5640381 -0.711618,-1.1795944 -0.711618,-1.8451814 0,-0.9927581 0.572093,-1.7710351 1.716451,-2.3351077 1.144362,-0.5753173 2.636724,-0.8635642 4.478865,-0.8635642 z m 1.110327,10.3738083 c -4.005262,0.5302 -6.007576,2.75279 -6.007576,6.667322 0,2.01932 0.49495,3.587291 1.485805,4.704157 1.004806,1.116832 2.169696,1.675299 3.495479,1.675299 1.381602,0 2.520072,-0.535632 3.413229,-1.60738 0.893168,-1.082959 1.339187,-2.545264 1.339187,-4.384079 0,-2.662348 -1.242022,-5.013441 -3.726124,-7.055319 z" />

benches/contacts.rs

@@ -8,7 +8,9 @@ async fn address_book_benchmark(n: u32, read_count: u32) {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
 
     let book = (0..n)
         .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))

benches/create_account.rs

@@ -8,7 +8,7 @@ async fn create_accounts(n: u32) {
     let dir = tempdir().unwrap();
     let p: PathBuf = dir.path().join("accounts").into();
 
-    let mut accounts = Accounts::new(p.clone()).await.unwrap();
+    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
     for expected_id in 2..n {
         let id = accounts.add_account().await.unwrap();

benches/search_msgs.rs

@@ -6,7 +6,9 @@ use std::path::Path;
 async fn search_benchmark(path: impl AsRef<Path>) {
     let dbfile = path.as_ref();
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
 
     for _ in 0..10u32 {
         context.search_msgs(None, "hello").await.unwrap();

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.72.0"
+version = "1.60.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -17,13 +17,13 @@ crate-type = ["cdylib", "staticlib"]
 [dependencies]
 deltachat = { path = "../", default-features = false }
 libc = "0.2"
-human-panic = "1"
-num-traits = "0.2"
+human-panic = "1.0.1"
+num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1"
-anyhow = "1"
-thiserror = "1"
-rand = "0.7"
+async-std = "1.10.0"
+anyhow = "1.0.43"
+thiserror = "1.0.29"
+rand = "0.7.3"
 
 [features]
 default = ["vendored"]

deltachat-ffi/Doxyfile

@@ -583,7 +583,7 @@ SORT_MEMBERS_CTORS_1ST = NO
 # appear in their defined order.
 # The default value is: NO.
 
-SORT_GROUP_NAMES       = YES
+SORT_GROUP_NAMES       = NO
 
 # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
 # fully-qualified names, including namespaces. If set to NO, the class list will

deltachat-ffi/README.md

@@ -1,9 +1,5 @@
 # Delta Chat C Interface
 
-## Installation
-
-see `Installing libdeltachat system wide` in [../README.md](../README.md)
-
 ## Documentation
 
 To generate the C Interface documentation,

deltachat-ffi/src/lib.rs

@@ -15,7 +15,7 @@ extern crate num_traits;
 extern crate serde_json;
 
 use std::collections::BTreeMap;
-use std::convert::TryFrom;
+use std::convert::TryInto;
 use std::fmt::Write;
 use std::ops::Deref;
 use std::ptr;
@@ -25,9 +25,7 @@ use std::time::{Duration, SystemTime};
 use anyhow::Context as _;
 use async_std::sync::RwLock;
 use async_std::task::{block_on, spawn};
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use num_traits::{FromPrimitive, ToPrimitive};
-use rand::Rng;
 
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
@@ -37,12 +35,10 @@ use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
 use deltachat::stock_str::StockMessage;
-use deltachat::webxdc::StatusUpdateId;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
 
 mod dc_array;
-mod lot;
 
 mod string;
 use self::string::*;
@@ -65,7 +61,7 @@ pub type dc_context_t = Context;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_context_new(
-    _os_name: *const libc::c_char,
+    os_name: *const libc::c_char,
     dbfile: *const libc::c_char,
     blobdir: *const libc::c_char,
 ) -> *mut dc_context_t {
@@ -76,10 +72,21 @@ pub unsafe extern "C" fn dc_context_new(
         return ptr::null_mut();
     }
 
+    let os_name = if os_name.is_null() {
+        String::from("DcFFI")
+    } else {
+        to_string_lossy(os_name)
+    };
+
     let ctx = if blobdir.is_null() || *blobdir == 0 {
+        use rand::Rng;
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
-        block_on(Context::new(as_path(dbfile).to_path_buf().into(), id))
+        block_on(Context::new(
+            os_name,
+            as_path(dbfile).to_path_buf().into(),
+            id,
+        ))
     } else {
         eprintln!("blobdir can not be defined explicitly anymore");
         return ptr::null_mut();
@@ -87,63 +94,12 @@ pub unsafe extern "C" fn dc_context_new(
     match ctx {
         Ok(ctx) => Box::into_raw(Box::new(ctx)),
         Err(err) => {
-            eprintln!("failed to create context: {:#}", err);
+            eprintln!("failed to create context: {}", err);
             ptr::null_mut()
         }
     }
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *mut dc_context_t {
-    setup_panic!();
-
-    if dbfile.is_null() {
-        eprintln!("ignoring careless call to dc_context_new_closed()");
-        return ptr::null_mut();
-    }
-
-    let id = rand::thread_rng().gen();
-    match block_on(Context::new_closed(
-        as_path(dbfile).to_path_buf().into(),
-        id,
-    )) {
-        Ok(context) => Box::into_raw(Box::new(context)),
-        Err(err) => {
-            eprintln!("failed to create context: {:#}", err);
-            ptr::null_mut()
-        }
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_context_open(
-    context: *mut dc_context_t,
-    passphrase: *const libc::c_char,
-) -> libc::c_int {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_context_open()");
-        return 0;
-    }
-
-    let ctx = &*context;
-    let passphrase = to_string_lossy(passphrase);
-    block_on(ctx.open(passphrase))
-        .log_err(ctx, "dc_context_open() failed")
-        .map(|b| b as libc::c_int)
-        .unwrap_or(0)
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_context_is_open(context: *mut dc_context_t) -> libc::c_int {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_context_is_open()");
-        return 0;
-    }
-
-    let ctx = &*context;
-    block_on(ctx.is_open()) as libc::c_int
-}
-
 /// Release the context structure.
 ///
 /// This function releases the memory of the `dc_context_t` structure.
@@ -178,30 +134,20 @@ pub unsafe extern "C" fn dc_set_config(
     }
     let ctx = &*context;
     let key = to_string_lossy(key);
-    let value = to_opt_string_lossy(value);
-
-    block_on(async move {
-        if key.starts_with("ui.") {
-            ctx.set_ui_config(&key, value.as_deref())
+    match config::Config::from_str(&key) {
+        Ok(key) => block_on(async move {
+            let value = to_opt_string_lossy(value);
+            ctx.set_config(key, value.as_deref())
                 .await
                 .with_context(|| format!("Can't set {} to {:?}", key, value))
                 .log_err(ctx, "dc_set_config() failed")
                 .is_ok() as libc::c_int
-        } else {
-            match config::Config::from_str(&key) {
-                Ok(key) => ctx
-                    .set_config(key, value.as_deref())
-                    .await
-                    .with_context(|| format!("Can't set {} to {:?}", key, value))
-                    .log_err(ctx, "dc_set_config() failed")
-                    .is_ok() as libc::c_int,
-                Err(_) => {
-                    warn!(ctx, "dc_set_config(): invalid key");
-                    0
-                }
-            }
+        }),
+        Err(_) => {
+            warn!(ctx, "dc_set_config(): invalid key");
+            0
         }
-    })
+    }
 }
 
 #[no_mangle]
@@ -214,33 +160,20 @@ pub unsafe extern "C" fn dc_get_config(
         return "".strdup();
     }
     let ctx = &*context;
-
-    let key = to_string_lossy(key);
-
-    block_on(async move {
-        if key.starts_with("ui.") {
-            ctx.get_ui_config(&key)
+    match config::Config::from_str(&to_string_lossy(key)) {
+        Ok(key) => block_on(async move {
+            ctx.get_config(key)
                 .await
-                .log_err(ctx, "Can't get ui-config")
+                .log_err(ctx, "Can't get config")
                 .unwrap_or_default()
                 .unwrap_or_default()
                 .strdup()
-        } else {
-            match config::Config::from_str(&key) {
-                Ok(key) => ctx
-                    .get_config(key)
-                    .await
-                    .log_err(ctx, "Can't get config")
-                    .unwrap_or_default()
-                    .unwrap_or_default()
-                    .strdup(),
-                Err(_) => {
-                    warn!(ctx, "dc_get_config(): invalid key '{}'", &key);
-                    "".strdup()
-                }
-            }
+        }),
+        Err(_) => {
+            warn!(ctx, "dc_get_config(): invalid key");
+            "".strdup()
         }
-    })
+    }
 }
 
 #[no_mangle]
@@ -481,7 +414,6 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::Warning(_)
         | EventType::Error(_)
         | EventType::ConnectivityChanged
-        | EventType::SelfavatarChanged
         | EventType::ErrorSelfNotInGroup(_) => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -501,7 +433,6 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         EventType::ImexFileWritten(_) => 0,
         EventType::SecurejoinInviterProgress { contact_id, .. }
         | EventType::SecurejoinJoinerProgress { contact_id, .. } => *contact_id as libc::c_int,
-        EventType::WebxdcStatusUpdate { msg_id, .. } => msg_id.to_u32() as libc::c_int,
     }
 }
 
@@ -533,7 +464,6 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ImexFileWritten(_)
         | EventType::MsgsNoticed(_)
         | EventType::ConnectivityChanged
-        | EventType::SelfavatarChanged
         | EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
@@ -543,9 +473,6 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         EventType::SecurejoinInviterProgress { progress, .. }
         | EventType::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
         EventType::ChatEphemeralTimerModified { timer, .. } => timer.to_u32() as libc::c_int,
-        EventType::WebxdcStatusUpdate {
-            status_update_id, ..
-        } => status_update_id.to_u32() as libc::c_int,
     }
 }
 
@@ -586,8 +513,6 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SecurejoinInviterProgress { .. }
         | EventType::SecurejoinJoinerProgress { .. }
         | EventType::ConnectivityChanged
-        | EventType::SelfavatarChanged
-        | EventType::WebxdcStatusUpdate { .. }
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -689,8 +614,8 @@ pub unsafe extern "C" fn dc_preconfigure_keypair(
     let ctx = &*context;
     block_on(async move {
         let addr = dc_tools::EmailAddress::new(&to_string_lossy(addr))?;
-        let public = key::SignedPublicKey::from_asc(&to_string_lossy(public_data))?.0;
-        let secret = key::SignedSecretKey::from_asc(&to_string_lossy(secret_data))?.0;
+        let public = key::SignedPublicKey::from_base64(&to_string_lossy(public_data))?;
+        let secret = key::SignedSecretKey::from_base64(&to_string_lossy(secret_data))?;
         let keypair = key::KeyPair {
             addr,
             public,
@@ -877,52 +802,6 @@ pub unsafe extern "C" fn dc_send_videochat_invitation(
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_send_webxdc_status_update(
-    context: *mut dc_context_t,
-    msg_id: u32,
-    json: *const libc::c_char,
-    descr: *const libc::c_char,
-) -> libc::c_int {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_send_webxdc_status_update()");
-        return 0;
-    }
-    let ctx = &*context;
-
-    block_on(ctx.send_webxdc_status_update(
-        MsgId::new(msg_id),
-        &to_string_lossy(json),
-        &to_string_lossy(descr),
-    ))
-    .log_err(ctx, "Failed to send webxdc update")
-    .is_ok() as libc::c_int
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_get_webxdc_status_updates(
-    context: *mut dc_context_t,
-    msg_id: u32,
-    status_update_id: u32,
-) -> *mut libc::c_char {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_get_webxdc_status_updates()");
-        return "".strdup();
-    }
-    let ctx = &*context;
-
-    block_on(ctx.get_webxdc_status_updates(
-        MsgId::new(msg_id),
-        if status_update_id == 0 {
-            None
-        } else {
-            Some(StatusUpdateId::new(status_update_id))
-        },
-    ))
-    .unwrap_or_else(|_| "".to_string())
-    .strdup()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_draft(
     context: *mut dc_context_t,
@@ -1428,19 +1307,6 @@ pub unsafe extern "C" fn dc_create_group_chat(
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_create_broadcast_list(context: *mut dc_context_t) -> u32 {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_create_broadcast_list()");
-        return 0;
-    }
-    let ctx = &*context;
-    block_on(chat::create_broadcast_list(ctx))
-        .log_err(ctx, "Failed to create broadcast list")
-        .map(|id| id.to_u32())
-        .unwrap_or(0)
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_is_contact_in_chat(
     context: *mut dc_context_t,
@@ -1453,13 +1319,8 @@ pub unsafe extern "C" fn dc_is_contact_in_chat(
     }
     let ctx = &*context;
 
-    block_on(chat::is_contact_in_chat(
-        ctx,
-        ChatId::new(chat_id),
-        contact_id,
-    ))
-    .log_err(ctx, "is_contact_in_chat failed")
-    .unwrap_or_default() as libc::c_int
+    block_on(async move { chat::is_contact_in_chat(ctx, ChatId::new(chat_id), contact_id).await })
+        .into()
 }
 
 #[no_mangle]
@@ -1474,13 +1335,9 @@ pub unsafe extern "C" fn dc_add_contact_to_chat(
     }
     let ctx = &*context;
 
-    block_on(chat::add_contact_to_chat(
-        ctx,
-        ChatId::new(chat_id),
-        contact_id,
-    ))
-    .log_err(ctx, "Failed to add contact")
-    .is_ok() as libc::c_int
+    block_on(async move {
+        chat::add_contact_to_chat(ctx, ChatId::new(chat_id), contact_id).await as libc::c_int
+    })
 }
 
 #[no_mangle]
@@ -1495,13 +1352,12 @@ pub unsafe extern "C" fn dc_remove_contact_from_chat(
     }
     let ctx = &*context;
 
-    block_on(chat::remove_contact_from_chat(
-        ctx,
-        ChatId::new(chat_id),
-        contact_id,
-    ))
-    .log_err(ctx, "Failed to remove contact")
-    .is_ok() as libc::c_int
+    block_on(async move {
+        chat::remove_contact_from_chat(ctx, ChatId::new(chat_id), contact_id)
+            .await
+            .map(|_| 1)
+            .unwrap_or_log_default(ctx, "Failed to remove contact")
+    })
 }
 
 #[no_mangle]
@@ -1796,18 +1652,6 @@ pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) ->
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_download_full_msg(context: *mut dc_context_t, msg_id: u32) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_download_full_msg()");
-        return;
-    }
-    let ctx = &*context;
-    block_on(MsgId::new(msg_id).download_full(ctx))
-        .log_err(ctx, "Failed to download message fully.")
-        .ok();
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_may_be_valid_addr(addr: *const libc::c_char) -> libc::c_int {
     if addr.is_null() {
@@ -1830,7 +1674,7 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     let ctx = &*context;
 
     block_on(async move {
-        Contact::lookup_id_by_addr(ctx, &to_string_lossy(addr), Origin::IncomingReplyTo)
+        Contact::lookup_id_by_addr(ctx, to_string_lossy(addr), Origin::IncomingReplyTo)
             .await
             .unwrap_or_log_default(ctx, "failed to lookup id")
             .unwrap_or(0)
@@ -2022,7 +1866,7 @@ pub unsafe extern "C" fn dc_imex(
     context: *mut dc_context_t,
     what_raw: libc::c_int,
     param1: *const libc::c_char,
-    param2: *const libc::c_char,
+    _param2: *const libc::c_char,
 ) {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_imex()");
@@ -2035,13 +1879,12 @@ pub unsafe extern "C" fn dc_imex(
             return;
         }
     };
-    let passphrase = to_opt_string_lossy(param2);
 
     let ctx = &*context;
 
     if let Some(param1) = to_opt_string_lossy(param1) {
         spawn(async move {
-            imex::imex(ctx, what, param1.as_ref(), passphrase)
+            imex::imex(ctx, what, param1.as_ref())
                 .await
                 .log_err(ctx, "IMEX failed")
         });
@@ -2142,11 +1985,10 @@ pub unsafe extern "C" fn dc_check_qr(
     }
     let ctx = &*context;
 
-    let lot = match block_on(qr::check_qr(ctx, &to_string_lossy(qr))) {
-        Ok(qr) => qr.into(),
-        Err(err) => err.into(),
-    };
-    Box::into_raw(Box::new(lot))
+    block_on(async move {
+        let lot = qr::check_qr(ctx, &to_string_lossy(qr)).await;
+        Box::into_raw(Box::new(lot))
+    })
 }
 
 #[no_mangle]
@@ -2165,30 +2007,12 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
         Some(ChatId::new(chat_id))
     };
 
-    block_on(securejoin::dc_get_securejoin_qr(ctx, chat_id))
-        .unwrap_or_else(|_| "".to_string())
-        .strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_get_securejoin_qr_svg(
-    context: *mut dc_context_t,
-    chat_id: u32,
-) -> *mut libc::c_char {
-    if context.is_null() {
-        eprintln!("ignoring careless call to generate_verification_qr()");
-        return "".strdup();
-    }
-    let ctx = &*context;
-    let chat_id = if chat_id == 0 {
-        None
-    } else {
-        Some(ChatId::new(chat_id))
-    };
-
-    block_on(get_securejoin_qr_svg(ctx, chat_id))
-        .unwrap_or_else(|_| "".to_string())
-        .strdup()
+    block_on(async move {
+        securejoin::dc_get_securejoin_qr(ctx, chat_id)
+            .await
+            .unwrap_or_else(|| "".to_string())
+            .strdup()
+    })
 }
 
 #[no_mangle]
@@ -2322,16 +2146,6 @@ pub unsafe extern "C" fn dc_delete_all_locations(context: *mut dc_context_t) {
     });
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_get_last_error(context: *mut dc_context_t) -> *mut libc::c_char {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_get_last_error()");
-        return "".strdup();
-    }
-    let ctx = &*context;
-    block_on(ctx.get_last_error()).strdup()
-}
-
 // dc_array_t
 
 pub type dc_array_t = dc_array::dc_array_t;
@@ -2547,14 +2361,7 @@ pub unsafe extern "C" fn dc_chatlist_get_chat_id(
         return 0;
     }
     let ffi_list = &*chatlist;
-    let ctx = &*ffi_list.context;
-    match ffi_list.list.get_chat_id(index as usize) {
-        Ok(chat_id) => chat_id.to_u32(),
-        Err(err) => {
-            warn!(ctx, "get_chat_id failed: {}", err);
-            0
-        }
-    }
+    ffi_list.list.get_chat_id(index as usize).to_u32()
 }
 
 #[no_mangle]
@@ -2796,10 +2603,8 @@ pub unsafe extern "C" fn dc_chat_can_send(chat: *mut dc_chat_t) -> libc::c_int {
         return 0;
     }
     let ffi_chat = &*chat;
-    let ctx = &*ffi_chat.context;
-    block_on(ffi_chat.chat.can_send(ctx))
-        .log_err(ctx, "can_send failed")
-        .unwrap_or_default() as libc::c_int
+    let cxt = &*ffi_chat.context;
+    block_on(ffi_chat.chat.can_send(cxt)) as libc::c_int
 }
 
 #[no_mangle]
@@ -2986,16 +2791,6 @@ pub unsafe extern "C" fn dc_msg_get_state(msg: *mut dc_msg_t) -> libc::c_int {
     ffi_msg.message.get_state() as libc::c_int
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_get_download_state(msg: *mut dc_msg_t) -> libc::c_int {
-    if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_get_download_state()");
-        return 0;
-    }
-    let ffi_msg = &*msg;
-    ffi_msg.message.download_state() as libc::c_int
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_timestamp(msg: *mut dc_msg_t) -> i64 {
     if msg.is_null() {
@@ -3071,61 +2866,6 @@ pub unsafe extern "C" fn dc_msg_get_filename(msg: *mut dc_msg_t) -> *mut libc::c
     ffi_msg.message.get_filename().unwrap_or_default().strdup()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_get_webxdc_blob(
-    msg: *mut dc_msg_t,
-    filename: *const libc::c_char,
-    ret_bytes: *mut libc::size_t,
-) -> *mut libc::c_char {
-    if msg.is_null() || filename.is_null() || ret_bytes.is_null() {
-        eprintln!("ignoring careless call to dc_msg_get_webxdc_blob()");
-        return ptr::null_mut();
-    }
-    let ffi_msg = &*msg;
-    let ctx = &*ffi_msg.context;
-    let blob = block_on(async move {
-        ffi_msg
-            .message
-            .get_webxdc_blob(ctx, &to_string_lossy(filename))
-            .await
-    });
-    match blob {
-        Ok(blob) => {
-            *ret_bytes = blob.len();
-            let ptr = libc::malloc(*ret_bytes);
-            libc::memcpy(ptr, blob.as_ptr() as *mut libc::c_void, *ret_bytes);
-            ptr as *mut libc::c_char
-        }
-        Err(err) => {
-            eprintln!("failed read blob from archive: {}", err);
-            ptr::null_mut()
-        }
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_get_webxdc_info(msg: *mut dc_msg_t) -> *mut libc::c_char {
-    if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_get_webxdc_info()");
-        return "".strdup();
-    }
-    let ffi_msg = &*msg;
-    let ctx = &*ffi_msg.context;
-
-    block_on(async move {
-        let info = match ffi_msg.message.get_webxdc_info(ctx).await {
-            Ok(info) => info,
-            Err(err) => {
-                error!(ctx, "dc_msg_get_webxdc_info() failed to get info: {}", err);
-                return "".strdup();
-            }
-        };
-        serde_json::to_string(&info)
-            .unwrap_or_log_default(ctx, "dc_msg_get_webxdc_info() failed to serialise to json")
-            .strdup()
-    })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_filemime(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -3240,7 +2980,7 @@ pub unsafe extern "C" fn dc_msg_get_summary(
     let ffi_msg = &mut *msg;
     let ctx = &*ffi_msg.context;
 
-    let summary = block_on(ffi_msg.message.get_summary(ctx, maybe_chat))
+    let summary = block_on(async move { ffi_msg.message.get_summary(ctx, maybe_chat).await })
         .log_err(ctx, "dc_msg_get_summary failed")
         .unwrap_or_default();
     Box::into_raw(Box::new(summary.into()))
@@ -3258,13 +2998,12 @@ pub unsafe extern "C" fn dc_msg_get_summarytext(
     let ffi_msg = &mut *msg;
     let ctx = &*ffi_msg.context;
 
-    let summary = block_on(ffi_msg.message.get_summary(ctx, None))
-        .log_err(ctx, "dc_msg_get_summarytext failed")
-        .unwrap_or_default();
-    match usize::try_from(approx_characters) {
-        Ok(chars) => summary.truncated_text(chars).strdup(),
-        Err(_) => summary.text.strdup(),
-    }
+    block_on({
+        ffi_msg
+            .message
+            .get_summarytext(ctx, approx_characters.try_into().unwrap_or_default())
+    })
+    .strdup()
 }
 
 #[no_mangle]
@@ -3538,21 +3277,17 @@ pub unsafe extern "C" fn dc_msg_set_quote(msg: *mut dc_msg_t, quote: *const dc_m
         return;
     }
     let ffi_msg = &mut *msg;
-    let quote_msg = if quote.is_null() {
-        None
-    } else {
-        let ffi_quote = &*quote;
-        if ffi_msg.context != ffi_quote.context {
-            eprintln!("ignoring attempt to quote message from a different context");
-            return;
-        }
-        Some(&ffi_quote.message)
-    };
+    let ffi_quote = &*quote;
+
+    if ffi_msg.context != ffi_quote.context {
+        eprintln!("ignoring attempt to quote message from a different context");
+        return;
+    }
 
     block_on(async move {
         ffi_msg
             .message
-            .set_quote(&*ffi_msg.context, quote_msg)
+            .set_quote(&*ffi_msg.context, &ffi_quote.message)
             .await
             .log_err(&*ffi_msg.context, "failed to set quote")
             .ok();
@@ -3595,39 +3330,6 @@ pub unsafe extern "C" fn dc_msg_get_quoted_msg(msg: *const dc_msg_t) -> *mut dc_
     }
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_get_parent(msg: *const dc_msg_t) -> *mut dc_msg_t {
-    if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_get_parent()");
-        return ptr::null_mut();
-    }
-    let ffi_msg: &MessageWrapper = &*msg;
-    let context = &*ffi_msg.context;
-    let res = block_on(async move {
-        ffi_msg
-            .message
-            .parent(context)
-            .await
-            .log_err(context, "failed to get parent message")
-            .unwrap_or(None)
-    });
-
-    match res {
-        Some(message) => Box::into_raw(Box::new(MessageWrapper { context, message })),
-        None => ptr::null_mut(),
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_force_plaintext(msg: *mut dc_msg_t) {
-    if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_force_plaintext()");
-        return;
-    }
-    let ffi_msg = &mut *msg;
-    ffi_msg.message.force_plaintext();
-}
-
 // dc_contact_t
 
 /// FFI struct for [dc_contact_t]
@@ -3759,16 +3461,6 @@ pub unsafe extern "C" fn dc_contact_get_status(contact: *mut dc_contact_t) -> *m
     ffi_contact.contact.get_status().strdup()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_contact_get_last_seen(contact: *mut dc_contact_t) -> i64 {
-    if contact.is_null() {
-        eprintln!("ignoring careless call to dc_contact_get_last_seen()");
-        return 0;
-    }
-    let ffi_contact = &*contact;
-    ffi_contact.contact.last_seen()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_is_blocked(contact: *mut dc_contact_t) -> libc::c_int {
     if contact.is_null() {
@@ -3788,9 +3480,7 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
     let ffi_contact = &*contact;
     let ctx = &*ffi_contact.context;
 
-    block_on(ffi_contact.contact.is_verified(ctx))
-        .log_err(ctx, "is_verified failed")
-        .unwrap_or_default() as libc::c_int
+    block_on(async move { ffi_contact.contact.is_verified(ctx).await as libc::c_int })
 }
 
 // dc_lot_t
@@ -3848,7 +3538,7 @@ pub unsafe extern "C" fn dc_lot_get_state(lot: *mut dc_lot_t) -> libc::c_int {
     }
 
     let lot = &*lot;
-    lot.get_state() as libc::c_int
+    lot.get_state().to_i64().expect("impossible") as libc::c_int
 }
 
 #[no_mangle]
@@ -3946,11 +3636,7 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
 
     match socks5_enabled {
         Ok(socks5_enabled) => {
-            match block_on(provider::get_provider_info(
-                ctx,
-                addr.as_str(),
-                socks5_enabled,
-            )) {
+            match block_on(provider::get_provider_info(addr.as_str(), socks5_enabled)) {
                 Some(provider) => provider,
                 None => ptr::null_mut(),
             }
@@ -4032,7 +3718,7 @@ pub type dc_accounts_t = AccountsWrapper;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_new(
-    _os_name: *const libc::c_char,
+    os_name: *const libc::c_char,
     dbfile: *const libc::c_char,
 ) -> *mut dc_accounts_t {
     setup_panic!();
@@ -4042,7 +3728,13 @@ pub unsafe extern "C" fn dc_accounts_new(
         return ptr::null_mut();
     }
 
-    let accs = block_on(Accounts::new(as_path(dbfile).to_path_buf().into()));
+    let os_name = if os_name.is_null() {
+        String::from("DcFFI")
+    } else {
+        to_string_lossy(os_name)
+    };
+
+    let accs = block_on(Accounts::new(os_name, as_path(dbfile).to_path_buf().into()));
 
     match accs {
         Ok(accs) => Box::into_raw(Box::new(AccountsWrapper::new(accs))),
@@ -4108,19 +3800,9 @@ pub unsafe extern "C" fn dc_accounts_select_account(
     }
 
     let accounts = &*accounts;
-    block_on(async move {
-        let mut accounts = accounts.write().await;
-        match accounts.select_account(id).await {
-            Ok(()) => 1,
-            Err(err) => {
-                accounts.emit_event(EventType::Error(format!(
-                    "Failed to select account: {:#}",
-                    err
-                )));
-                0
-            }
-        }
-    })
+    block_on(async move { accounts.write().await.select_account(id).await })
+        .map(|_| 1)
+        .unwrap_or(0)
 }
 
 #[no_mangle]
@@ -4132,43 +3814,7 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -
 
     let accounts = &mut *accounts;
 
-    block_on(async move {
-        let mut accounts = accounts.write().await;
-        match accounts.add_account().await {
-            Ok(id) => id,
-            Err(err) => {
-                accounts.emit_event(EventType::Error(format!(
-                    "Failed to add account: {:#}",
-                    err
-                )));
-                0
-            }
-        }
-    })
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_accounts_add_closed_account(accounts: *mut dc_accounts_t) -> u32 {
-    if accounts.is_null() {
-        eprintln!("ignoring careless call to dc_accounts_add_closed_account()");
-        return 0;
-    }
-
-    let accounts = &mut *accounts;
-
-    block_on(async move {
-        let mut accounts = accounts.write().await;
-        match accounts.add_closed_account().await {
-            Ok(id) => id,
-            Err(err) => {
-                accounts.emit_event(EventType::Error(format!(
-                    "Failed to add account: {:#}",
-                    err
-                )));
-                0
-            }
-        }
-    })
+    block_on(async move { accounts.write().await.add_account().await }).unwrap_or(0)
 }
 
 #[no_mangle]
@@ -4183,19 +3829,9 @@ pub unsafe extern "C" fn dc_accounts_remove_account(
 
     let accounts = &mut *accounts;
 
-    block_on(async move {
-        let mut accounts = accounts.write().await;
-        match accounts.remove_account(id).await {
-            Ok(()) => 1,
-            Err(err) => {
-                accounts.emit_event(EventType::Error(format!(
-                    "Failed to remove account: {:#}",
-                    err
-                )));
-                0
-            }
-        }
-    })
+    block_on(async move { accounts.write().await.remove_account(id).await })
+        .map(|_| 1)
+        .unwrap_or_else(|_| 0)
 }
 
 #[no_mangle]
@@ -4212,21 +3848,14 @@ pub unsafe extern "C" fn dc_accounts_migrate_account(
     let dbfile = to_string_lossy(dbfile);
 
     block_on(async move {
-        let mut accounts = accounts.write().await;
-        match accounts
+        accounts
+            .write()
+            .await
             .migrate_account(async_std::path::PathBuf::from(dbfile))
             .await
-        {
-            Ok(id) => id,
-            Err(err) => {
-                accounts.emit_event(EventType::Error(format!(
-                    "Failed to migrate account: {:#}",
-                    err
-                )));
-                0
-            }
-        }
     })
+    .map(|_| 1)
+    .unwrap_or_else(|_| 0)
 }
 
 #[no_mangle]

deltachat-ffi/src/lot.rs

@@ -1,245 +0,0 @@
-//! # Legacy generic return values for C API.
-
-use crate::message::MessageState;
-use crate::qr::Qr;
-use crate::summary::{Summary, SummaryPrefix};
-use anyhow::Error;
-use std::borrow::Cow;
-
-/// An object containing a set of values.
-/// The meaning of the values is defined by the function returning the object.
-/// Lot objects are created
-/// eg. by chatlist.get_summary() or dc_msg_get_summary().
-///
-/// *Lot* is used in the meaning *heap* here.
-#[derive(Debug)]
-pub enum Lot {
-    Summary(Summary),
-    Qr(Qr),
-    Error(String),
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Meaning {
-    None = 0,
-    Text1Draft = 1,
-    Text1Username = 2,
-    Text1Self = 3,
-}
-
-impl Default for Meaning {
-    fn default() -> Self {
-        Meaning::None
-    }
-}
-
-impl Lot {
-    pub fn get_text1(&self) -> Option<&str> {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => None,
-                Some(SummaryPrefix::Draft(text)) => Some(text),
-                Some(SummaryPrefix::Username(username)) => Some(username),
-                Some(SummaryPrefix::Me(text)) => Some(text),
-            },
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => None,
-                Qr::AskVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::FprOk { .. } => None,
-                Qr::FprMismatch { .. } => None,
-                Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
-                Qr::Account { domain } => Some(domain),
-                Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { .. } => None,
-                Qr::Url { url } => Some(url),
-                Qr::Text { text } => Some(text),
-                Qr::WithdrawVerifyContact { .. } => None,
-                Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::ReviveVerifyContact { .. } => None,
-                Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
-            },
-            Self::Error(err) => Some(err),
-        }
-    }
-
-    pub fn get_text2(&self) -> Option<Cow<str>> {
-        match self {
-            Self::Summary(summary) => Some(summary.truncated_text(160)),
-            Self::Qr(_) => None,
-            Self::Error(_) => None,
-        }
-    }
-
-    pub fn get_text1_meaning(&self) -> Meaning {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => Meaning::None,
-                Some(SummaryPrefix::Draft(_text)) => Meaning::Text1Draft,
-                Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
-                Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
-            },
-            Self::Qr(_qr) => Meaning::None,
-            Self::Error(_err) => Meaning::None,
-        }
-    }
-
-    pub fn get_state(&self) -> LotState {
-        match self {
-            Self::Summary(summary) => summary.state.into(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => LotState::QrAskVerifyContact,
-                Qr::AskVerifyGroup { .. } => LotState::QrAskVerifyGroup,
-                Qr::FprOk { .. } => LotState::QrFprOk,
-                Qr::FprMismatch { .. } => LotState::QrFprMismatch,
-                Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
-                Qr::Account { .. } => LotState::QrAccount,
-                Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
-                Qr::Addr { .. } => LotState::QrAddr,
-                Qr::Url { .. } => LotState::QrUrl,
-                Qr::Text { .. } => LotState::QrText,
-                Qr::WithdrawVerifyContact { .. } => LotState::QrWithdrawVerifyContact,
-                Qr::WithdrawVerifyGroup { .. } => LotState::QrWithdrawVerifyGroup,
-                Qr::ReviveVerifyContact { .. } => LotState::QrReviveVerifyContact,
-                Qr::ReviveVerifyGroup { .. } => LotState::QrReviveVerifyGroup,
-            },
-            Self::Error(_err) => LotState::QrError,
-        }
-    }
-
-    pub fn get_id(&self) -> u32 {
-        match self {
-            Self::Summary(_) => Default::default(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { contact_id, .. } => *contact_id,
-                Qr::AskVerifyGroup { .. } => Default::default(),
-                Qr::FprOk { contact_id } => *contact_id,
-                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default(),
-                Qr::FprWithoutAddr { .. } => Default::default(),
-                Qr::Account { .. } => Default::default(),
-                Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => *contact_id,
-                Qr::Url { .. } => Default::default(),
-                Qr::Text { .. } => Default::default(),
-                Qr::WithdrawVerifyContact { contact_id, .. } => *contact_id,
-                Qr::WithdrawVerifyGroup { .. } => Default::default(),
-                Qr::ReviveVerifyContact { contact_id, .. } => *contact_id,
-                Qr::ReviveVerifyGroup { .. } => Default::default(),
-            },
-            Self::Error(_) => Default::default(),
-        }
-    }
-
-    pub fn get_timestamp(&self) -> i64 {
-        match self {
-            Self::Summary(summary) => summary.timestamp,
-            Self::Qr(_) => Default::default(),
-            Self::Error(_) => Default::default(),
-        }
-    }
-}
-
-#[repr(u32)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum LotState {
-    // Default
-    Undefined = 0,
-
-    // Qr States
-    /// id=contact
-    QrAskVerifyContact = 200,
-
-    /// text1=groupname
-    QrAskVerifyGroup = 202,
-
-    /// id=contact
-    QrFprOk = 210,
-
-    /// id=contact
-    QrFprMismatch = 220,
-
-    /// text1=formatted fingerprint
-    QrFprWithoutAddr = 230,
-
-    /// text1=domain
-    QrAccount = 250,
-
-    /// text1=domain, text2=instance pattern
-    QrWebrtcInstance = 260,
-
-    /// id=contact
-    QrAddr = 320,
-
-    /// text1=text
-    QrText = 330,
-
-    /// text1=URL
-    QrUrl = 332,
-
-    /// text1=error string
-    QrError = 400,
-
-    QrWithdrawVerifyContact = 500,
-
-    /// text1=groupname
-    QrWithdrawVerifyGroup = 502,
-
-    QrReviveVerifyContact = 510,
-
-    /// text1=groupname
-    QrReviveVerifyGroup = 512,
-
-    // Message States
-    MsgInFresh = 10,
-    MsgInNoticed = 13,
-    MsgInSeen = 16,
-    MsgOutPreparing = 18,
-    MsgOutDraft = 19,
-    MsgOutPending = 20,
-    MsgOutFailed = 24,
-    MsgOutDelivered = 26,
-    MsgOutMdnRcvd = 28,
-}
-
-impl Default for LotState {
-    fn default() -> Self {
-        LotState::Undefined
-    }
-}
-
-impl From<MessageState> for LotState {
-    fn from(s: MessageState) -> Self {
-        use MessageState::*;
-        match s {
-            Undefined => LotState::Undefined,
-            InFresh => LotState::MsgInFresh,
-            InNoticed => LotState::MsgInNoticed,
-            InSeen => LotState::MsgInSeen,
-            OutPreparing => LotState::MsgOutPreparing,
-            OutDraft => LotState::MsgOutDraft,
-            OutPending => LotState::MsgOutPending,
-            OutFailed => LotState::MsgOutFailed,
-            OutDelivered => LotState::MsgOutDelivered,
-            OutMdnRcvd => LotState::MsgOutMdnRcvd,
-        }
-    }
-}
-
-impl From<Summary> for Lot {
-    fn from(summary: Summary) -> Self {
-        Lot::Summary(summary)
-    }
-}
-
-impl From<Qr> for Lot {
-    fn from(qr: Qr) -> Self {
-        Lot::Qr(qr)
-    }
-}
-
-// Make it easy to convert errors into the final `Lot`.
-impl From<Error> for Lot {
-    fn from(error: Error) -> Self {
-        Lot::Error(error.to_string())
-    }
-}

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1"
-quote = "1"
+syn = "1.0.76"
+quote = "1.0.2"

draft/aeap-mvp.rst

@@ -1,33 +0,0 @@
-AEAP MVP
-========
-
-Changes to the UIs
-------------------
-
-- The secondary self addresses (see below) are shown in the UI, but not editable.
-
-- When the user changed the email address in the configure screen, show a dialog to the user, either directly explaining things or with a link to the FAQ (see "Other" below)
-
-Changes in the core
--------------------
-
-- We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
-
-- If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
-
-  - don't forget to deduplicate secondary self addresses in case the user switches back and forth between addresses).
-
-  - The key stays the same.
-
-- No changes for 1:1 chats, there simply is a new one
-
-- When we send a message to a group, and the primary address is not a member of a group, but a secondary address is:
-  
-  Add Chat-Group-Member-Removed=<old address> and Chat-Group-Member-Added=<new address> headers to this message
-
-  - On the receiving side, make sure that we accept this (even in verified groups) if the message is signed and the key stayed the same
-
-Other
------
-
-- The user is responsible that messages to the old address arrive at the new address, for example by configuring the old provider to forward all emails to the new one.

draft/webxdc-dev-reference.md

@@ -1,194 +0,0 @@
-# Webxdc Developer Reference
-
-## Webxdc File Format
-
-- a **Webxdc app** is a **ZIP-file** with the extension `.xdc`
-- the ZIP-file must use the default compression methods as of RFC 1950,
-  this is "Deflate" or "Store"
-- the ZIP-file must contain at least the file `index.html`
-- if the Webxdc app is started, `index.html` is opened in a restricted webview
-  that allow accessing resources only from the ZIP-file
-
-
-## Webxdc API
-
-There are some additional APIs available once `webxdc.js` is included
-(the file will be provided by the concrete implementations,
-no need to add `webxdc.js` to your ZIP-file):
-
-```html
-<script src="webxdc.js"></script>
-```
-
-### sendUpdate()
-
-```js
-window.webxdc.sendUpdate(update, descr);
-```
-
-Webxdc apps are usually shared in a chat and run independently on each peer.
-To get a shared state, the peers use `sendUpdate()` to send updates to each other.
-
-- `update`: an object with the following fields:  
-    - `update.payload`: any javascript primitive, array or object.
-    - `update.info`: optional, short, informational message that will be added to the chat,
-       eg. "Alice voted" or "Bob scored 123 in MyGame";
-       usually only one line of text is shown,
-       use this option sparingly to not spam the chat.
-    - `update.summary`: optional, short text, shown beside app icon;
-       it is recommended to use some aggregated value,  eg. "8 votes", "Highscore: 123"
-
-- `descr`: short, human-readable description what this update is about.
-  this is shown eg. as a fallback text in an email program.
-
-All peers, including the sending one,
-will receive the update by the callback given to `setUpdateListener()`.
-
-There are situations where the user cannot send messages to a chat,
-eg. contact requests or if the user has left a group.
-In these cases, you can still call `sendUpdate()`,
-however, the update won't be sent to other peers
-and you won't get the update by `setUpdateListener()` nor by `getAllUpdates()`.
-
-
-### setUpdateListener()
-
-```js
-window.webxdc.setUpdateListener((update) => {});
-```
-
-With `setUpdateListener()` you define a callback that receives the updates
-sent by `sendUpdate()`.
-
-- `update`: passed to the callback on updates with the following fields:  
-  `update.payload`: equals the payload given to `sendUpdate()`
-
-The callback is called for updates sent by you or other peers.
-
-
-### getAllUpdates()
-
-```js
-updates = await window.webxdc.getAllUpdates();
-```
-
-In case your Webxdc was just started,
-you may want to reconstruct the state from the last run -
-and also incorporate updates that may have arrived while the app was not running.
-
-- `updates`: All previous updates in an array, 
-  eg. `[{payload: "foo"},{payload: "bar"}]`
-  if `webxdc.sendUpdate({payload: "foo"}); webxdc.sendUpdate({payload: "bar"};` was called on the last run.
-
-The updates are wrapped into a Promise that you can `await` for.
-If you are not in an async function and cannot use `await` therefore,
-you can get the updates with `then()`:
-
-```js
-window.webxdc.getAllUpdates().then(updates => {});
-```
-
-
-### selfAddr
-
-```js
-window.webxdc.selfAddr
-```
-
-Property with the peer's own address.
-This is esp. useful if you want to differ between different peers -
-just send the address along with the payload,
-and, if needed, compare the payload addresses against selfAddr() later on.
-
-
-### selfName
-
-```js
-window.webxdc.selfName
-```
-
-Property with the peer's own name.
-This is name chosen by the user in their settings,
-if there is nothing set, that defaults to the peer's address.
-
-
-## manifest.toml
-
-If the ZIP-file contains a `manifest.toml` in its root directory,
-some basic information are read and used from there.
-
-the `manifest.toml` has the following format
-
-```toml
-name = "My App Name"
-```
-
-- **name** - The name of the app.
-  If no name is set or if there is no manifest, the filename is used as the app name.
-
-
-## App Icon
-
-If the ZIP-root contains an `icon.png` or `icon.jpg`,
-these files are used as the icon for the app.
-The icon should be a square at reasonable width/height;
-round corners etc. will be added by the implementations as needed.
-If no icon is set, a default icon will be used.
-
-
-## Webxdc Examples
-
-The following example shows an input field and  every input is show on all peers.
-
-```html
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8"/>
-    <script src="webxdc.js"></script>
-  </head>
-  <body>
-    <input id="input" type="text"/>
-    <a href="" onclick="sendMsg(); return false;">Send</a>
-    <p id="output"></p>
-    <script>
-    
-      function sendMsg() {
-        msg = document.getElementById("input").value;
-        window.webxdc.sendUpdate({payload: msg}, 'Someone typed "'+msg+'".');
-      }
-    
-      function receiveUpdate(update) {
-        document.getElementById('output').innerHTML += update.payload + "<br>";
-      }
-    
-      window.webxdc.setUpdateListener(receiveUpdate);
-      window.webxdc.getAllUpdates().then(updates => updates.forEach(receiveUpdate));
-
-    </script>
-  </body>
-</html>
-```
-
-[Webxdc Development Tool](https://github.com/deltachat/webxdc-dev)
-offers an **Webxdc Simulator** that can be used in many browsers without any installation needed.
-You can also use that repository as a template for your own app -
-just clone and start adapting things to your need.
-
-
-### Advanced Examples
-
-- [2048](https://github.com/adbenitez/2048.xdc)
-- [Draw](https://github.com/adbenitez/draw.xdc)
-- [Poll](https://github.com/r10s/webxdc-poll/)
-- [Tic Tac Toe](https://github.com/Simon-Laux/tictactoe.xdc)
-- Even more with [Topic #webxdc on Github](https://github.com/topics/webxdc)
-
-
-## Closing Remarks
-
-- older devices might not have the newest js features in their webview,
-  you may want to transpile your code down to an older js version eg. with https://babeljs.io
-- there are tons of ideas for enhancements of the API and the file format,
-  eg. in the future, we will may define icon- and manifest-files,
-  allow to aggregate the state or add metadata.

examples/repl/cmdline.rs

@@ -2,7 +2,7 @@ extern crate dirs;
 
 use std::str::FromStr;
 
-use anyhow::{bail, ensure, Result};
+use anyhow::{bail, ensure, Error};
 use async_std::path::Path;
 use deltachat::chat::{
     self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
@@ -13,7 +13,6 @@ use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
-use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
@@ -98,10 +97,10 @@ async fn reset_tables(context: &Context, bits: i32) {
     });
 }
 
-async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
+async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
     let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", false).await {
+    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -189,18 +188,10 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         MessageState::OutFailed => " !!",
         _ => "",
     };
-
-    let downloadstate = match msg.download_state() {
-        DownloadState::Done => "",
-        DownloadState::Available => " [⬇ Download available]",
-        DownloadState::InProgress => " [⬇ Download in progress...]️",
-        DownloadState::Failure => " [⬇ Download failed]",
-    };
-
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -234,12 +225,11 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             ""
         },
         statestr,
-        downloadstate,
         &temp2,
     );
 }
 
-async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
+async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
         if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
@@ -267,59 +257,59 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
     Ok(())
 }
 
-async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
+async fn log_contactlist(context: &Context, contacts: &[u32]) {
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        let contact = Contact::get_by_id(context, *contact_id).await?;
-        let name = contact.get_display_name();
-        let addr = contact.get_addr();
-        let verified_state = contact.is_verified(context).await?;
-        let verified_str = if VerifiedStatus::Unverified != verified_state {
-            if verified_state == VerifiedStatus::BidirectVerified {
-                " √√"
+        if let Ok(contact) = Contact::get_by_id(context, *contact_id).await {
+            let name = contact.get_display_name();
+            let addr = contact.get_addr();
+            let verified_state = contact.is_verified(context).await;
+            let verified_str = if VerifiedStatus::Unverified != verified_state {
+                if verified_state == VerifiedStatus::BidirectVerified {
+                    " √√"
+                } else {
+                    " √"
+                }
             } else {
-                " √"
-            }
-        } else {
-            ""
-        };
-        line = format!(
-            "{}{} <{}>",
-            if !name.is_empty() {
-                &name
-            } else {
-                "<name unset>"
-            },
-            verified_str,
-            if !addr.is_empty() {
-                &addr
-            } else {
-                "addr unset"
-            }
-        );
-        let peerstate = Peerstate::from_addr(context, &addr)
-            .await
-            .expect("peerstate error");
-        if peerstate.is_some() && *contact_id != 1 {
-            line2 = format!(
-                ", prefer-encrypt={}",
-                peerstate.as_ref().unwrap().prefer_encrypt
+                ""
+            };
+            line = format!(
+                "{}{} <{}>",
+                if !name.is_empty() {
+                    &name
+                } else {
+                    "<name unset>"
+                },
+                verified_str,
+                if !addr.is_empty() {
+                    &addr
+                } else {
+                    "addr unset"
+                }
             );
-        }
+            let peerstate = Peerstate::from_addr(context, &addr)
+                .await
+                .expect("peerstate error");
+            if peerstate.is_some() && *contact_id != 1 {
+                line2 = format!(
+                    ", prefer-encrypt={}",
+                    peerstate.as_ref().unwrap().prefer_encrypt
+                );
+            }
 
-        println!("Contact#{}: {}{}", *contact_id, line, line2);
+            println!("Contact#{}: {}{}", *contact_id, line, line2);
+        }
     }
-    Ok(())
 }
 
 fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<()> {
+pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<(), Error> {
     let mut sel_chat = if !chat_id.is_unset() {
-        Some(Chat::load_from_db(&context, *chat_id).await?)
+        Chat::load_from_db(&context, *chat_id).await.ok()
     } else {
         None
     };
@@ -370,7 +360,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  chat [<chat-id>|0]\n\
                  createchat <contact-id>\n\
                  creategroup <name>\n\
-                 createbroadcast\n\
                  createprotected <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
@@ -386,8 +375,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  sendsticker <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
                  sendhtml <file for html-part> [<text for plain-part>]\n\
-                 sendsyncmsg\n\
-                 sendupdate <msg-id> <json status update>\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -406,7 +393,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
-                 download <msg-id>\n\
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
@@ -424,10 +410,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listblocked\n\
                  ======================================Misc.==\n\
                  getqr [<chat-id>]\n\
-                 getqrsvg [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
-                 joinqr <qr-content>\n\
                  setqr <qr-content>\n\
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
@@ -465,39 +449,27 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            continue_key_transfer(&context, MsgId::new(arg1.parse()?), arg2).await?;
+            continue_key_transfer(&context, MsgId::new(arg1.parse()?), &arg2).await?;
         }
         "has-backup" => {
             has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(
-                &context,
-                ImexMode::ExportBackup,
-                dir.as_ref(),
-                Some(arg2.to_string()),
-            )
-            .await?;
+            imex(&context, ImexMode::ExportBackup, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(
-                &context,
-                ImexMode::ImportBackup,
-                arg1.as_ref(),
-                Some(arg2.to_string()),
-            )
-            .await?;
+            imex(&context, ImexMode::ImportBackup, arg1.as_ref()).await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref(), None).await?;
+            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref(), None).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref()).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -524,13 +496,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "set" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(arg1)?;
+            let key = config::Config::from_str(&arg1)?;
             let value = if arg2.is_empty() { None } else { Some(arg2) };
             context.set_config(key, value).await?;
         }
         "get" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(arg1)?;
+            let key = config::Config::from_str(&arg1)?;
             let val = context.get_config(key).await;
             println!("{}={:?}", key, val);
         }
@@ -576,7 +548,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 );
 
                 for i in (0..cnt).rev() {
-                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)?).await?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
                         "{}#{}: {} [{} fresh] {}{}{}{}",
                         chat_prefix(&chat),
@@ -731,11 +703,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!("Group#{} created successfully.", chat_id);
         }
-        "createbroadcast" => {
-            let chat_id = chat::create_broadcast_list(&context).await?;
-
-            println!("Broadcast#{} created successfully.", chat_id);
-        }
         "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
@@ -748,9 +715,17 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
             let contact_id_0: u32 = arg1.parse()?;
-            chat::add_contact_to_chat(&context, sel_chat.as_ref().unwrap().get_id(), contact_id_0)
-                .await?;
-            println!("Contact added to chat.");
+            if chat::add_contact_to_chat(
+                &context,
+                sel_chat.as_ref().unwrap().get_id(),
+                contact_id_0,
+            )
+            .await
+            {
+                println!("Contact added to chat.");
+            } else {
+                bail!("Cannot add contact to chat.");
+            }
         }
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -768,12 +743,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "groupname" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            chat::set_chat_name(
-                &context,
-                sel_chat.as_ref().unwrap().get_id(),
-                &format!("{} {}", arg1, arg2).trim(),
-            )
-            .await?;
+            chat::set_chat_name(&context, sel_chat.as_ref().unwrap().get_id(), arg1).await?;
 
             println!("Chat name set");
         }
@@ -793,7 +763,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await?;
             println!("Memberlist:");
 
-            log_contactlist(&context, &contacts).await?;
+            log_contactlist(&context, &contacts).await;
             println!(
                 "{} contacts\nLocation streaming: {}",
                 contacts.len(),
@@ -916,20 +886,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }));
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
-        "sendsyncmsg" => match context.send_sync_msg().await? {
-            Some(msg_id) => println!("sync message sent as {}.", msg_id),
-            None => println!("sync message not needed."),
-        },
-        "sendupdate" => {
-            ensure!(
-                !arg1.is_empty() && !arg2.is_empty(),
-                "Arguments <msg-id> <json status update> expected"
-            );
-            let msg_id = MsgId::new(arg1.parse()?);
-            context
-                .send_webxdc_status_update(msg_id, arg2, "this is a webxdc status update")
-                .await?;
-        }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -937,7 +893,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
-            let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
+            let chat = if let Some(ref sel_chat) = sel_chat {
+                Some(sel_chat.get_id())
+            } else {
+                None
+            };
+
             let time_start = std::time::SystemTime::now();
             let msglist = context.search_msgs(chat, arg1).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
@@ -1067,12 +1028,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let res = message::get_msg_info(&context, id).await?;
             println!("{}", res);
         }
-        "download" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = MsgId::new(arg1.parse()?);
-            println!("Scheduling download for {:?}", id);
-            id.download_full(&context).await?;
-        }
         "html" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
@@ -1123,7 +1078,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 Some(arg1),
             )
             .await?;
-            log_contactlist(&context, &contacts).await?;
+            log_contactlist(&context, &contacts).await;
             println!("{} contacts.", contacts.len());
         }
         "addcontact" => {
@@ -1165,7 +1120,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     if 0 != i {
                         res += ", ";
                     }
-                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)?).await?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     res += &format!("{}#{}", chat_prefix(&chat), chat.get_id());
                 }
             }
@@ -1188,13 +1143,19 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "listblocked" => {
             let contacts = Contact::get_all_blocked(&context).await?;
-            log_contactlist(&context, &contacts).await?;
+            log_contactlist(&context, &contacts).await;
             println!("{} blocked contacts.", contacts.len());
         }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            let qr = check_qr(&context, arg1).await?;
-            println!("qr={:?}", qr);
+            let res = check_qr(&context, arg1).await;
+            println!(
+                "state={}, id={}, text1={:?}, text2={:?}",
+                res.get_state(),
+                res.get_id(),
+                res.get_text1(),
+                res.get_text2()
+            );
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
@@ -1208,7 +1169,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let socks5_enabled = context
                 .get_config_bool(config::Config::Socks5Enabled)
                 .await?;
-            match provider::get_provider_info(&context, arg1, socks5_enabled).await {
+            match provider::get_provider_info(arg1, socks5_enabled).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);

examples/repl/main.rs

@@ -33,8 +33,6 @@ use rustyline::{
 
 mod cmdline;
 use self::cmdline::*;
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
-use std::fs;
 
 /// Event Handler
 fn receive_event(event: EventType) {
@@ -169,14 +167,13 @@ const DB_COMMANDS: [&str; 10] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 36] = [
+const CHAT_COMMANDS: [&str; 33] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
     "creategroup",
-    "createbroadcast",
-    "createprotected",
+    "createverified",
     "addmember",
     "removemember",
     "groupname",
@@ -190,8 +187,6 @@ const CHAT_COMMANDS: [&str; 36] = [
     "sendimage",
     "sendfile",
     "sendhtml",
-    "sendsyncmsg",
-    "sendupdate",
     "videochat",
     "draft",
     "listmedia",
@@ -207,14 +202,13 @@ const CHAT_COMMANDS: [&str; 36] = [
     "accept",
     "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 7] = [
+const MESSAGE_COMMANDS: [&str; 6] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
     "delmsg",
-    "download",
 ];
 const CONTACT_COMMANDS: [&str; 9] = [
     "listcontacts",
@@ -227,12 +221,10 @@ const CONTACT_COMMANDS: [&str; 9] = [
     "unblock",
     "listblocked",
 ];
-const MISC_COMMANDS: [&str; 12] = [
+const MISC_COMMANDS: [&str; 10] = [
     "getqr",
-    "getqrsvg",
     "getbadqr",
     "checkqr",
-    "joinqr",
     "event",
     "fileinfo",
     "clear",
@@ -298,7 +290,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
@@ -333,7 +325,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
 
         loop {
             let p = "> ";
-            let readline = rl.readline(p);
+            let readline = rl.readline(&p);
 
             match readline {
                 Ok(line) => {
@@ -417,31 +409,18 @@ async fn handle_cmd(
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
             let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
-            if !qr.is_empty() {
-                if arg0 == "getbadqr" && qr.len() > 40 {
-                    qr.replace_range(12..22, "0000000000")
-                }
-                println!("{}", qr);
-                let output = Command::new("qrencode")
-                    .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
-                    .output()
-                    .expect("failed to execute process");
-                io::stdout().write_all(&output.stdout).unwrap();
-                io::stderr().write_all(&output.stderr).unwrap();
-            }
-        }
-        "getqrsvg" => {
-            ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
-            match get_securejoin_qr_svg(&ctx, group).await {
-                Ok(svg) => {
-                    fs::write(&file, svg)?;
-                    println!("QR code svg written to: {:#?}", file);
-                }
-                Err(err) => {
-                    bail!("Failed to get QR code svg: {}", err);
+            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
+                if !qr.is_empty() {
+                    if arg0 == "getbadqr" && qr.len() > 40 {
+                        qr.replace_range(12..22, "0000000000")
+                    }
+                    println!("{}", qr);
+                    let output = Command::new("qrencode")
+                        .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
+                        .output()
+                        .expect("failed to execute process");
+                    io::stdout().write_all(&output.stdout).unwrap();
+                    io::stderr().write_all(&output.stderr).unwrap();
                 }
             }
         }

examples/simple.rs

@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new(dbfile.into(), 0)
+    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;

proptest-regressions/dc_tools.txt

@@ -7,5 +7,3 @@
 cc c310754465ee0261807b96fa9bcc4861ff9aa286e94667524b5960c69f9b6620 # shrinks to buf = "", approx_chars = 0, do_unwrap = false
 cc 5fd8d730b0a9cdf7308ce58818ca9aefc0255c9ba2a0878944fc48d43a67315b # shrinks to buf = "𑒀ὐ￠🜀\u{1e01b}A a🟠", approx_chars = 0, do_unwrap = false
 cc c6a0029a54137a4b9efc9ef2ea6d9a7dd1d60d1c937bb472b66a174618ba8013 # shrinks to buf = "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ  ", approx_chars = 0, do_unwrap = false
-cc 9796807baeda701227dcdcfc9fdaa93ddd556da2bb1630381bfe2e037bee73f6 # shrinks to buf = " ꫛ®a\u{11300}a", approx_chars = 0
-cc 063a4c42ac1ec9aa37af54521b210ba9cd82dcc9cc3be296ca2fedf8240072d4 # shrinks to buf = "a᪠  0A", approx_chars = 0

python/mypy.ini

@@ -1,19 +0,0 @@
-[mypy]
-
-[mypy-deltachat.capi.*]
-ignore_missing_imports = True
-
-[mypy-pluggy.*]
-ignore_missing_imports = True
-
-[mypy-cffi.*]
-ignore_missing_imports = True
-
-[mypy-imapclient.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-_pytest.*]
-ignore_missing_imports = True

python/pyproject.toml

@@ -1,8 +0,0 @@
-[build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools_scm]
-root = ".."
-tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
-git_describe_command = "git describe --dirty --tags --long --match py-*.*"

python/setup.py

@@ -8,11 +8,17 @@ def main():
         long_description = f.read()
     setuptools.setup(
         name='deltachat',
+        setup_requires=['setuptools_scm', 'cffi>=1.0.0'],
+        use_scm_version = {
+            "root": "..",
+            "relative_to": __file__,
+            'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
+            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
+        },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
         install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
-        setup_requires=['setuptools_scm'], # required for compatibility with `python3 setup.py sdist`
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -19,9 +19,9 @@ except DistributionNotFound:
 
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
-        for name in dir(const):
+        for name, val in vars(const).items():
             if name.startswith("DC_EVENT_"):
-                _DC_EVENTNAME_MAP[getattr(const, name)] = name
+                _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
 
 
@@ -75,6 +75,7 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("addr", args.email)
         ac.set_config("mail_pw", args.password)
         ac.set_config("mvbox_move", "0")
+        ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
         ac.set_config("bot", "1")
         configtracker = ac.configure()

python/src/deltachat/account.py

@@ -8,14 +8,13 @@ import os
 from array import array
 from . import const
 from .capi import ffi, lib
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array, DCLot
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
 from .tracker import ImexTracker, ConfigureTracker
 from . import hookspec
 from .events import EventThread
-from typing import Union, Any, Dict, Optional, List, Generator
 
 
 class MissingCredentials(ValueError):
@@ -29,7 +28,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True) -> None:
+    def __init__(self, db_path, os_name=None, logging=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -59,11 +58,11 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
-    def disable_logging(self) -> None:
+    def disable_logging(self):
         """ disable logging. """
         self._logging = False
 
-    def enable_logging(self) -> None:
+    def enable_logging(self):
         """ re-enable logging. """
         self._logging = True
 
@@ -74,12 +73,12 @@ class Account(object):
         if self._logging:
             self._pm.hook.ac_log_line(message=msg)
 
-    def _check_config_key(self, name: str) -> None:
+    def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
                            name, self._configkeys))
 
-    def get_info(self) -> Dict[str, str]:
+    def get_info(self):
         """ return dictionary of built config parameters. """
         lines = from_dc_charpointer(lib.dc_get_info(self._dc_context))
         d = {}
@@ -106,19 +105,19 @@ class Account(object):
             cursor += len(entry) + 1
         log("")
 
-    def set_stock_translation(self, id: int, string: str) -> None:
+    def set_stock_translation(self, id, string):
         """ set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
         :returns: None
         """
-        bytestring = string.encode("utf8")
-        res = lib.dc_set_stock_translation(self._dc_context, id, bytestring)
+        string = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, string)
         if res == 0:
             raise ValueError("could not set translation string")
 
-    def set_config(self, name: str, value: Optional[str]) -> None:
+    def set_config(self, name, value):
         """ set configuration values.
 
         :param name: config key name (unicode)
@@ -126,16 +125,16 @@ class Account(object):
         :returns: None
         """
         self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        if namebytes == b"addr" and self.is_configured():
+        name = name.encode("utf8")
+        if name == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
         if value is not None:
-            valuebytes = value.encode("utf8")
+            value = value.encode("utf8")
         else:
-            valuebytes = ffi.NULL
-        lib.dc_set_config(self._dc_context, namebytes, valuebytes)
+            value = ffi.NULL
+        lib.dc_set_config(self._dc_context, name, value)
 
-    def get_config(self, name: str) -> str:
+    def get_config(self, name):
         """ return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
@@ -144,12 +143,12 @@ class Account(object):
         """
         if name != "sys.config_keys":
             self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        res = lib.dc_get_config(self._dc_context, namebytes)
+        name = name.encode("utf8")
+        res = lib.dc_get_config(self._dc_context, name)
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None:
+    def _preconfigure_keypair(self, addr, public, secret):
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
@@ -161,7 +160,7 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def update_config(self, kwargs: Dict[str, Any]) -> None:
+    def update_config(self, kwargs):
         """ update config values.
 
         :param kwargs: name=value config settings for this account.
@@ -171,7 +170,7 @@ class Account(object):
         for key, value in kwargs.items():
             self.set_config(key, str(value))
 
-    def is_configured(self) -> bool:
+    def is_configured(self):
         """ determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
@@ -179,13 +178,7 @@ class Account(object):
         """
         return True if lib.dc_is_configured(self._dc_context) else False
 
-    def is_open(self) -> bool:
-        """Determine if account is open
-
-        :returns True if account is open."""
-        return True if lib.dc_context_is_open(self._dc_context) else False
-
-    def set_avatar(self, img_path: Optional[str]) -> None:
+    def set_avatar(self, img_path):
         """Set self avatar.
 
         :raises ValueError: if profile image could not be set
@@ -197,18 +190,20 @@ class Account(object):
             assert os.path.exists(img_path), img_path
             self.set_config("selfavatar", img_path)
 
-    def check_is_configured(self) -> None:
+    def check_is_configured(self):
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_latest_backupfile(self, backupdir) -> Optional[str]:
+    def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
-        return from_optional_dc_charpointer(res)
+        if res == ffi.NULL:
+            return None
+        return from_dc_charpointer(res)
 
-    def get_blobdir(self) -> str:
+    def get_blobdir(self):
         """ return the directory for files.
 
         All sent files are copied to this directory if necessary.
@@ -216,15 +211,15 @@ class Account(object):
         """
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
-    def get_self_contact(self) -> Contact:
+    def get_self_contact(self):
         """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name: Optional[str] = None) -> Contact:
-        """create a (new) Contact or return an existing one.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
         Calling this method will always result in the same
         underlying contact id.  If there already is a Contact
@@ -241,13 +236,13 @@ class Account(object):
         contact_id = lib.dc_create_contact(self._dc_context, name, addr)
         return Contact(self, contact_id)
 
-    def get_contact(self, obj) -> Optional[Contact]:
+    def get_contact(self, obj):
         if isinstance(obj, Contact):
             return obj
         (_, addr) = self.get_contact_addr_and_name(obj)
         return self.get_contact_by_addr(addr)
 
-    def get_contact_addr_and_name(self, obj, name: Optional[str] = None):
+    def get_contact_addr_and_name(self, obj, name=None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -265,7 +260,7 @@ class Account(object):
             name = displayname
         return (name, addr)
 
-    def delete_contact(self, contact: Contact) -> bool:
+    def delete_contact(self, contact):
         """ delete a Contact.
 
         :param contact: contact object obtained
@@ -276,23 +271,22 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email: str) -> Optional[Contact]:
+    def get_contact_by_addr(self, email):
         """ get a contact for the email address or None if it's blocked or doesn't exist. """
         _, addr = parseaddr(email)
         addr = as_dc_charpointer(addr)
         contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
         if contact_id:
             return self.get_contact_by_id(contact_id)
-        return None
 
-    def get_contact_by_id(self, contact_id: int) -> Contact:
-        """ return Contact instance or raise an exception.
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
         :param contact_id: integer id of this contact.
-        :returns: :class:`deltachat.contact.Contact` instance.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
         """
         return Contact(self, contact_id)
 
-    def get_blocked_contacts(self) -> List[Contact]:
+    def get_blocked_contacts(self):
         """ return a list of all blocked contacts.
 
         :returns: list of :class:`deltachat.contact.Contact` objects.
@@ -303,13 +297,8 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_contacts(
-        self,
-        query: Optional[str] = None,
-        with_self: bool = False,
-        only_verified: bool = False,
-    ) -> List[Contact]:
-        """get a (filtered) list of contacts.
+    def get_contacts(self, query=None, with_self=False, only_verified=False):
+        """ get a (filtered) list of contacts.
 
         :param query: if a string is specified, only return contacts
                       whose name or e-mail matches query.
@@ -329,7 +318,7 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_fresh_messages(self) -> Generator[Message, None, None]:
+    def get_fresh_messages(self):
         """ yield all fresh messages from all chats. """
         dc_array = ffi.gc(
             lib.dc_get_fresh_msgs(self._dc_context),
@@ -337,17 +326,12 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj) -> Chat:
+    def create_chat(self, obj):
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
-    def create_group_chat(
-        self,
-        name: str,
-        contacts: Optional[List[Contact]] = None,
-        verified: bool = False,
-    ) -> Chat:
-        """create a new group chat object.
+    def create_group_chat(self, name, contacts=None, verified=False):
+        """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
@@ -363,7 +347,7 @@ class Account(object):
                 chat.add_contact(contact)
         return chat
 
-    def get_chats(self) -> List[Chat]:
+    def get_chats(self):
         """ return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
@@ -380,17 +364,17 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self) -> Chat:
+    def get_device_chat(self):
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
-    def get_message_by_id(self, msg_id: int) -> Message:
+    def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
-    def get_chat_by_id(self, chat_id: int) -> Chat:
+    def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
@@ -402,21 +386,19 @@ class Account(object):
         lib.dc_chat_unref(res)
         return Chat(self, chat_id)
 
-    def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
+    def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
         arr = array("i")
         for msg in messages:
-            if isinstance(msg, Message):
-                arr.append(msg.id)
-            else:
-                arr.append(msg)
+            msg = getattr(msg, "id", msg)
+            arr.append(msg)
         msg_ids = ffi.cast("uint32_t*", ffi.from_buffer(arr))
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
-    def forward_messages(self, messages: List[Message], chat: Chat) -> None:
+    def forward_messages(self, messages, chat):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -426,7 +408,7 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
-    def delete_messages(self, messages: List[Message]) -> None:
+    def delete_messages(self, messages):
         """ delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -484,7 +466,7 @@ class Account(object):
     def imex(self, path, imex_cmd):
         lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
 
-    def initiate_key_transfer(self) -> str:
+    def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
@@ -495,7 +477,7 @@ class Account(object):
             raise RuntimeError("could not send out autocrypt setup message")
         return from_dc_charpointer(res)
 
-    def get_setup_contact_qr(self) -> str:
+    def get_setup_contact_qr(self):
         """ get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -545,9 +527,7 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(
-        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
-    ) -> None:
+    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -591,7 +571,7 @@ class Account(object):
     def get_connectivity(self):
         return lib.dc_get_connectivity(self._dc_context)
 
-    def get_connectivity_html(self) -> str:
+    def get_connectivity_html(self):
         return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
 
     def all_work_done(self):
@@ -641,7 +621,7 @@ class Account(object):
         """
         lib.dc_maybe_network(self._dc_context)
 
-    def configure(self, reconfigure: bool = False) -> ConfigureTracker:
+    def configure(self, reconfigure=False):
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
@@ -654,11 +634,11 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def wait_shutdown(self) -> None:
+    def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def stop_io(self) -> None:
+    def stop_io(self):
         """ stop core IO scheduler if it is running. """
         self.log("stop_ongoing")
         self.stop_ongoing()
@@ -666,7 +646,7 @@ class Account(object):
         self.log("dc_stop_io (stop core IO scheduler)")
         lib.dc_stop_io(self._dc_context)
 
-    def shutdown(self) -> None:
+    def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove
         underlying dc_context)."""
         if self._dc_context is None:

python/src/deltachat/chat.py

@@ -3,13 +3,12 @@
 import mimetypes
 import calendar
 import json
-from datetime import datetime, timezone
+from datetime import datetime
 import os
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
-from typing import Optional
 
 
 class Chat(object):
@@ -18,20 +17,20 @@ class Chat(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id) -> None:
+    def __init__(self, account, id):
         from .account import Account
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other):
         return self.id == getattr(other, "id", None) and \
                self.account._dc_context == other.account._dc_context
 
-    def __ne__(self, other) -> bool:
+    def __ne__(self, other):
         return not (self == other)
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
@@ -41,7 +40,7 @@ class Chat(object):
             lib.dc_chat_unref
         )
 
-    def delete(self) -> None:
+    def delete(self):
         """Delete this chat and all its messages.
 
         Note:
@@ -51,24 +50,24 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
-    def block(self) -> None:
+    def block(self):
         """Block this chat."""
         lib.dc_block_chat(self.account._dc_context, self.id)
 
-    def accept(self) -> None:
+    def accept(self):
         """Accept this contact request chat."""
         lib.dc_accept_chat(self.account._dc_context, self.id)
 
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self) -> bool:
+    def is_group(self):
         """ return true if this chat is a group chat.
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
-    def is_muted(self) -> bool:
+    def is_muted(self):
         """ return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
@@ -91,7 +90,7 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self) -> bool:
+    def can_send(self):
         """Check if messages can be sent to a give chat.
         This is not true eg. for the contact requests or for the device-talk
 
@@ -99,30 +98,30 @@ class Chat(object):
         """
         return lib.dc_chat_can_send(self._dc_chat)
 
-    def is_protected(self) -> bool:
+    def is_protected(self):
         """ return True if this chat is a protected chat.
 
         :returns: True if chat is protected, False otherwise.
         """
         return lib.dc_chat_is_protected(self._dc_chat)
 
-    def get_name(self) -> Optional[str]:
+    def get_name(self):
         """ return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
-    def set_name(self, name: str) -> bool:
+    def set_name(self, name):
         """ set name of this chat.
 
         :param name: as a unicode string.
-        :returns: True on success, False otherwise
+        :returns: None
         """
         name = as_dc_charpointer(name)
-        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
+        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
 
-    def mute(self, duration: Optional[int] = None) -> None:
+    def mute(self, duration=None):
         """ mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
@@ -136,7 +135,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
-    def unmute(self) -> None:
+    def unmute(self):
         """ unmutes the chat
 
         :returns: None
@@ -145,7 +144,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
-    def get_mute_duration(self) -> int:
+    def get_mute_duration(self):
         """ Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
@@ -153,37 +152,37 @@ class Chat(object):
         """
         return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
 
-    def get_ephemeral_timer(self) -> int:
+    def get_ephemeral_timer(self):
         """ get ephemeral timer.
 
         :returns: ephemeral timer value in seconds
         """
         return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
 
-    def set_ephemeral_timer(self, timer: int) -> bool:
+    def set_ephemeral_timer(self, timer):
         """ set ephemeral timer.
 
         :param: timer value in seconds
 
-        :returns: True on success, False otherwise
+        :returns: None
         """
-        return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
+        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
 
-    def get_type(self) -> int:
+    def get_type(self):
         """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self) -> Optional[str]:
+    def get_encryption_info(self):
         """Return encryption info for this chat.
 
         :returns: a string with encryption preferences of all chat members"""
         res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
         return from_dc_charpointer(res)
 
-    def get_join_qr(self) -> Optional[str]:
+    def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -195,7 +194,7 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
-    def send_msg(self, msg: Message) -> Message:
+    def send_msg(self, msg):
         """send a message by using a ready Message object.
 
         :param msg: a :class:`deltachat.message.Message` instance
@@ -513,9 +512,8 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.fromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i),
-                    timezone.utc
+                timestamp=datetime.utcfromtimestamp(
+                    lib.dc_array_get_timestamp(dc_array, i)
                 ),
                 marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )

python/src/deltachat/const.py

@@ -1,13 +1,7 @@
-from typing import Any, List
-
 from .capi import lib
 
 
-def __getattr__(name: str) -> Any:
+for name in dir(lib):
     if name.startswith("DC_"):
-        return getattr(lib, name)
-    return globals()[name]
-
-
-def __dir__() -> List[str]:
-    return sorted(name for name in dir(lib) if name.startswith("DC_"))
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -1,12 +1,10 @@
 """ Contact object. """
 
-from datetime import date, datetime, timezone
-from typing import Optional
-
-from . import const, props
-from .capi import ffi, lib
+from . import props
+from .cutil import from_dc_charpointer
+from .capi import lib, ffi
 from .chat import Chat
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer
+from . import const
 
 
 class Contact(object):
@@ -37,25 +35,18 @@ class Contact(object):
         )
 
     @props.with_doc
-    def addr(self) -> str:
+    def addr(self):
         """ normalized e-mail address for this account. """
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self) -> str:
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
     # deprecated alias
     display_name = name
 
-    @props.with_doc
-    def last_seen(self) -> date:
-        """Last seen timestamp."""
-        return datetime.fromtimestamp(
-            lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc
-        )
-
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -76,13 +67,15 @@ class Contact(object):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
 
-    def get_profile_image(self) -> Optional[str]:
+    def get_profile_image(self):
         """Get contact profile image.
 
         :returns: path to profile image, None if no profile image exists.
         """
         dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
-        return from_optional_dc_charpointer(dc_res)
+        if dc_res == ffi.NULL:
+            return None
+        return from_dc_charpointer(dc_res)
 
     @property
     def status(self):

python/src/deltachat/cutil.py

@@ -1,9 +1,6 @@
 from .capi import lib
 from .capi import ffi
-from datetime import datetime, timezone
-from typing import Optional, TypeVar, Generator, Callable
-
-T = TypeVar('T')
+from datetime import datetime
 
 
 def as_dc_charpointer(obj):
@@ -14,28 +11,21 @@ def as_dc_charpointer(obj):
     return obj
 
 
-def iter_array(dc_array_t, constructor: Callable[[int], T]) -> Generator[T, None, None]:
+def iter_array(dc_array_t, constructor):
     for i in range(0, lib.dc_array_get_cnt(dc_array_t)):
         yield constructor(lib.dc_array_get_id(dc_array_t, i))
 
 
-def from_dc_charpointer(obj) -> str:
+def from_dc_charpointer(obj):
     if obj != ffi.NULL:
         return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    raise ValueError
-
-
-def from_optional_dc_charpointer(obj) -> Optional[str]:
-    if obj != ffi.NULL:
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    return None
 
 
 class DCLot:
-    def __init__(self, dc_lot) -> None:
+    def __init__(self, dc_lot):
         self._dc_lot = dc_lot
 
-    def id(self) -> int:
+    def id(self):
         return lib.dc_lot_get_id(self._dc_lot)
 
     def state(self):
@@ -54,4 +44,4 @@ class DCLot:
         ts = lib.dc_lot_get_timestamp(self._dc_lot)
         if ts == 0:
             return None
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)

python/src/deltachat/direct_imap.py

@@ -11,7 +11,7 @@ from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
 import imaplib
 import deltachat
-from deltachat import const, Account
+from deltachat import const
 
 
 SEEN = b'\\Seen'
@@ -62,7 +62,7 @@ def dc_account_after_shutdown(account):
 
 
 class DirectImap:
-    def __init__(self, account: Account) -> None:
+    def __init__(self, account):
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
         self._idling = False

python/src/deltachat/events.py

@@ -9,11 +9,11 @@ from .hookspec import account_hookimpl
 from contextlib import contextmanager
 from .capi import ffi, lib
 from .message import map_system_message
-from .cutil import from_optional_dc_charpointer
+from .cutil import from_dc_charpointer
 
 
 class FFIEvent:
-    def __init__(self, name: str, data1, data2):
+    def __init__(self, name, data1, data2):
         self.name = name
         self.data1 = data1
         self.data2 = data2
@@ -29,18 +29,18 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account) -> None:
+    def __init__(self, account):
         self.account = account
         self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
+    def ac_process_ffi_event(self, ffi_event):
         self.account.log(str(ffi_event))
 
     @account_hookimpl
     def ac_log_line(self, message):
-        t = threading.current_thread()
+        t = threading.currentThread()
         tname = getattr(t, "name", t)
         if tname == "MainThread":
             tname = "MAIN"
@@ -69,7 +69,7 @@ class FFIEventTracker:
         self._event_queue = Queue()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent):
+    def ac_process_ffi_event(self, ffi_event):
         self._event_queue.put(ffi_event)
 
     def set_timeout(self, timeout):
@@ -96,7 +96,7 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_contains(self, regex: str) -> FFIEvent:
+    def get_info_contains(self, regex):
         rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
@@ -176,7 +176,6 @@ class FFIEventTracker:
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
-        return None
 
     def wait_msg_delivered(self, msg):
         ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
@@ -190,10 +189,10 @@ class EventThread(threading.Thread):
 
     With each Account init this callback thread is started.
     """
-    def __init__(self, account) -> None:
+    def __init__(self, account):
         self.account = account
         super(EventThread, self).__init__(name="events")
-        self.daemon = True
+        self.setDaemon(True)
         self._marked_for_shutdown = False
         self.start()
 
@@ -203,17 +202,17 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def mark_shutdown(self) -> None:
+    def mark_shutdown(self):
         self._marked_for_shutdown = True
 
-    def wait(self, timeout=None) -> None:
+    def wait(self, timeout=None):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
         self.join(timeout=timeout)
 
-    def run(self) -> None:
+    def run(self):
         """ get and run events until shutdown. """
         with self.log_execution("EVENT THREAD"):
             self._inner_run()
@@ -235,7 +234,7 @@ class EventThread(threading.Thread):
             # function which provides us signature info of an event call
             evt_name = deltachat.get_dc_event_name(evt)
             if lib.dc_event_has_string_data(evt):
-                data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event))
+                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
             else:
                 data2 = lib.dc_event_get_data2_int(event)
 
@@ -251,7 +250,7 @@ class EventThread(threading.Thread):
                 if self.account._dc_context is not None:
                     raise
 
-    def _map_ffi_event(self, ffi_event: FFIEvent):
+    def _map_ffi_event(self, ffi_event):
         name = ffi_event.name
         account = self.account
         if name == "DC_EVENT_CONFIGURE_PROGRESS":

python/src/deltachat/message.py

@@ -3,11 +3,10 @@
 import os
 import re
 from . import props
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer, as_dc_charpointer
+from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
 from . import const
-from datetime import datetime, timezone
-from typing import Optional
+from datetime import datetime
 
 
 class Message(object):
@@ -76,7 +75,7 @@ class Message(object):
         return lib.dc_msg_get_id(self._dc_msg)
 
     @props.with_doc
-    def text(self) -> str:
+    def text(self):
         """unicode text of this messages (might be empty if not a text message). """
         return from_dc_charpointer(lib.dc_msg_get_text(self._dc_msg))
 
@@ -85,9 +84,9 @@ class Message(object):
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
     @props.with_doc
-    def html(self) -> str:
+    def html(self):
         """html text of this messages (might be empty if not an html message). """
-        return from_optional_dc_charpointer(
+        return from_dc_charpointer(
             lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
 
     def has_html(self):
@@ -114,13 +113,12 @@ class Message(object):
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
-    def basename(self) -> str:
+    def basename(self):
         """basename of the attachment if it exists, otherwise empty string. """
-        # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 
     @props.with_doc
-    def filemime(self) -> str:
+    def filemime(self):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
@@ -132,7 +130,7 @@ class Message(object):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
-    def get_setupcodebegin(self) -> str:
+    def get_setupcodebegin(self):
         """ return the first characters of a setup code in a setup message. """
         return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
 
@@ -148,7 +146,7 @@ class Message(object):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
 
-    def get_message_info(self) -> str:
+    def get_message_info(self):
         """ Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
@@ -172,7 +170,7 @@ class Message(object):
         :returns: naive datetime.datetime() object.
         """
         ts = lib.dc_msg_get_timestamp(self._dc_msg)
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)
 
     @props.with_doc
     def time_received(self):
@@ -182,7 +180,7 @@ class Message(object):
         """
         ts = lib.dc_msg_get_received_timestamp(self._dc_msg)
         if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
+            return datetime.utcfromtimestamp(ts)
 
     @props.with_doc
     def ephemeral_timer(self):
@@ -202,14 +200,14 @@ class Message(object):
         """
         ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
         if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
+            return datetime.utcfromtimestamp(ts)
 
     @property
-    def quoted_text(self) -> Optional[str]:
+    def quoted_text(self):
         """Text inside the quote
 
         :returns: Quoted text"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
+        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
 
     @property
     def quote(self):
@@ -225,10 +223,6 @@ class Message(object):
         """Quote setter"""
         lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
 
-    def force_plaintext(self) -> None:
-        """Force the message to be sent in plain text."""
-        lib.dc_msg_force_plaintext(self._dc_msg)
-
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -246,9 +240,9 @@ class Message(object):
             return email.message_from_string(s)
 
     @property
-    def error(self) -> Optional[str]:
+    def error(self):
         """Error message"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
+        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
 
     @property
     def chat(self):
@@ -261,12 +255,12 @@ class Message(object):
         return Chat(self.account, chat_id)
 
     @props.with_doc
-    def override_sender_name(self) -> Optional[str]:
+    def override_sender_name(self):
         """the name that should be shown over the message instead of the contact display name.
 
         Usually used to impersonate someone else.
         """
-        return from_optional_dc_charpointer(
+        return from_dc_charpointer(
             lib.dc_msg_get_override_sender_name(self._dc_msg))
 
     def set_override_sender_name(self, name):

python/src/deltachat/provider.py

@@ -14,7 +14,7 @@ class Provider(object):
     :param domain: The email to get the provider info for.
     """
 
-    def __init__(self, account, addr) -> None:
+    def __init__(self, account, addr):
         provider = ffi.gc(
             lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
             lib.dc_provider_unref,
@@ -24,19 +24,19 @@ class Provider(object):
         self._provider = provider
 
     @property
-    def overview_page(self) -> str:
+    def overview_page(self):
         """URL to the overview page of the provider on providers.delta.chat."""
         return from_dc_charpointer(
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self) -> str:
+    def get_before_login_hints(self):
         """Should be shown to the user on login."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hint(self._provider))
+            lib.dc_provider_get_before_login_hints(self._provider))
 
     @property
-    def status(self) -> int:
+    def status(self):
         """The status of the provider information.
 
         This is one of the

python/src/deltachat/testplugin.py

@@ -9,7 +9,6 @@ import fnmatch
 import time
 import weakref
 import tempfile
-from typing import List, Dict, Callable
 
 import pytest
 import requests
@@ -127,7 +126,7 @@ def pytest_report_header(config, startdir):
 
 
 class SessionLiveConfigFromFile:
-    def __init__(self, fn) -> None:
+    def __init__(self, fn):
         self.fn = fn
         self.configlist = []
         for line in open(fn):
@@ -138,21 +137,19 @@ class SessionLiveConfigFromFile:
                     d[name] = value
                 self.configlist.append(d)
 
-    def get(self, index: int):
+    def get(self, index):
         return self.configlist[index]
 
-    def exists(self) -> bool:
+    def exists(self):
         return bool(self.configlist)
 
 
 class SessionLiveConfigFromURL:
-    configlist: List[Dict[str, str]]
-
-    def __init__(self, url: str) -> None:
+    def __init__(self, url):
         self.configlist = []
         self.url = url
 
-    def get(self, index: int):
+    def get(self, index):
         try:
             return self.configlist[index]
         except IndexError:
@@ -165,7 +162,7 @@ class SessionLiveConfigFromURL:
             self.configlist.append(config)
             return config
 
-    def exists(self) -> bool:
+    def exists(self):
         return bool(self.configlist)
 
 
@@ -182,7 +179,7 @@ def session_liveconfig(request):
 @pytest.fixture
 def data(request):
     class Data:
-        def __init__(self) -> None:
+        def __init__(self):
             # trying to find test data heuristically
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
@@ -213,10 +210,7 @@ def data(request):
 def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
     class AccountMaker:
-        _finalizers: List[Callable[[], None]]
-        _accounts: List[Account]
-
-        def __init__(self) -> None:
+        def __init__(self):
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
@@ -303,20 +297,21 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 self._preconfigure_key(ac, configdict['addr'])
             return ac, dict(configdict)
 
-        def get_online_configuring_account(self, sentbox=False, move=False,
+        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
                                            pre_generated_key=True, quiet=False, config={}):
             ac, configdict = self.get_online_config(
                 pre_generated_key=pre_generated_key, quiet=quiet)
             configdict.update(config)
+            configdict["mvbox_watch"] = str(int(mvbox))
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
             ac._configtracker = ac.configure()
             return ac
 
-        def get_one_online_account(self, pre_generated_key=True, move=False):
+        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key, move=move)
+                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
             self.wait_configure_and_start_io([ac1])
             return ac1
 
@@ -335,7 +330,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
-            """ Clones addr, mail_pw, mvbox_move, sentbox_watch and the
+            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
             direct_imap object of an online account. This simulates the user setting
             up a new device without importing a backup.
 
@@ -350,6 +345,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
+                mvbox_watch=account.get_config("mvbox_watch"),
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
@@ -427,7 +423,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                         pass
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
-        def get_accepted_chat(self, ac1: Account, ac2: Account):
+        def get_accepted_chat(self, ac1, ac2):
             ac2.create_chat(ac1)
             return ac1.create_chat(ac2)
 
@@ -455,9 +451,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
 
 class BotProcess:
-    stdout_queue: queue.Queue
-
-    def __init__(self, popen, bot_cfg) -> None:
+    def __init__(self, popen, bot_cfg):
         self.popen = popen
         self.addr = bot_cfg["addr"]
 
@@ -465,10 +459,10 @@ class BotProcess:
         # the (unicode) lines available for readers through a queue.
         self.stdout_queue = queue.Queue()
         self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.daemon = True
+        t.setDaemon(1)
         t.start()
 
-    def _run_stdout_thread(self) -> None:
+    def _run_stdout_thread(self):
         try:
             while 1:
                 line = self.popen.stdout.readline()
@@ -480,10 +474,10 @@ class BotProcess:
         finally:
             self.stdout_queue.put(None)
 
-    def kill(self) -> None:
+    def kill(self):
         self.popen.kill()
 
-    def wait(self, timeout=30) -> None:
+    def wait(self, timeout=30):
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
@@ -515,14 +509,14 @@ def tmp_db_path(tmpdir):
 @pytest.fixture
 def lp():
     class Printer:
-        def sec(self, msg: str) -> None:
+        def sec(self, msg):
             print()
             print("=" * 10, msg, "=" * 10)
 
-        def step(self, msg: str) -> None:
+        def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
 
-        def indent(self, msg: str) -> None:
+        def indent(self, msg):
             print("  " + msg)
 
     return Printer()

python/src/deltachat/tracker.py

@@ -90,11 +90,11 @@ class ConfigureTracker:
             if data1 is None or evdata == data1:
                 break
 
-    def wait_finish(self, timeout=None):
+    def wait_finish(self):
         """ wait until configure is completed.
 
         Raise Exception if Configure failed
         """
-        if not self._configure_events.get(timeout=timeout):
+        if not self._configure_events.get():
             content = "\n".join(map(str, self._ffi_events))
             raise ConfigureFailed(content)

python/tests/test_account.py

@@ -10,7 +10,7 @@ from deltachat.tracker import ImexTracker
 from deltachat.hookspec import account_hookimpl
 from deltachat.capi import ffi, lib
 from deltachat.cutil import iter_array
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 
 
 @pytest.mark.parametrize("msgtext,res", [
@@ -41,8 +41,8 @@ class TestOfflineAccountBasic:
     def test_wrong_db(self, tmpdir):
         p = tmpdir.join("hello.db")
         p.write("123")
-        account = Account(p.strpath)
-        assert not account.is_open()
+        with pytest.raises(ValueError):
+            Account(p.strpath)
 
     def test_os_name(self, tmpdir):
         p = tmpdir.join("hello.db")
@@ -57,7 +57,7 @@ class TestOfflineAccountBasic:
         alice_public = data.read_path("key/alice-public.asc")
         alice_secret = data.read_path("key/alice-secret.asc")
         assert alice_public and alice_secret
-        ac._preconfigure_keypair("alice@example.org", alice_public, alice_secret)
+        ac._preconfigure_keypair("alice@example.com", alice_public, alice_secret)
 
     def test_getinfo(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
@@ -265,23 +265,23 @@ class TestOfflineChat:
         assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft()
 
     def test_group_chat_creation_with_translation(self, ac1):
-        ac1.set_stock_translation(const.DC_STR_MSGGRPNAME, "abc %1$s xyz %2$s")
+        ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %1$s")
         ac1._evtracker.consume_events()
         with pytest.raises(ValueError):
-            ac1.set_stock_translation(const.DC_STR_FILE, "xyz %1$s")
-        ac1._evtracker.get_matching("DC_EVENT_WARNING")
-        with pytest.raises(ValueError):
-            ac1.set_stock_translation(const.DC_STR_CONTACT_NOT_VERIFIED, "xyz %2$s")
+            ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %2$s")
         ac1._evtracker.get_matching("DC_EVENT_WARNING")
         with pytest.raises(ValueError):
             ac1.set_stock_translation(500, "xyz %1$s")
         ac1._evtracker.get_matching("DC_EVENT_WARNING")
-        chat = ac1.create_group_chat(name="homework", contacts=[])
-        assert chat.get_name() == "homework"
-        chat.send_text("Now we have a group for homework")
-        assert chat.is_promoted()
-        chat.set_name("Homework")
-        assert chat.get_messages()[-1].text == "abc homework xyz Homework by me."
+        contact1 = ac1.create_contact("some1@example.org", name="some1")
+        contact2 = ac1.create_contact("some2@example.org", name="some2")
+        chat = ac1.create_group_chat(name="title1", contacts=[contact1, contact2])
+        assert chat.get_name() == "title1"
+        assert contact1 in chat.get_contacts()
+        assert contact2 in chat.get_contacts()
+        assert not chat.is_promoted()
+        msg = chat.get_draft()
+        assert msg.text == "xyz title1"
 
     @pytest.mark.parametrize("verified", [True, False])
     def test_group_chat_qr(self, acfactory, ac1, verified):
@@ -447,7 +447,7 @@ class TestOfflineChat:
         contact1.create_chat().send_text("hello")
 
     def test_chat_message_distinctions(self, ac1, chat1):
-        past1s = datetime.now(timezone.utc) - timedelta(seconds=1)
+        past1s = datetime.utcnow() - timedelta(seconds=1)
         msg = chat1.send_text("msg1")
         ts = msg.time_sent
         assert msg.time_received is None
@@ -737,6 +737,7 @@ class TestOnlineAccount:
         # make sure we are not sending message to ourselves
         assert self_addr not in ev.data2
         assert other_addr in ev.data2
+        ev = ac1._evtracker.get_matching("DC_EVENT_DELETED_BLOB_FILE")
 
         lp.sec("ac1: setting bcc_self=1")
         ac1.set_config("bcc_self", "1")
@@ -752,6 +753,7 @@ class TestOnlineAccount:
         # now make sure we are sending message to ourselves too
         assert self_addr in ev.data2
         assert other_addr in ev.data2
+        ev = ac1._evtracker.get_matching("DC_EVENT_DELETED_BLOB_FILE")
         assert ac1.direct_imap.idle_wait_for_seen()
 
         # Second client receives only second message, but not the first
@@ -858,7 +860,7 @@ class TestOnlineAccount:
 
     def test_mvbox_sentbox_threads(self, acfactory, lp):
         lp.sec("ac1: start with mvbox thread")
-        ac1 = acfactory.get_online_configuring_account(move=True, sentbox=True)
+        ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True, sentbox=True)
 
         lp.sec("ac2: start without mvbox/sentbox threads")
         ac2 = acfactory.get_online_configuring_account()
@@ -872,20 +874,16 @@ class TestOnlineAccount:
 
     def test_move_works(self, acfactory):
         ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account(move=True)
+        ac2 = acfactory.get_online_configuring_account(mvbox=True, move=True)
         acfactory.wait_configure_and_start_io()
         chat = acfactory.get_accepted_chat(ac1, ac2)
         chat.send_text("message1")
-
-        # Message is moved to the movebox
-        ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-
-        # Message is downloaded
         ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
         assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL
+        ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
     def test_move_works_on_self_sent(self, acfactory):
-        ac1 = acfactory.get_online_configuring_account(move=True)
+        ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True)
         ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure_and_start_io()
         ac1.set_config("bcc_self", "1")
@@ -955,7 +953,7 @@ class TestOnlineAccount:
         assert msg_in.is_forwarded()
 
     def test_send_self_message(self, acfactory, lp):
-        ac1 = acfactory.get_one_online_account(move=True)
+        ac1 = acfactory.get_one_online_account(mvbox=True, move=True)
         lp.sec("ac1: create self chat")
         chat = ac1.get_self_contact().create_chat()
         chat.send_text("hello")
@@ -975,7 +973,7 @@ class TestOnlineAccount:
         ac1._evtracker.wait_msg_delivered(msg1)
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_incoming_message()
+        msg2 = ac2._evtracker.wait_next_messages_changed()
         assert msg2.text == "message1"
         assert not msg2.is_forwarded()
         assert msg2.get_sender_contact().display_name == ac1.get_config("displayname")
@@ -985,8 +983,7 @@ class TestOnlineAccount:
         assert msg2 in chat2.get_messages()
         assert chat2.is_contact_request()
         assert chat2.count_fresh_messages() == 1
-        # Like it or not, this assert is flaky
-        # assert msg2.time_received >= msg1.time_sent
+        assert msg2.time_received >= msg1.time_sent
 
         lp.sec("create new chat with contact and verify it's proper")
         chat2b = msg2.create_chat()
@@ -1037,70 +1034,31 @@ class TestOnlineAccount:
 
     def test_moved_markseen(self, acfactory, lp):
         """Test that message already moved to DeltaChat folder is marked as seen."""
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account(move=True)
+        ac1 = acfactory.get_online_configuring_account(mvbox=True, config={"inbox_watch": "0"})
+        ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure_and_start_io([ac1, ac2])
+        ac1.set_config("bcc_self", "1")
 
-        ac2.stop_io()
-        ac2.direct_imap.idle_start()
-
+        ac1.direct_imap.idle_start()
         ac1.create_chat(ac2).send_text("Hello!")
-
-        # Wait for the message to arrive.
-        ac2.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.stop_io()
 
         # Emulate moving of the message to DeltaChat folder by Sieve rule.
         # mailcow server contains this rule by default.
-        ac2.direct_imap.conn.move(["*"], "DeltaChat")
+        ac1.direct_imap.conn.move(["*"], "DeltaChat")
 
-        ac2.direct_imap.select_folder("DeltaChat")
-        ac2.direct_imap.idle_start()
-        ac2.start_io()
-        msg = ac2._evtracker.wait_next_incoming_message()
+        ac1.direct_imap.select_folder("DeltaChat")
+        ac1.direct_imap.idle_start()
+        ac1.start_io()
+        ac1.direct_imap.idle_wait_for_seen()
+        ac1.direct_imap.idle_done()
 
-        # Accept the contact request.
-        msg.chat.accept()
-        ac2.mark_seen_messages([msg])
-        ac2.direct_imap.idle_wait_for_seen()
-        ac2.direct_imap.idle_done()
-
-        fetch = list(ac2.direct_imap.conn.fetch("*", b'FLAGS').values())
+        fetch = list(ac1.direct_imap.conn.fetch("*", b'FLAGS').values())
         flags = fetch[-1][b'FLAGS']
         is_seen = b'\\Seen' in flags
         assert is_seen
 
-    def test_multidevice_sync_seen(self, acfactory, lp):
-        """Test that message marked as seen on one device is marked as seen on another."""
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        ac1_clone = acfactory.clone_online_account(ac1)
-        acfactory.wait_configure_and_start_io()
-
-        ac1.set_config("bcc_self", "1")
-        ac1_clone.set_config("bcc_self", "1")
-
-        ac1_chat = ac1.create_chat(ac2)
-        ac1_clone_chat = ac1_clone.create_chat(ac2)
-        ac2_chat = ac2.create_chat(ac1)
-
-        lp.sec("Send a message from ac2 to ac1 and check that it's 'fresh'")
-        ac2_chat.send_text("Hi")
-        ac1_message = ac1._evtracker.wait_next_incoming_message()
-        ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
-        assert ac1_chat.count_fresh_messages() == 1
-        assert ac1_clone_chat.count_fresh_messages() == 1
-        assert ac1_message.is_in_fresh
-        assert ac1_clone_message.is_in_fresh
-
-        lp.sec("ac1 marks message as seen on the first device")
-        ac1.mark_seen_messages([ac1_message])
-        assert ac1_message.is_in_seen
-
-        lp.sec("ac1 clone detects that message is marked as seen")
-        ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
-        assert ev.data1 == ac1_clone_chat.id
-        assert ac1_clone_message.is_in_seen
-
     def test_message_override_sender_name(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         chat = acfactory.get_accepted_chat(ac1, ac2)
@@ -1137,8 +1095,8 @@ class TestOnlineAccount:
     def test_markseen_message_and_mdn(self, acfactory, mvbox_move):
         # Please only change this test if you are very sure that it will still catch the issues it catches now.
         # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
-        ac1 = acfactory.get_online_configuring_account(move=mvbox_move)
-        ac2 = acfactory.get_online_configuring_account(move=mvbox_move)
+        ac1 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
 
         acfactory.wait_configure_and_start_io()
         # Do not send BCC to self, we only want to test MDN on ac1.
@@ -1167,7 +1125,7 @@ class TestOnlineAccount:
         group1.add_contact(ac2)
         group1.send_text("hello")
 
-        msg2 = ac2._evtracker.wait_next_incoming_message()
+        msg2 = ac2._evtracker.wait_next_messages_changed()
         group2 = msg2.create_chat()
         assert group2.get_name() == group1.get_name()
 
@@ -1184,7 +1142,7 @@ class TestOnlineAccount:
         assert not msg_reply1.chat.is_group()
         assert msg_reply1.chat.id == private_chat1.id
 
-    def test_mdn_asymmetric(self, acfactory, lp):
+    def test_mdn_asymetric(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts(move=True)
 
         lp.sec("ac1: create chat with ac2")
@@ -1208,9 +1166,6 @@ class TestOnlineAccount:
 
         assert len(msg.chat.get_messages()) == 1
 
-        ac1.direct_imap.select_config_folder("mvbox")
-        ac1.direct_imap.idle_start()
-
         lp.sec("ac2: mark incoming message as seen")
         ac2.mark_seen_messages([msg])
 
@@ -1220,9 +1175,6 @@ class TestOnlineAccount:
 
         assert len(chat.get_messages()) == 1
 
-        # Wait for the message to be marked as seen on IMAP.
-        assert ac1.direct_imap.idle_wait_for_seen()
-
         # MDN is received even though MDNs are already disabled
         assert msg_out.is_out_mdn_received()
 
@@ -1236,7 +1188,7 @@ class TestOnlineAccount:
         chat.send_text("message1")
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_incoming_message()
+        msg2 = ac2._evtracker.wait_next_messages_changed()
         assert msg2.text == "message1"
 
         lp.sec("create new chat with contact and send back (encrypted) message")
@@ -1269,40 +1221,6 @@ class TestOnlineAccount:
         assert not msg.is_encrypted()
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
-    def test_gossip_optimization(self, acfactory, lp):
-        """Test that gossip timestamp is updated when someone else sends gossip,
-        so we don't have to send gossip ourselves.
-        """
-        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
-
-        acfactory.introduce_each_other([ac1, ac2])
-        acfactory.introduce_each_other([ac2, ac3])
-
-        lp.sec("ac1 creates a group chat with ac2")
-        group_chat = ac1.create_group_chat("hello")
-        group_chat.add_contact(ac2)
-        msg = group_chat.send_text("hi")
-
-        # No Autocrypt gossip was sent yet.
-        gossiped_timestamp = msg.chat.get_summary()["gossiped_timestamp"]
-        assert gossiped_timestamp == 0
-
-        msg = ac2._evtracker.wait_next_incoming_message()
-        assert msg.is_encrypted()
-        assert msg.text == "hi"
-
-        lp.sec("ac2 adds ac3 to the group")
-        msg.chat.add_contact(ac3)
-
-        lp.sec("ac1 receives message from ac2 and updates gossip timestamp")
-        msg = ac1._evtracker.wait_next_incoming_message()
-        assert msg.is_encrypted()
-
-        # ac1 has updated the gossip timestamp even though no gossip was sent by ac1.
-        # ac1 does not need to send gossip because ac2 already did it.
-        gossiped_timestamp = msg.chat.get_summary()["gossiped_timestamp"]
-        assert gossiped_timestamp == int(msg.time_sent.timestamp())
-
     def test_gossip_encryption_preference(self, acfactory, lp):
         """Test that encryption preference of group members is gossiped to new members.
         This is a Delta Chat extension to Autocrypt 1.1.0, which Autocrypt-Gossip headers
@@ -1407,7 +1325,7 @@ class TestOnlineAccount:
         If the draft email is sent out later (i.e. moved to "Sent"), it must be shown."""
         ac1 = acfactory.get_online_configuring_account()
         ac1.set_config("show_emails", "2")
-        ac1.create_contact("alice@example.org").create_chat()
+        ac1.create_contact("alice@example.com").create_chat()
 
         acfactory.wait_configure(ac1)
         ac1.direct_imap.create_folder("Drafts")
@@ -1421,7 +1339,7 @@ class TestOnlineAccount:
         ac1.direct_imap.append("Drafts", """
             From: ac1 <{}>
             Subject: subj
-            To: alice@example.org
+            To: alice@example.com
             Message-ID: <aepiors@example.org>
             Content-Type: text/plain; charset=utf-8
 
@@ -1430,7 +1348,7 @@ class TestOnlineAccount:
         ac1.direct_imap.append("Sent", """
             From: ac1 <{}>
             Subject: subj
-            To: alice@example.org
+            To: alice@example.com
             Message-ID: <hsabaeni@example.org>
             Content-Type: text/plain; charset=utf-8
 
@@ -1461,35 +1379,6 @@ class TestOnlineAccount:
         assert msg2.text == "subj – message in Drafts that is moved to Sent later"
         assert len(msg.chat.get_messages()) == 2
 
-    def test_no_old_msg_is_fresh(self, acfactory, lp):
-        ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account()
-        ac1_clone = acfactory.clone_online_account(ac1)
-        acfactory.wait_configure_and_start_io()
-
-        ac1.set_config("e2ee_enabled", "0")
-        ac1_clone.set_config("e2ee_enabled", "0")
-        ac2.set_config("e2ee_enabled", "0")
-
-        ac1_clone.set_config("bcc_self", "1")
-
-        ac1.create_chat(ac2)
-        ac1_clone.create_chat(ac2)
-
-        lp.sec("Send a first message from ac2 to ac1 and check that it's 'fresh'")
-        first_msg_id = ac2.create_chat(ac1).send_text("Hi")
-        ac1._evtracker.wait_next_incoming_message()
-        assert ac1.create_chat(ac2).count_fresh_messages() == 1
-        assert len(list(ac1.get_fresh_messages())) == 1
-
-        lp.sec("Send a message from ac1_clone to ac2 and check that ac1 marks the first message as 'noticed'")
-        ac1_clone.create_chat(ac2).send_text("Hi back")
-        ev = ac1._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
-
-        assert ev.data1 == first_msg_id.chat.id
-        assert ac1.create_chat(ac2).count_fresh_messages() == 0
-        assert len(list(ac1.get_fresh_messages())) == 0
-
     def test_prefer_encrypt(self, acfactory, lp):
         """Test quorum rule for encryption preference in 1:1 and group chat."""
         ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
@@ -1579,7 +1468,7 @@ class TestOnlineAccount:
         assert not msg1.is_encrypted()
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_incoming_message()
+        msg2 = ac2._evtracker.wait_next_messages_changed()
         assert msg2.text == "message1"
         assert not msg2.is_encrypted()
 
@@ -1628,7 +1517,7 @@ class TestOnlineAccount:
         chat1.send_text("hi")
 
         lp.sec("ac2 receives contact request from ac1")
-        received_message = ac2._evtracker.wait_next_incoming_message()
+        received_message = ac2._evtracker.wait_next_messages_changed()
         assert received_message.text == "hi"
 
         basename = "attachment.txt"
@@ -1663,7 +1552,7 @@ class TestOnlineAccount:
         assert msg_out.get_mime_headers() is None
 
         lp.sec("wait for ac2 to receive message")
-        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
+        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
         in_id = ev.data2
         mime = ac2.get_message_by_id(in_id).get_mime_headers()
         assert mime.get_all("From")
@@ -1962,7 +1851,7 @@ class TestOnlineAccount:
         ac1.create_chat(ac2).send_text("with avatar!")
 
         lp.sec("ac2: wait for receiving message and avatar from ac1")
-        msg2 = ac2._evtracker.wait_next_incoming_message()
+        msg2 = ac2._evtracker.wait_next_messages_changed()
         assert msg2.chat.is_contact_request()
         received_path = msg2.get_sender_contact().get_profile_image()
         assert open(received_path, "rb").read() == open(p, "rb").read()
@@ -2154,8 +2043,10 @@ class TestOnlineAccount:
         assert msg_back.chat == chat
         assert chat.get_profile_image() is None
 
-    def test_connectivity(self, acfactory, lp):
+    @pytest.mark.parametrize("inbox_watch", ["0", "1"])
+    def test_connectivity(self, acfactory, lp, inbox_watch):
         ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1.set_config("inbox_watch", inbox_watch)
         ac1.set_config("scan_all_folders_debounce_secs", "0")
 
         ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTED)
@@ -2272,7 +2163,7 @@ class TestOnlineAccount:
                 break  # DC is done with reading messages
 
     def test_send_receive_locations(self, acfactory, lp):
-        now = datetime.now(timezone.utc)
+        now = datetime.utcnow()
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         lp.sec("ac1: create chat with ac2")
@@ -2395,7 +2286,7 @@ class TestOnlineAccount:
 
     def test_immediate_autodelete(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account(move=False, sentbox=False)
+        ac2 = acfactory.get_online_configuring_account(mvbox=False, move=False, sentbox=False)
 
         # "1" means delete immediately, while "0" means do not delete
         ac2.set_config("delete_server_after", "1")
@@ -2656,26 +2547,31 @@ class TestOnlineAccount:
         assert received_reply.quoted_text == "hello"
         assert received_reply.quote.id == out_msg.id
 
-    @pytest.mark.parametrize("folder,move,expected_destination,", [
-        ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
-        ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
-        ("Spam", False, "INBOX"),  # ...emails are moved from the spam folder to the Inbox
+    @pytest.mark.parametrize("folder,move,expected_destination,inbox_watch,", [
+        ("xyz", False, "xyz", "1"),  # Test that emails are recognized in a random folder but not moved
+        ("xyz", True, "DeltaChat", "1"),  # ...emails are found in a random folder and moved to DeltaChat
+        ("Spam", False, "INBOX", "1"),  # ...emails are moved from the spam folder to the Inbox
+        ("INBOX", False, "INBOX", "0"),  # ...emails are found in the `Inbox` folder even if `inbox_watch` is "0"
     ])
     # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
     # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
-    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination):
+    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination, inbox_watch):
         """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
         variant = folder + "-" + str(move) + "-" + expected_destination
         lp.sec("Testing variant " + variant)
         ac1 = acfactory.get_online_configuring_account(move=move)
         ac2 = acfactory.get_online_configuring_account()
+        ac1.set_config("inbox_watch", inbox_watch)
 
         acfactory.wait_configure(ac1)
         ac1.direct_imap.create_folder(folder)
 
         acfactory.wait_configure_and_start_io()
         # Wait until each folder was selected once and we are IDLEing:
-        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        if inbox_watch == "1":
+            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        else:
+            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
         ac1.stop_io()
 
         # Send a message to ac1 and move it to the mvbox:
@@ -2691,7 +2587,11 @@ class TestOnlineAccount:
         msg = ac1._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        # The message has been downloaded, which means it has reached its destination.
+        # Wait until the message was moved (if at all) and we are IDLEing again:
+        if inbox_watch == "1":
+            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        else:
+            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
         ac1.direct_imap.select_folder(expected_destination)
         assert len(ac1.direct_imap.get_all_messages()) == 1
         if folder != expected_destination:
@@ -2714,7 +2614,7 @@ class TestOnlineAccount:
             if mvbox_move:
                 assert ac.get_config("configured_mvbox_folder")
 
-        ac1 = acfactory.get_online_configuring_account(move=mvbox_move)
+        ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
         ac1.set_config("sentbox_move", "1")
         ac2 = acfactory.get_online_configuring_account()
 
@@ -2781,7 +2681,7 @@ class TestOnlineAccount:
 
         lp.sec("receive a message")
         ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
-        ac1._evtracker.wait_next_incoming_message()
+        ac1._evtracker.wait_next_messages_changed()
 
         lp.sec("send out message with bcc to ourselves")
         ac1.direct_imap.idle_start()
@@ -2812,22 +2712,6 @@ class TestOnlineAccount:
         # We can't decrypt the message in this chat, so the chat is empty:
         assert len(private_messages) == 0
 
-    def test_delete_deltachat_folder(self, acfactory):
-        """Test that DeltaChat folder is recreated if user deletes it manually."""
-        ac1 = acfactory.get_online_configuring_account(move=True)
-        ac2 = acfactory.get_online_configuring_account()
-        acfactory.wait_configure(ac1)
-
-        ac1.direct_imap.conn.delete_folder("DeltaChat")
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 0
-        acfactory.wait_configure_and_start_io()
-
-        ac2.create_chat(ac1).send_text("hello")
-        msg = ac1._evtracker.wait_next_incoming_message()
-        assert msg.text == "hello"
-
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 1
-
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):
@@ -2942,8 +2826,7 @@ class TestGroupStressTests:
         lp.sec("ac2: check that ac3 is removed")
         msg = ac2._evtracker.wait_next_incoming_message()
 
-        assert chat.num_contacts() == 2
-        assert msg.chat.num_contacts() == 2
+        assert msg.chat.num_contacts() == chat.num_contacts()
         acfactory.dump_imap_summary(sys.stdout)
 
 

python/tests/test_increation.py

@@ -6,6 +6,8 @@ import shutil
 import pytest
 from filecmp import cmp
 
+from deltachat import const
+
 
 def wait_msg_delivered(account, msg_list):
     """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
@@ -100,10 +102,14 @@ class TestOnlineInCreation:
         ])
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        received_original = ac2._evtracker.wait_next_incoming_message()
+        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1.data2)
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        received_copy = ac2._evtracker.wait_next_incoming_message()
-        assert received_copy.id != received_original.id
+        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2.data1 != ev1.data1
+        received_copy = ac2.get_message_by_id(ev2.data2)
         assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_lowlevel.py

@@ -36,8 +36,7 @@ def test_wrong_db(tmpdir):
     # write an invalid database file
     p.write("x123" * 10)
 
-    context = lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
-    assert not lib.dc_context_is_open(context)
+    assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
 
 
 def test_empty_blobdir(tmpdir):

python/tox.ini

@@ -1,9 +1,7 @@
 [tox]
-isolated_build = true
 envlist =
     py3
     lint
-    mypy
     auditwheels
 
 [testenv]
@@ -44,20 +42,12 @@ commands =
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
-[testenv:mypy]
-deps =
-    mypy
-    typing
-    types-setuptools
-    types-requests
-commands =
-    mypy --no-incremental src/
-
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx
-    breathe
+    # Pin dependencies to the versions which actually work with Python 3.5.
+    sphinx==3.4.3
+    breathe==4.28.0
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -2,13 +2,13 @@
 
 set -x -e
 
-# we use the python3.7 environment as the base environment 
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel 
+# we use the python3.6 environment as the base environment 
+/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel 
 
 pushd /usr/bin
 
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
+ln -s /opt/_internal/cpython-3.6.*/bin/tox
+ln -s /opt/_internal/cpython-3.6.*/bin/devpi
+ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
 
 popd

scripts/docker-coredeps/deps/build_python.sh

@@ -2,13 +2,13 @@
 
 set -x -e
 
-# we use the python3.7 environment as the base environment
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel
+# we use the python3.6 environment as the base environment
+/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel
 
 pushd /usr/bin
 
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
+ln -s /opt/_internal/cpython-3.6.*/bin/tox
+ln -s /opt/_internal/cpython-3.6.*/bin/devpi
+ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
 
 popd

scripts/run_all.sh

@@ -19,9 +19,11 @@ export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp37-cp37m/bin
+export PATH=$PATH:/opt/python/cp36-cp36m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
+rm -f python3.6
+ln -s /opt/python/cp36-cp36m/bin/python3.6
 rm -f python3.7
 ln -s /opt/python/cp37-cp37m/bin/python3.7
 rm -f python3.8

scripts/set_core_version.py

@@ -86,8 +86,6 @@ def main():
     print("")
     print("   git tag -a {}".format(newversion))
     print("   git push origin {}".format(newversion))
-    print("   git tag -a py-{}".format(newversion))
-    print("   git push origin py-{}".format(newversion))
     print("")
 
 

src/accounts.rs

@@ -13,7 +13,7 @@ use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
-use crate::events::{Event, EventType, Events};
+use crate::events::Event;
 
 /// Account manager, that can handle multiple accounts in a single place.
 #[derive(Debug)]
@@ -23,27 +23,31 @@ pub struct Accounts {
     accounts: BTreeMap<u32, Context>,
     emitter: EventEmitter,
 
-    /// Event channel to emit account manager errors.
-    events: Events,
+    /// Sender side of the fake event channel.
+    ///
+    /// We never send any events over this channel, but hold it during the account manager lifetime
+    /// to prevent `EventEmitter` from returning `None` as long as account manager is alive, even if
+    /// it holds no accounts which could emit events.
+    fake_sender: Sender<crate::events::Event>,
 }
 
 impl Accounts {
     /// Loads or creates an accounts folder at the given `dir`.
-    pub async fn new(dir: PathBuf) -> Result<Self> {
+    pub async fn new(os_name: String, dir: PathBuf) -> Result<Self> {
         if !dir.exists().await {
-            Accounts::create(&dir).await?;
+            Accounts::create(os_name, &dir).await?;
         }
 
         Accounts::open(dir).await
     }
 
     /// Creates a new default structure.
-    pub async fn create(dir: &PathBuf) -> Result<()> {
+    pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
         fs::create_dir_all(dir)
             .await
             .context("failed to create folder")?;
 
-        Config::new(dir).await?;
+        Config::new(os_name.clone(), dir).await?;
 
         Ok(())
     }
@@ -56,24 +60,17 @@ impl Accounts {
         let config_file = dir.join(CONFIG_NAME);
         ensure!(config_file.exists().await, "accounts.toml does not exist");
 
-        let config = Config::from_file(config_file)
-            .await
-            .context("failed to load accounts config")?;
-        let accounts = config
-            .load_accounts()
-            .await
-            .context("failed to load accounts")?;
+        let config = Config::from_file(config_file).await?;
+        let accounts = config.load_accounts().await?;
 
         let emitter = EventEmitter::new();
 
-        let events = Events::default();
-
-        emitter.sender.send(events.get_emitter()).await?;
+        // Fake event stream to prevent event emitter from closing.
+        let (fake_sender, fake_receiver) = channel::bounded(1);
+        emitter.sender.send(fake_receiver).await?;
 
         for account in accounts.values() {
-            emitter.add_account(account).await.with_context(|| {
-                format!("failed to add account {} to event emitter", account.id)
-            })?;
+            emitter.add_account(account).await?;
         }
 
         Ok(Self {
@@ -81,7 +78,7 @@ impl Accounts {
             config,
             accounts,
             emitter,
-            events,
+            fake_sender,
         })
     }
 
@@ -111,24 +108,12 @@ impl Accounts {
         Ok(())
     }
 
-    /// Add a new account and opens it.
-    ///
-    /// Returns account ID.
+    /// Add a new account.
     pub async fn add_account(&mut self) -> Result<u32> {
+        let os_name = self.config.os_name().await;
         let account_config = self.config.new_account(&self.dir).await?;
 
-        let ctx = Context::new(account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
-        self.accounts.insert(account_config.id, ctx);
-
-        Ok(account_config.id)
-    }
-
-    /// Adds a new closed account.
-    pub async fn add_closed_account(&mut self) -> Result<u32> {
-        let account_config = self.config.new_account(&self.dir).await?;
-
-        let ctx = Context::new_closed(account_config.dbfile().into(), account_config.id).await?;
+        let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
         self.emitter.add_account(&ctx).await?;
         self.accounts.insert(account_config.id, ctx);
 
@@ -202,7 +187,13 @@ impl Accounts {
 
         match res {
             Ok(_) => {
-                let ctx = Context::new(new_dbfile, account_config.id).await?;
+                let ctx = Context::with_blobdir(
+                    self.config.os_name().await,
+                    new_dbfile,
+                    new_blobdir,
+                    account_config.id,
+                )
+                .await?;
                 self.emitter.add_account(&ctx).await?;
                 self.accounts.insert(account_config.id, ctx);
                 Ok(account_config.id)
@@ -271,11 +262,6 @@ impl Accounts {
         }
     }
 
-    /// Emits a single event.
-    pub fn emit_event(&self, event: EventType) {
-        self.events.emit(Event { id: 0, typ: event })
-    }
-
     /// Returns unified event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
         self.emitter.clone()
@@ -286,13 +272,13 @@ impl Accounts {
 #[derive(Debug, Clone)]
 pub struct EventEmitter {
     /// Aggregate stream of events from all accounts.
-    stream: Arc<RwLock<futures::stream::SelectAll<crate::events::EventEmitter>>>,
+    stream: Arc<RwLock<futures::stream::SelectAll<Receiver<crate::events::Event>>>>,
 
     /// Sender for the channel where new account emitters will be pushed.
-    sender: Sender<crate::events::EventEmitter>,
+    sender: Sender<Receiver<crate::events::Event>>,
 
     /// Receiver for the channel where new account emitters will be pushed.
-    receiver: Receiver<crate::events::EventEmitter>,
+    receiver: Receiver<Receiver<crate::events::Event>>,
 }
 
 impl EventEmitter {
@@ -325,7 +311,9 @@ impl EventEmitter {
 
     /// Add event emitter of a new account to the aggregate event emitter.
     pub async fn add_account(&self, context: &Context) -> Result<()> {
-        self.sender.send(context.get_event_emitter()).await?;
+        self.sender
+            .send(context.get_event_emitter().into_inner())
+            .await?;
         Ok(())
     }
 }
@@ -362,6 +350,7 @@ pub struct Config {
 /// This is serialized into TOML.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 struct InnerConfig {
+    pub os_name: String,
     /// The currently selected account.
     pub selected_account: u32,
     pub next_id: u32,
@@ -369,8 +358,9 @@ struct InnerConfig {
 }
 
 impl Config {
-    pub async fn new(dir: &PathBuf) -> Result<Self> {
+    pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
         let inner = InnerConfig {
+            os_name,
             accounts: Vec::new(),
             selected_account: 0,
             next_id: 1,
@@ -385,6 +375,10 @@ impl Config {
         Ok(cfg)
     }
 
+    pub async fn os_name(&self) -> String {
+        self.inner.os_name.clone()
+    }
+
     /// Sync the inmemory representation to disk.
     async fn sync(&self) -> Result<()> {
         fs::write(&self.file, toml::to_string_pretty(&self.inner)?)
@@ -403,15 +397,12 @@ impl Config {
     pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
         let mut accounts = BTreeMap::new();
         for account_config in &self.inner.accounts {
-            let ctx = Context::new(account_config.dbfile().into(), account_config.id)
-                .await
-                .with_context(|| {
-                    format!(
-                        "failed to create context from file {:?}",
-                        account_config.dbfile()
-                    )
-                })?;
-
+            let ctx = Context::new(
+                self.inner.os_name.clone(),
+                account_config.dbfile().into(),
+                account_config.id,
+            )
+            .await?;
             accounts.insert(account_config.id, ctx);
         }
 
@@ -436,13 +427,8 @@ impl Config {
 
         self.sync().await?;
 
-        self.select_account(id)
-            .await
-            .context("failed to select just added account")?;
-        let cfg = self
-            .get_account(id)
-            .await
-            .context("failed to get just added account")?;
+        self.select_account(id).await.expect("just added");
+        let cfg = self.get_account(id).await.expect("just added");
         Ok(cfg)
     }
 
@@ -513,7 +499,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts1").into();
 
-        let mut accounts1 = Accounts::new(p.clone()).await.unwrap();
+        let mut accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
         accounts1.add_account().await.unwrap();
 
         let accounts2 = Accounts::open(p).await.unwrap();
@@ -531,7 +517,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new(p.clone()).await.unwrap();
+        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
         assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
@@ -558,7 +544,7 @@ mod tests {
         let dir = tempfile::tempdir()?;
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new(p.clone()).await?;
+        let mut accounts = Accounts::new("my_os".into(), p.clone()).await?;
         assert!(accounts.get_selected_account().await.is_none());
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
@@ -579,12 +565,14 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new(p.clone()).await.unwrap();
+        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
         assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
         let extern_dbfile: PathBuf = dir.path().join("other").into();
-        let ctx = Context::new(extern_dbfile.clone(), 0).await.unwrap();
+        let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
+            .await
+            .unwrap();
         ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
             .await
             .unwrap();
@@ -614,7 +602,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new(p.clone()).await.unwrap();
+        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
         for expected_id in 1..10 {
             let id = accounts.add_account().await.unwrap();
@@ -634,7 +622,7 @@ mod tests {
         let dummy_accounts = 10;
 
         let (id0, id1, id2) = {
-            let mut accounts = Accounts::new(p.clone()).await?;
+            let mut accounts = Accounts::new("my_os".into(), p.clone()).await?;
             accounts.add_account().await?;
             let ids = accounts.get_all().await;
             assert_eq!(ids.len(), 1);
@@ -669,7 +657,7 @@ mod tests {
         assert!(id2 > id1 + dummy_accounts);
 
         let (id0_reopened, id1_reopened, id2_reopened) = {
-            let accounts = Accounts::new(p.clone()).await?;
+            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
             let ctx = accounts.get_selected_account().await.unwrap();
             assert_eq!(
                 ctx.get_config(crate::config::Config::Addr).await?,
@@ -714,7 +702,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new(p.clone()).await?;
+        let accounts = Accounts::new("my_os".into(), p.clone()).await?;
 
         // Make sure there are no accounts.
         assert_eq!(accounts.accounts.len(), 0);
@@ -734,49 +722,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_encrypted_account() -> Result<()> {
-        let dir = tempfile::tempdir().context("failed to create tempdir")?;
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let mut accounts = Accounts::new(p.clone())
-            .await
-            .context("failed to create accounts manager")?;
-
-        assert_eq!(accounts.accounts.len(), 0);
-        let account_id = accounts
-            .add_closed_account()
-            .await
-            .context("failed to add closed account")?;
-        let account = accounts
-            .get_selected_account()
-            .await
-            .context("failed to get account")?;
-        assert_eq!(account.id, account_id);
-        let passphrase_set_success = account
-            .open("foobar".to_string())
-            .await
-            .context("failed to set passphrase")?;
-        assert!(passphrase_set_success);
-        drop(accounts);
-
-        let accounts = Accounts::new(p.clone())
-            .await
-            .context("failed to create second accounts manager")?;
-        let account = accounts
-            .get_selected_account()
-            .await
-            .context("failed to get account")?;
-        assert_eq!(account.is_open().await, false);
-
-        // Try wrong passphrase.
-        assert_eq!(account.open("barfoo".to_string()).await?, false);
-        assert_eq!(account.open("".to_string()).await?, false);
-
-        assert_eq!(account.open("foobar".to_string()).await?, true);
-        assert_eq!(account.is_open().await, true);
-
-        Ok(())
-    }
 }

src/aheader.rs

@@ -2,12 +2,12 @@
 //!
 //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
 
-use anyhow::{bail, Context as _, Error, Result};
 use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
 use crate::contact::addr_cmp;
+use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};
 
@@ -37,13 +37,13 @@ impl fmt::Display for EncryptPreference {
 }
 
 impl str::FromStr for EncryptPreference {
-    type Err = Error;
+    type Err = ();
 
-    fn from_str(s: &str) -> Result<Self> {
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "mutual" => Ok(EncryptPreference::Mutual),
             "nopreference" => Ok(EncryptPreference::NoPreference),
-            _ => bail!("Cannot parse encryption preference {}", s),
+            _ => Err(()),
         }
     }
 }
@@ -70,27 +70,28 @@ impl Aheader {
         }
     }
 
-    /// Tries to parse Autocrypt header.
-    ///
-    /// If there is none, returns None. If the header is present but cannot be parsed, returns an
-    /// error.
     pub fn from_headers(
+        context: &Context,
         wanted_from: &str,
         headers: &[mailparse::MailHeader<'_>],
-    ) -> Result<Option<Self>> {
+    ) -> Option<Self> {
         if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) {
-            let header = Self::from_str(&value)?;
-            if !addr_cmp(&header.addr, wanted_from) {
-                bail!(
-                    "Autocrypt header address {:?} is not {:?}",
-                    header.addr,
-                    wanted_from
-                );
+            match Self::from_str(&value) {
+                Ok(header) => {
+                    if addr_cmp(&header.addr, wanted_from) {
+                        return Some(header);
+                    }
+                }
+                Err(err) => {
+                    warn!(
+                        context,
+                        "found invalid autocrypt header {}: {:?}", value, err
+                    );
+                }
             }
-            Ok(Some(header))
-        } else {
-            Ok(None)
         }
+
+        None
     }
 }
 
@@ -119,9 +120,9 @@ impl fmt::Display for Aheader {
 }
 
 impl str::FromStr for Aheader {
-    type Err = Error;
+    type Err = ();
 
-    fn from_str(s: &str) -> Result<Self> {
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
         let mut attributes: BTreeMap<String, String> = s
             .split(';')
             .filter_map(|a| {
@@ -135,19 +136,15 @@ impl str::FromStr for Aheader {
 
         let addr = match attributes.remove("addr") {
             Some(addr) => addr,
-            None => bail!("Autocrypt header has no addr"),
+            None => {
+                return Err(());
+            }
         };
         let public_key: SignedPublicKey = attributes
             .remove("keydata")
-            .context("keydata attribute is not found")
-            .and_then(|raw| {
-                SignedPublicKey::from_base64(&raw).context("autocrypt key cannot be decoded")
-            })
-            .and_then(|key| {
-                key.verify()
-                    .and(Ok(key))
-                    .context("autocrypt key cannot be verified")
-            })?;
+            .ok_or(())
+            .and_then(|raw| SignedPublicKey::from_base64(&raw).or(Err(())))
+            .and_then(|key| key.verify().and(Ok(key)).or(Err(())))?;
 
         let prefer_encrypt = attributes
             .remove("prefer-encrypt")
@@ -157,7 +154,7 @@ impl str::FromStr for Aheader {
         // Autocrypt-Level0: unknown attributes starting with an underscore can be safely ignored
         // Autocrypt-Level0: unknown attribute, treat the header as invalid
         if attributes.keys().any(|k| !k.starts_with('_')) {
-            bail!("Unknown Autocrypt attribute found");
+            return Err(());
         }
 
         Ok(Aheader {
@@ -175,40 +172,38 @@ mod tests {
     const RAWKEY: &str = "xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=";
 
     #[test]
-    fn test_from_str() -> Result<()> {
+    fn test_from_str() {
         let h: Aheader = format!(
             "addr=me@mail.com; prefer-encrypt=mutual; keydata={}",
             RAWKEY
         )
-        .parse()?;
+        .parse()
+        .expect("failed to parse");
 
         assert_eq!(h.addr, "me@mail.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::Mutual);
-        Ok(())
     }
 
     // EncryptPreference::Reset is an internal value, parser should never return it
     #[test]
-    fn test_from_str_reset() -> Result<()> {
+    fn test_from_str_reset() {
         let raw = format!(
             "addr=reset@example.com; prefer-encrypt=reset; keydata={}",
             RAWKEY
         );
-        let h: Aheader = raw.parse()?;
+        let h: Aheader = raw.parse().expect("failed to parse");
 
         assert_eq!(h.addr, "reset@example.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
-        Ok(())
     }
 
     #[test]
-    fn test_from_str_non_critical() -> Result<()> {
+    fn test_from_str_non_critical() {
         let raw = format!("addr=me@mail.com; _foo=one; _bar=two; keydata={}", RAWKEY);
-        let h: Aheader = raw.parse()?;
+        let h: Aheader = raw.parse().expect("failed to parse");
 
         assert_eq!(h.addr, "me@mail.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
-        Ok(())
     }
 
     #[test]
@@ -221,7 +216,7 @@ mod tests {
     }
 
     #[test]
-    fn test_good_headers() -> Result<()> {
+    fn test_good_headers() {
         let fixed_header = concat!(
             "addr=a@b.example.org; prefer-encrypt=mutual; ",
             "keydata=xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJg",
@@ -247,7 +242,7 @@ mod tests {
             " wm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc="
         );
 
-        let ah = Aheader::from_str(fixed_header)?;
+        let ah = Aheader::from_str(fixed_header).expect("failed to parse");
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
         assert_eq!(format!("{}", ah), fixed_header);
@@ -255,17 +250,18 @@ mod tests {
         let rendered = ah.to_string();
         assert_eq!(rendered, fixed_header);
 
-        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY))?;
+        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY)).expect("failed to parse");
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
 
         Aheader::from_str(&format!(
             "addr=a@b.example.org; prefer-encrypt=ignoreUnknownValues; keydata={}",
             RAWKEY
-        ))?;
+        ))
+        .expect("failed to parse");
 
-        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))?;
-        Ok(())
+        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))
+            .expect("failed to parse");
     }
 
     #[test]

src/blob.rs

@@ -24,7 +24,6 @@ use crate::constants::{
 };
 use crate::context::Context;
 use crate::events::EventType;
-use crate::log::LogExt;
 use crate::message;
 
 /// Represents a file in the blob directory.
@@ -64,7 +63,7 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name);
-        let (name, mut file) = BlobObject::create_new_file(context, blobdir, &stem, &ext).await?;
+        let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
             .map_err(|err| BlobError::WriteFailure {
@@ -88,16 +87,13 @@ impl<'a> BlobObject<'a> {
 
     // Creates a new file, returning a tuple of the name and the handle.
     async fn create_new_file(
-        context: &Context,
         dir: &Path,
         stem: &str,
         ext: &str,
     ) -> Result<(String, fs::File), BlobError> {
-        const MAX_ATTEMPT: u32 = 16;
-        let mut attempt = 0;
+        let max_attempt = 15;
         let mut name = format!("{}{}", stem, ext);
-        loop {
-            attempt += 1;
+        for attempt in 0..max_attempt {
             let path = dir.join(&name);
             match fs::OpenOptions::new()
                 .create_new(true)
@@ -107,20 +103,24 @@ impl<'a> BlobObject<'a> {
             {
                 Ok(file) => return Ok((name, file)),
                 Err(err) => {
-                    if attempt >= MAX_ATTEMPT {
+                    if attempt == max_attempt {
                         return Err(BlobError::CreateFailure {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
                         });
-                    } else if attempt == 1 && !dir.exists().await {
-                        fs::create_dir_all(dir).await.ok_or_log(context);
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
                     }
                 }
             }
         }
+        // This is supposed to be unreachable, but the compiler doesn't know.
+        Err(BlobError::CreateFailure {
+            blobdir: dir.to_path_buf(),
+            blobname: name,
+            cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
+        })
     }
 
     /// Creates a new blob object with unique name by copying an existing file.
@@ -149,7 +149,7 @@ impl<'a> BlobObject<'a> {
             })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
         let (name, mut dst_file) =
-            BlobObject::create_new_file(context, context.get_blobdir(), &stem, &ext).await?;
+            BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
         if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
             {
@@ -621,13 +621,7 @@ mod tests {
 
     use super::*;
 
-    use crate::chat::{create_group_chat, ProtectionStatus};
-    use crate::{
-        chat,
-        message::Message,
-        test_utils::{self, TestContext},
-    };
-    use anyhow::Result;
+    use crate::{message::Message, test_utils::TestContext};
     use image::Pixel;
 
     #[async_std::test]
@@ -876,10 +870,11 @@ mod tests {
     async fn test_selfavatar_in_blobdir() {
         let t = TestContext::new().await;
         let avatar_src = t.get_blobdir().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
             .await
             .unwrap()
-            .write_all(test_utils::AVATAR_900x900_BYTES)
+            .write_all(avatar_bytes)
             .await
             .unwrap();
 
@@ -1069,38 +1064,4 @@ mod tests {
         assert_eq!(img.height() as u32, compressed_height);
         Ok(img)
     }
-
-    #[async_std::test]
-    async fn test_increation_in_blobdir() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
-
-        let file = t.get_blobdir().join("anyfile.dat");
-        File::create(&file).await?.write_all("bla".as_ref()).await?;
-        let mut msg = Message::new(Viewtype::File);
-        msg.set_file(file.to_str().unwrap(), None);
-        let prepared_id = chat::prepare_msg(&t, chat_id, &mut msg).await?;
-        assert_eq!(prepared_id, msg.id);
-        assert!(msg.is_increation());
-
-        let msg = Message::load_from_db(&t, prepared_id).await?;
-        assert!(msg.is_increation());
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_increation_not_blobdir() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
-        assert_ne!(t.get_blobdir().to_str(), t.dir.path().to_str());
-
-        let file = t.dir.path().join("anyfile.dat");
-        File::create(&file).await?.write_all("bla".as_ref()).await?;
-        let mut msg = Message::new(Viewtype::File);
-        msg.set_file(file.to_str().unwrap(), None);
-        assert!(chat::prepare_msg(&t, chat_id, &mut msg).await.is_err());
-
-        Ok(())
-    }
 }

src/chatlist.rs

@@ -1,6 +1,6 @@
 //! # Chat list module.
 
-use anyhow::{ensure, Context as _, Result};
+use anyhow::{bail, ensure, Result};
 
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
 use crate::constants::{
@@ -271,23 +271,21 @@ impl Chatlist {
     /// Get a single chat ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_chat().
-    pub fn get_chat_id(&self, index: usize) -> Result<ChatId> {
-        let (chat_id, _msg_id) = self
-            .ids
-            .get(index)
-            .context("chatlist index is out of range")?;
-        Ok(*chat_id)
+    pub fn get_chat_id(&self, index: usize) -> ChatId {
+        match self.ids.get(index) {
+            Some((chat_id, _msg_id)) => *chat_id,
+            None => ChatId::new(0),
+        }
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
     pub fn get_msg_id(&self, index: usize) -> Result<Option<MsgId>> {
-        let (_chat_id, msg_id) = self
-            .ids
-            .get(index)
-            .context("chatlist index is out of range")?;
-        Ok(*msg_id)
+        match self.ids.get(index) {
+            Some((_chat_id, msg_id)) => Ok(*msg_id),
+            None => bail!("Chatlist index out of range"),
+        }
     }
 
     /// Returns a summary for a given chatlist index.
@@ -301,10 +299,11 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let (chat_id, lastmsg_id) = self
-            .ids
-            .get(index)
-            .context("chatlist index is out of range")?;
+        let (chat_id, lastmsg_id) = match self.ids.get(index) {
+            Some(ids) => ids,
+            None => bail!("Chatlist index out of range"),
+        };
+
         Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
     }
 
@@ -330,7 +329,7 @@ impl Chatlist {
                 (Some(lastmsg), None)
             } else {
                 match chat.typ {
-                    Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
+                    Chattype::Group | Chattype::Mailinglist => {
                         let lastcontact = Contact::load_from_db(context, lastmsg.from_id).await?;
                         (Some(lastmsg), Some(lastcontact))
                     }
@@ -364,7 +363,7 @@ pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
         .sql
         .count(
             "SELECT COUNT(*) FROM chats WHERE blocked!=? AND archived=?;",
-            paramsv![Blocked::Yes, ChatVisibility::Archived],
+            paramsv![Blocked::Manually, ChatVisibility::Archived],
         )
         .await?;
     Ok(count)
@@ -396,26 +395,16 @@ mod tests {
         // check that the chatlist starts with the most recent message
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
-        assert_eq!(chats.get_chat_id(0).unwrap(), chat_id3);
-        assert_eq!(chats.get_chat_id(1).unwrap(), chat_id2);
-        assert_eq!(chats.get_chat_id(2).unwrap(), chat_id1);
-
-        // New drafts are sorted to the top
-        // We have to set a draft on the other two messages, too, as
-        // chat timestamps are only exact to the second and sorting by timestamp
-        // would not work.
-        // Message timestamps are "smeared" and unique, so we don't have this problem
-        // if we have any message (can be a draft) in all chats.
-        // Instead of setting drafts for chat_id1 and chat_id3, we could also sleep
-        // 2s here.
-        for chat_id in &[chat_id1, chat_id3, chat_id2] {
-            let mut msg = Message::new(Viewtype::Text);
-            msg.set_text(Some("hello".to_string()));
-            chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
-        }
+        assert_eq!(chats.get_chat_id(0), chat_id3);
+        assert_eq!(chats.get_chat_id(1), chat_id2);
+        assert_eq!(chats.get_chat_id(2), chat_id1);
 
+        // drafts are sorted to the top
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("hello".to_string()));
+        chat_id2.set_draft(&t, Some(&mut msg)).await.unwrap();
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.get_chat_id(0).unwrap(), chat_id2);
+        assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
         let chats = Chatlist::try_load(&t, 0, Some("b"), None).await.unwrap();
@@ -446,7 +435,7 @@ mod tests {
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0).unwrap())
+        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -455,7 +444,7 @@ mod tests {
             .await
             .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t, chats.get_chat_id(0).unwrap())
+        assert!(Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -500,7 +489,7 @@ mod tests {
         dc_receive_imf(
             &t,
             b"From: Bob Authname <bob@example.org>\n\
-                 To: alice@example.org\n\
+                 To: alice@example.com\n\
                  Subject: foo\n\
                  Message-ID: <msg1234@example.org>\n\
                  Chat-Version: 1.0\n\
@@ -508,6 +497,7 @@ mod tests {
                  \n\
                  hello foo\n",
             "INBOX",
+            1,
             false,
         )
         .await?;
@@ -528,7 +518,7 @@ mod tests {
         // check, the one-to-one-chat can be found using chatlist search query
         let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0).unwrap(), chat_id);
+        assert_eq!(chats.get_chat_id(0), chat_id);
 
         // change the name of the contact; this also changes the name of the one-to-one-chat
         let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
@@ -561,7 +551,7 @@ mod tests {
         dc_receive_imf(
             &t,
             b"From: bob@example.org\n\
-                 To: alice@example.org\n\
+                 To: alice@example.com\n\
                  Subject: foo\n\
                  Message-ID: <msg5678@example.org>\n\
                  Chat-Version: 1.0\n\
@@ -569,6 +559,7 @@ mod tests {
                  \n\
                  hello foo\n",
             "INBOX",
+            1,
             false,
         )
         .await?;
@@ -584,7 +575,7 @@ mod tests {
         // check, the one-to-one-chat can be found using chatlist search query
         let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0)?, chat_id);
+        assert_eq!(chats.get_chat_id(0), chat_id);
 
         // change the name of the contact; this also changes the name of the one-to-one-chat
         let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
@@ -595,7 +586,7 @@ mod tests {
         assert_eq!(chats.len(), 0); // email-addresses are searchable in contacts, not in chats
         let chats = Chatlist::try_load(&t, 0, Some("Bob Nickname"), None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0)?, chat_id);
+        assert_eq!(chats.get_chat_id(0), chat_id);
 
         // revert name change, this again changes the name of the one-to-one-chat to the email-address
         let test_id = Contact::create(&t, "", "bob@example.org").await?;

src/color.rs

@@ -35,10 +35,6 @@ pub(crate) fn str_to_color(s: &str) -> u32 {
     rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
 }
 
-pub fn color_int_to_hex_string(color: u32) -> String {
-    format!("{:#08x}", color).replace("0x", "#")
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

src/config.rs

@@ -1,6 +1,6 @@
 //! # Key-value configuration management.
 
-use anyhow::{ensure, Result};
+use anyhow::Result;
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
@@ -10,9 +10,11 @@ use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
+use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
+use crate::stock_str;
 
 /// The available configuration keys.
 #[derive(
@@ -65,9 +67,15 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MdnsEnabled,
 
-    #[strum(props(default = "0"))]
+    #[strum(props(default = "1"))]
+    InboxWatch,
+
+    #[strum(props(default = "1"))]
     SentboxWatch,
 
+    #[strum(props(default = "1"))]
+    MvboxWatch,
+
     #[strum(props(default = "1"))]
     MvboxMove,
 
@@ -162,16 +170,6 @@ pub enum Config {
     /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
     #[strum(props(default = "60"))]
     ScanAllFoldersDebounceSecs,
-
-    /// Defines the max. size (in bytes) of messages downloaded automatically.
-    /// 0 = no limit.
-    #[strum(props(default = "0"))]
-    DownloadLimit,
-
-    /// Send sync messages, requires `BccSelf` to be set as well.
-    /// In a future versions, this switch may be removed.
-    #[strum(props(default = "0"))]
-    SendSyncMsgs,
 }
 
 impl Context {
@@ -198,6 +196,7 @@ impl Context {
 
         // Default values
         match key {
+            Config::Selfstatus => Ok(Some(stock_str::status_line(self).await)),
             Config::ConfiguredInboxFolder => Ok(Some("INBOX".to_owned())),
             _ => Ok(key.get_str("default").map(|s| s.to_string())),
         }
@@ -275,12 +274,23 @@ impl Context {
                         let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
                         blob.recode_to_avatar_size(self).await?;
                         self.sql.set_raw_config(key, Some(blob.as_name())).await?;
+                        Ok(())
                     }
                     None => {
                         self.sql.set_raw_config(key, None).await?;
+                        Ok(())
                     }
                 }
-                self.emit_event(EventType::SelfavatarChanged);
+            }
+            Config::Selfstatus => {
+                let def = stock_str::status_line(self).await;
+                let val = if value.is_none() || value.unwrap() == def {
+                    None
+                } else {
+                    value
+                };
+
+                self.sql.set_raw_config(key, val).await?;
                 Ok(())
             }
             Config::DeleteDeviceAfter => {
@@ -301,6 +311,15 @@ impl Context {
                 self.sql.set_raw_config(key, value.as_deref()).await?;
                 Ok(())
             }
+            Config::DeleteServerAfter => {
+                let ret = self
+                    .sql
+                    .set_raw_config(key, value)
+                    .await
+                    .map_err(Into::into);
+                job::schedule_resync(self).await?;
+                ret
+            }
             _ => {
                 self.sql.set_raw_config(key, value).await?;
                 Ok(())
@@ -309,25 +328,10 @@ impl Context {
     }
 
     pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
-        self.set_config(key, if value { Some("1") } else { Some("0") })
+        self.set_config(key, if value { Some("1") } else { None })
             .await?;
         Ok(())
     }
-
-    /// Sets an ui-specific key-value pair.
-    /// Keys must be prefixed by `ui.`
-    /// and should be followed by the name of the system and maybe subsystem,
-    /// eg. `ui.desktop.linux.foo`, `ui.desktop.macos.bar`, `ui.ios.foobar`.
-    pub async fn set_ui_config(&self, key: &str, value: Option<&str>) -> Result<()> {
-        ensure!(key.starts_with("ui."), "set_ui_config(): prefix missing.");
-        self.sql.set_raw_config(key, value).await
-    }
-
-    /// Gets an ui-specific value set by set_ui_config().
-    pub async fn get_ui_config(&self, key: &str) -> Result<Option<String>> {
-        ensure!(key.starts_with("ui."), "get_ui_config(): prefix missing.");
-        self.sql.get_raw_config(key).await
-    }
 }
 
 /// Returns all available configuration keys concated together.
@@ -380,38 +384,4 @@ mod tests {
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
         assert_eq!(media_quality, constants::MediaQuality::Worse);
     }
-
-    #[async_std::test]
-    async fn test_ui_config() -> Result<()> {
-        let t = TestContext::new().await;
-
-        assert_eq!(t.get_ui_config("ui.desktop.linux.systray").await?, None);
-
-        t.set_ui_config("ui.android.screen_security", Some("safe"))
-            .await?;
-        assert_eq!(
-            t.get_ui_config("ui.android.screen_security").await?,
-            Some("safe".to_string())
-        );
-
-        t.set_ui_config("ui.android.screen_security", None).await?;
-        assert_eq!(t.get_ui_config("ui.android.screen_security").await?, None);
-
-        assert!(t.set_ui_config("configured", Some("bar")).await.is_err());
-
-        Ok(())
-    }
-
-    /// Regression test for https://github.com/deltachat/deltachat-core-rust/issues/3012
-    #[async_std::test]
-    async fn test_set_config_bool() -> Result<()> {
-        let t = TestContext::new().await;
-
-        // We need some config that defaults to true
-        let c = Config::E2eeEnabled;
-        assert_eq!(t.get_config_bool(c).await?, true);
-        t.set_config_bool(c, false).await?;
-        assert_eq!(t.get_config_bool(c).await?, false);
-        Ok(())
-    }
 }

src/configure.rs

@@ -8,12 +8,14 @@ mod server_params;
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
 use async_std::task;
+use itertools::Itertools;
 use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
-use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
+use crate::login_param::Socks5Config;
+use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
 use crate::oauth2::dc_get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
@@ -221,9 +223,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             "checking internal provider-info for offline autoconfig"
         );
 
-        if let Some(provider) =
-            provider::get_provider_info(ctx, &param_domain, socks5_enabled).await
-        {
+        if let Some(provider) = provider::get_provider_info(&param_domain, socks5_enabled).await {
             param.provider = Some(provider);
             match provider.status {
                 provider::Status::Ok | provider::Status::Preparation => {
@@ -250,7 +250,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                                         }
                                     }
                                 },
-                                strict_tls: Some(provider.strict_tls),
                             })
                             .collect();
 
@@ -291,7 +290,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.imap.port,
             socket: param.imap.security,
             username: param.imap.user.clone(),
-            strict_tls: None,
         })
     }
     if !servers
@@ -304,24 +302,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.smtp.port,
             socket: param.smtp.security,
             username: param.smtp.user.clone(),
-            strict_tls: None,
         })
     }
-
-    // respect certificate setting from function parameters
-    for mut server in &mut servers {
-        let certificate_checks = match server.protocol {
-            Protocol::Imap => param.imap.certificate_checks,
-            Protocol::Smtp => param.smtp.certificate_checks,
-        };
-        server.strict_tls = match certificate_checks {
-            CertificateChecks::AcceptInvalidCertificates
-            | CertificateChecks::AcceptInvalidCertificates2 => Some(false),
-            CertificateChecks::Strict => Some(true),
-            CertificateChecks::Automatic => server.strict_tls,
-        };
-    }
-
     let servers = expand_param_vector(servers, &param.addr, &param_domain);
 
     progress!(ctx, 550);
@@ -349,11 +331,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             smtp_param.server = smtp_server.hostname.clone();
             smtp_param.port = smtp_server.port;
             smtp_param.security = smtp_server.socket;
-            smtp_param.certificate_checks = match smtp_server.strict_tls {
-                Some(true) => CertificateChecks::Strict,
-                Some(false) => CertificateChecks::AcceptInvalidCertificates,
-                None => CertificateChecks::Automatic,
-            };
 
             match try_smtp_one_param(
                 &context_smtp,
@@ -397,11 +374,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         param.imap.server = imap_server.hostname.clone();
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
-        param.imap.certificate_checks = match imap_server.strict_tls {
-            Some(true) => CertificateChecks::Strict,
-            Some(false) => CertificateChecks::AcceptInvalidCertificates,
-            None => CertificateChecks::Automatic,
-        };
 
         match try_imap_one_param(
             ctx,
@@ -443,7 +415,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 900);
 
-    let create_mvbox = ctx.get_config_bool(Config::MvboxMove).await?;
+    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await?
+        || ctx.get_config_bool(Config::MvboxMove).await?;
 
     imap.configure_folders(ctx, create_mvbox).await?;
 
@@ -454,8 +427,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     drop(imap);
 
     progress!(ctx, 910);
+    // configuration success - write back the configured parameters with the
+    // "configured_" prefix; also write the "configured"-flag */
     // the trailing underscore is correct
     param.save_to_database(ctx, "configured_").await?;
+    ctx.sql.set_raw_config_bool("configured", true).await?;
     ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
@@ -468,13 +444,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         ctx,
         job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
     )
-    .await?;
+    .await;
 
     progress!(ctx, 940);
     update_device_chats_handle.await?;
 
-    ctx.sql.set_raw_config_bool("configured", true).await?;
-
     Ok(())
 }
 
@@ -681,11 +655,7 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
         return first_err.msg.to_string();
     }
 
-    errors
-        .iter()
-        .map(|e| e.to_string())
-        .collect::<Vec<String>>()
-        .join("\n\n")
+    errors.iter().map(|e| e.to_string()).join("\n\n")
 }
 
 #[derive(Debug, thiserror::Error)]

src/configure/auto_mozilla.rs

@@ -243,7 +243,6 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
                 hostname: server.hostname,
                 port: server.port,
                 username: server.username,
-                strict_tls: None,
             })
         })
         .collect();

src/configure/auto_outlook.rs

@@ -180,7 +180,6 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
                 hostname: protocol.server,
                 port: protocol.port,
                 username: String::new(),
-                strict_tls: None,
             })
         })
         .collect()

src/configure/server_params.rs

@@ -22,9 +22,6 @@ pub(crate) struct ServerParams {
 
     /// Username, empty if unknown.
     pub username: String,
-
-    /// Whether TLS certificates should be strictly checked or not, `None` for automatic.
-    pub strict_tls: Option<bool>,
 }
 
 impl ServerParams {
@@ -131,23 +128,6 @@ impl ServerParams {
             vec![self]
         }
     }
-
-    fn expand_strict_tls(self) -> Vec<ServerParams> {
-        if self.strict_tls.is_none() {
-            vec![
-                Self {
-                    strict_tls: Some(true), // Strict.
-                    ..self.clone()
-                },
-                Self {
-                    strict_tls: None, // Automatic.
-                    ..self
-                },
-            ]
-        } else {
-            vec![self]
-        }
-    }
 }
 
 /// Expands vector of `ServerParams`, replacing placeholders with
@@ -158,32 +138,10 @@ pub(crate) fn expand_param_vector(
     domain: &str,
 ) -> Vec<ServerParams> {
     v.into_iter()
-        .map(|params| {
-            if params.socket == Socket::Plain {
-                ServerParams {
-                    // Avoid expanding plaintext configuration into configuration with and without
-                    // `strict_tls` if `strict_tls` is set to `None` as `strict_tls` is not used for
-                    // plaintext connections. Always setting it to "enabled", just in case.
-                    strict_tls: Some(true),
-                    ..params
-                }
-            } else {
-                params
-            }
-        })
-        // The order of expansion is important.
-        //
-        // Ports are expanded the last, so they are changed the first.  Username is only changed if
-        // default value (address with domain) didn't work for all available hosts and ports.
-        //
-        // Strict TLS must be expanded first, so we try all configurations with strict TLS first
-        // and only then try again without strict TLS. Otherwise we may lock to wrong hostname
-        // without strict TLS when another hostname with strict TLS is available.  For example, if
-        // both smtp.example.net and mail.example.net are running an SMTP server, but both use a
-        // certificate that is only valid for mail.example.net, we want to skip smtp.example.net
-        // and use mail.example.net with strict TLS instead of using smtp.example.net without
-        // strict TLS.
-        .flat_map(|params| params.expand_strict_tls().into_iter())
+        // The order of expansion is important: ports are expanded the
+        // last, so they are changed the first. Username is only
+        // changed if default value (address with domain) didn't work
+        // for all available hosts and ports.
         .flat_map(|params| params.expand_usernames(addr).into_iter())
         .flat_map(|params| params.expand_hostnames(domain).into_iter())
         .flat_map(|params| params.expand_ports().into_iter())
@@ -203,7 +161,6 @@ mod tests {
                 port: 0,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
-                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -217,7 +174,6 @@ mod tests {
                 port: 993,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
-                strict_tls: Some(true)
             }],
         );
 
@@ -228,7 +184,6 @@ mod tests {
                 port: 123,
                 socket: Socket::Automatic,
                 username: "foobar".to_string(),
-                strict_tls: None,
             }],
             "foobar@example.net",
             "example.net",
@@ -242,59 +197,16 @@ mod tests {
                     hostname: "example.net".to_string(),
                     port: 123,
                     socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: Some(true),
+                    username: "foobar".to_string()
                 },
                 ServerParams {
                     protocol: Protocol::Smtp,
                     hostname: "example.net".to_string(),
                     port: 123,
                     socket: Socket::Starttls,
-                    username: "foobar".to_string(),
-                    strict_tls: Some(true)
-                },
-                ServerParams {
-                    protocol: Protocol::Smtp,
-                    hostname: "example.net".to_string(),
-                    port: 123,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: None,
-                },
-                ServerParams {
-                    protocol: Protocol::Smtp,
-                    hostname: "example.net".to_string(),
-                    port: 123,
-                    socket: Socket::Starttls,
-                    username: "foobar".to_string(),
-                    strict_tls: None
+                    username: "foobar".to_string()
                 }
             ],
         );
-
-        // Test that strict_tls is not expanded for plaintext connections.
-        let v = expand_param_vector(
-            vec![ServerParams {
-                protocol: Protocol::Smtp,
-                hostname: "example.net".to_string(),
-                port: 123,
-                socket: Socket::Plain,
-                username: "foobar".to_string(),
-                strict_tls: None,
-            }],
-            "foobar@example.net",
-            "example.net",
-        );
-        assert_eq!(
-            v,
-            vec![ServerParams {
-                protocol: Protocol::Smtp,
-                hostname: "example.net".to_string(),
-                port: 123,
-                socket: Socket::Plain,
-                username: "foobar".to_string(),
-                strict_tls: Some(true)
-            }],
-        );
     }
 }

src/constants.rs

@@ -24,7 +24,7 @@ pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION")
 #[repr(i8)]
 pub enum Blocked {
     Not = 0,
-    Yes = 1,
+    Manually = 1,
     Request = 2,
 }
 
@@ -153,7 +153,6 @@ pub enum Chattype {
     Single = 100,
     Group = 120,
     Mailinglist = 140,
-    Broadcast = 160,
 }
 
 impl Default for Chattype {
@@ -295,9 +294,6 @@ pub enum Viewtype {
 
     /// Message is an invitation to a videochat.
     VideochatInvitation = 70,
-
-    /// Message is an webxdc instance.
-    Webxdc = 80,
 }
 
 impl Default for Viewtype {
@@ -342,7 +338,6 @@ mod tests {
             Viewtype::VideochatInvitation,
             Viewtype::from_i32(70).unwrap()
         );
-        assert_eq!(Viewtype::Webxdc, Viewtype::from_i32(80).unwrap());
     }
 
     #[test]
@@ -353,7 +348,6 @@ mod tests {
         assert_eq!(Chattype::Single, Chattype::from_i32(100).unwrap());
         assert_eq!(Chattype::Group, Chattype::from_i32(120).unwrap());
         assert_eq!(Chattype::Mailinglist, Chattype::from_i32(140).unwrap());
-        assert_eq!(Chattype::Broadcast, Chattype::from_i32(160).unwrap());
     }
 
     #[test]
@@ -389,7 +383,7 @@ mod tests {
         // values may be written to disk and must not change
         assert_eq!(Blocked::Not, Blocked::default());
         assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
-        assert_eq!(Blocked::Yes, Blocked::from_i32(1).unwrap());
+        assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
         assert_eq!(Blocked::Request, Blocked::from_i32(2).unwrap());
     }
 

src/contact.rs

@@ -2,9 +2,10 @@
 
 use std::convert::{TryFrom, TryInto};
 
-use anyhow::{bail, ensure, Context as _, Result};
+use anyhow::{bail, ensure, format_err, Result};
 use async_std::path::PathBuf;
 use deltachat_derive::{FromSql, ToSql};
+use itertools::Itertools;
 use once_cell::sync::Lazy;
 use regex::Regex;
 
@@ -66,9 +67,6 @@ pub struct Contact {
     /// Blocked state. Use dc_contact_is_blocked to access this field.
     pub blocked: bool,
 
-    /// Time when the contact was seen last time, Unix time in seconds.
-    last_seen: i64,
-
     /// The origin/source of the contact.
     pub origin: Origin,
 
@@ -176,19 +174,12 @@ pub enum VerifiedStatus {
     BidirectVerified = 2,
 }
 
-impl Default for VerifiedStatus {
-    fn default() -> Self {
-        Self::Unverified
-    }
-}
-
 impl Contact {
     pub async fn load_from_db(context: &Context, contact_id: u32) -> Result<Self> {
         let mut contact = context
             .sql
             .query_row(
-                "SELECT c.name, c.addr, c.origin, c.blocked, c.last_seen,
-                c.authname, c.param, c.status
+                "SELECT c.name, c.addr, c.origin, c.blocked, c.authname, c.param, c.status
                FROM contacts c
               WHERE c.id=?;",
                 paramsv![contact_id as i32],
@@ -197,17 +188,15 @@ impl Contact {
                     let addr: String = row.get(1)?;
                     let origin: Origin = row.get(2)?;
                     let blocked: Option<bool> = row.get(3)?;
-                    let last_seen: i64 = row.get(4)?;
-                    let authname: String = row.get(5)?;
-                    let param: String = row.get(6)?;
-                    let status: Option<String> = row.get(7)?;
+                    let authname: String = row.get(4)?;
+                    let param: String = row.get(5)?;
+                    let status: Option<String> = row.get(6)?;
                     let contact = Self {
                         id: contact_id,
                         name,
                         authname,
                         addr,
                         blocked: blocked.unwrap_or_default(),
-                        last_seen,
                         origin,
                         param: param.parse().unwrap_or_default(),
                         status: status.unwrap_or_default(),
@@ -239,15 +228,12 @@ impl Contact {
         self.blocked
     }
 
-    /// Returns last seen timestamp.
-    pub fn last_seen(&self) -> i64 {
-        self.last_seen
-    }
-
     /// Check if a contact is blocked.
-    pub async fn is_blocked_load(context: &Context, id: u32) -> Result<bool> {
-        let blocked = Self::load_from_db(context, id).await?.blocked;
-        Ok(blocked)
+    pub async fn is_blocked_load(context: &Context, id: u32) -> bool {
+        Self::load_from_db(context, id)
+            .await
+            .map(|contact| contact.blocked)
+            .unwrap_or_default()
     }
 
     /// Block the given contact.
@@ -277,7 +263,7 @@ impl Contact {
 
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated).await?;
-        let blocked = Contact::is_blocked_load(context, contact_id).await?;
+        let blocked = Contact::is_blocked_load(context, contact_id).await;
         match sth_modified {
             Modifier::None => {}
             Modifier::Modified | Modifier::Created => {
@@ -311,17 +297,17 @@ impl Contact {
     /// use `dc_may_be_valid_addr()`.
     pub async fn lookup_id_by_addr(
         context: &Context,
-        addr: &str,
+        addr: impl AsRef<str>,
         min_origin: Origin,
     ) -> Result<Option<u32>> {
-        if addr.is_empty() {
+        if addr.as_ref().is_empty() {
             bail!("lookup_id_by_addr: empty address");
         }
 
-        let addr_normalized = addr_normalize(addr);
+        let addr_normalized = addr_normalize(addr.as_ref());
 
         if let Some(addr_self) = context.get_config(Config::ConfiguredAddr).await? {
-            if addr_cmp(addr_normalized, &addr_self) {
+            if addr_cmp(addr_normalized, addr_self) {
                 return Ok(Some(DC_CONTACT_ID_SELF));
             }
         }
@@ -383,7 +369,7 @@ impl Contact {
             .await?
             .unwrap_or_default();
 
-        if addr_cmp(&addr, &addr_self) {
+        if addr_cmp(&addr, addr_self) {
             return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
@@ -582,7 +568,7 @@ impl Contact {
 
         for (name, addr) in split_address_book(addr_book).into_iter() {
             let (name, addr) = sanitize_name_and_addr(name, addr);
-            let name = normalize_name(&name);
+            let name = normalize_name(name);
             match Contact::add_or_lookup(context, &name, &addr, Origin::AddressBook).await {
                 Err(err) => {
                     warn!(
@@ -721,7 +707,7 @@ impl Contact {
             .sql
             .query_map(
                 "SELECT name, grpid FROM chats WHERE type=? AND blocked=?;",
-                paramsv![Chattype::Mailinglist, Blocked::Yes],
+                paramsv![Chattype::Mailinglist, Blocked::Manually],
                 |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
                 |rows| {
                     rows.collect::<std::result::Result<Vec<_>, _>>()
@@ -768,9 +754,12 @@ impl Contact {
 
     /// Get blocked contacts.
     pub async fn get_all_blocked(context: &Context) -> Result<Vec<u32>> {
-        Contact::update_blocked_mailinglist_contacts(context)
-            .await
-            .context("cannot update blocked mailinglist contacts")?;
+        if let Err(e) = Contact::update_blocked_mailinglist_contacts(context).await {
+            warn!(
+                context,
+                "Cannot update blocked mailinglist contacts: {:?}", e
+            );
+        }
 
         let list = context
             .sql
@@ -1029,7 +1018,7 @@ impl Contact {
     ///
     /// The UI may draw a checkbox or something like that beside verified contacts.
     ///
-    pub async fn is_verified(&self, context: &Context) -> Result<VerifiedStatus> {
+    pub async fn is_verified(&self, context: &Context) -> VerifiedStatus {
         self.is_verified_ex(context, None).await
     }
 
@@ -1040,46 +1029,54 @@ impl Contact {
         &self,
         context: &Context,
         peerstate: Option<&Peerstate>,
-    ) -> Result<VerifiedStatus> {
+    ) -> VerifiedStatus {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
         if self.id == DC_CONTACT_ID_SELF {
-            return Ok(VerifiedStatus::BidirectVerified);
+            return VerifiedStatus::BidirectVerified;
         }
 
         if let Some(peerstate) = peerstate {
             if peerstate.verified_key.is_some() {
-                return Ok(VerifiedStatus::BidirectVerified);
+                return VerifiedStatus::BidirectVerified;
             }
         }
 
-        if let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? {
-            if peerstate.verified_key.is_some() {
-                return Ok(VerifiedStatus::BidirectVerified);
+        let peerstate = match Peerstate::from_addr(context, &self.addr).await {
+            Ok(peerstate) => peerstate,
+            Err(err) => {
+                warn!(
+                    context,
+                    "Failed to load peerstate for address {}: {}", self.addr, err
+                );
+                return VerifiedStatus::Unverified;
+            }
+        };
+
+        if let Some(ps) = peerstate {
+            if ps.verified_key.is_some() {
+                return VerifiedStatus::BidirectVerified;
             }
         }
 
-        Ok(VerifiedStatus::Unverified)
+        VerifiedStatus::Unverified
     }
 
-    pub async fn addr_equals_contact(
-        context: &Context,
-        addr: &str,
-        contact_id: u32,
-    ) -> Result<bool> {
+    pub async fn addr_equals_contact(context: &Context, addr: &str, contact_id: u32) -> bool {
         if addr.is_empty() {
-            return Ok(false);
+            return false;
         }
 
-        let contact = Contact::load_from_db(context, contact_id).await?;
-        if !contact.addr.is_empty() {
-            let normalized_addr = addr_normalize(addr);
-            if contact.addr == normalized_addr {
-                return Ok(true);
+        if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
+            if !contact.addr.is_empty() {
+                let normalized_addr = addr_normalize(addr);
+                if contact.addr == normalized_addr {
+                    return true;
+                }
             }
         }
 
-        Ok(false)
+        false
     }
 
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
@@ -1097,19 +1094,19 @@ impl Contact {
         Ok(count)
     }
 
-    pub async fn real_exists_by_id(context: &Context, contact_id: u32) -> Result<bool> {
-        if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
-            return Ok(false);
+    pub async fn real_exists_by_id(context: &Context, contact_id: u32) -> bool {
+        if !context.sql.is_open().await || contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+            return false;
         }
 
-        let exists = context
+        context
             .sql
             .exists(
                 "SELECT COUNT(*) FROM contacts WHERE id=?;",
                 paramsv![contact_id as i32],
             )
-            .await?;
-        Ok(exists)
+            .await
+            .unwrap_or_default()
     }
 
     pub async fn scaleup_origin_by_id(
@@ -1210,8 +1207,7 @@ WHERE type=? AND id IN (
         // also unblock mailinglist
         // if the contact is a mailinglist address explicitly created to allow unblocking
         if !new_blocking && contact.origin == Origin::MailinglistAddress {
-            if let Some((chat_id, _, _)) =
-                chat::get_chat_id_by_grpid(context, &contact.addr).await?
+            if let Some((chat_id, _, _)) = chat::get_chat_id_by_grpid(context, contact.addr).await?
             {
                 chat_id.unblock(context).await?;
             }
@@ -1300,35 +1296,14 @@ pub(crate) async fn set_status(
     Ok(())
 }
 
-/// Updates last seen timestamp of the contact if it is earlier than the given `timestamp`.
-pub(crate) async fn update_last_seen(
-    context: &Context,
-    contact_id: u32,
-    timestamp: i64,
-) -> Result<()> {
-    ensure!(
-        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
-        "Can not update special contact last seen timestamp"
-    );
-
-    context
-        .sql
-        .execute(
-            "UPDATE contacts SET last_seen = ?1 WHERE last_seen < ?1 AND id = ?2",
-            paramsv![timestamp, contact_id],
-        )
-        .await?;
-    Ok(())
-}
-
 /// Normalize a name.
 ///
 /// - Remove quotes (come from some bad MUA implementations)
 /// - Trims the resulting string
 ///
 /// Typically, this function is not needed as it is called implicitly by `Contact::add_address_book`.
-pub fn normalize_name(full_name: &str) -> String {
-    let full_name = full_name.trim();
+pub fn normalize_name(full_name: impl AsRef<str>) -> String {
+    let full_name = full_name.as_ref().trim();
     if full_name.is_empty() {
         return full_name.into();
     }
@@ -1371,45 +1346,44 @@ fn cat_fingerprint(
 impl Context {
     /// determine whether the specified addr maps to the/a self addr
     pub async fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        if let Some(self_addr) = self.get_config(Config::ConfiguredAddr).await? {
-            Ok(addr_cmp(&self_addr, addr))
-        } else {
-            Ok(false)
-        }
+        let self_addr = self
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .ok_or_else(|| format_err!("Not configured"))?;
+
+        Ok(addr_cmp(self_addr, addr))
     }
 }
 
-pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
-    let norm1 = addr_normalize(addr1).to_lowercase();
-    let norm2 = addr_normalize(addr2).to_lowercase();
+pub fn addr_cmp(addr1: impl AsRef<str>, addr2: impl AsRef<str>) -> bool {
+    let norm1 = addr_normalize(addr1.as_ref()).to_lowercase();
+    let norm2 = addr_normalize(addr2.as_ref()).to_lowercase();
 
     norm1 == norm2
 }
 
 fn split_address_book(book: &str) -> Vec<(&str, &str)> {
     book.lines()
-        .collect::<Vec<&str>>()
         .chunks(2)
         .into_iter()
-        .filter_map(|chunk| {
-            let name = chunk.get(0)?;
-            let addr = chunk.get(1)?;
-            Some((*name, *addr))
+        .filter_map(|mut chunk| {
+            let name = chunk.next().unwrap();
+            let addr = match chunk.next() {
+                Some(a) => a,
+                None => return None,
+            };
+            Some((name, addr))
         })
         .collect()
 }
 
 #[cfg(test)]
 mod tests {
-    use async_std::fs::File;
-    use async_std::io::WriteExt;
-
     use super::*;
 
     use crate::chat::send_text_msg;
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::message::Message;
-    use crate::test_utils::{self, TestContext};
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_may_be_valid_addr() {
@@ -1524,11 +1498,11 @@ mod tests {
     #[async_std::test]
     async fn test_is_self_addr() -> Result<()> {
         let t = TestContext::new().await;
-        assert_eq!(t.is_self_addr("me@me.org").await?, false);
+        assert!(t.is_self_addr("me@me.org").await.is_err());
 
-        t.configure_addr("you@you.net").await;
+        let addr = t.configure_alice().await;
         assert_eq!(t.is_self_addr("me@me.org").await?, false);
-        assert_eq!(t.is_self_addr("you@you.net").await?, true);
+        assert_eq!(t.is_self_addr(&addr).await?, true);
 
         Ok(())
     }
@@ -1891,7 +1865,7 @@ mod tests {
 
         let alice = TestContext::new_alice().await;
 
-        let id = Contact::lookup_id_by_addr(&alice.ctx, "alice@example.org", Origin::Unknown)
+        let id = Contact::lookup_id_by_addr(&alice.ctx, "alice@example.com", Origin::Unknown)
             .await
             .unwrap();
         assert_eq!(id, Some(DC_CONTACT_ID_SELF));
@@ -1916,7 +1890,7 @@ mod tests {
 
         let bob = TestContext::new_bob().await;
         let chat_alice = bob
-            .create_chat_with_contact("Alice", "alice@example.org")
+            .create_chat_with_contact("Alice", "alice@example.com")
             .await;
         send_text_msg(&bob, chat_alice.id, "Hello".to_string()).await?;
         let msg = bob.pop_sent_msg().await;
@@ -1928,7 +1902,7 @@ mod tests {
             "End-to-end encryption preferred.
 Fingerprints:
 
-alice@example.org:
+alice@example.com:
 2E6F A2CB 23B5 32D7 2863
 4B58 64B0 8F61 A9ED 9443
 
@@ -1979,7 +1953,7 @@ CCCB 5AA9 F6E1 141C 9431
 
         // Bob replies.
         let chat = bob
-            .create_chat_with_contact("Alice", "alice@example.org")
+            .create_chat_with_contact("Alice", "alice@example.com")
             .await;
 
         send_text_msg(&bob, chat.id, "Reply".to_string()).await?;
@@ -2005,94 +1979,4 @@ CCCB 5AA9 F6E1 141C 9431
 
         Ok(())
     }
-
-    /// Tests that DC_EVENT_SELFAVATAR_CHANGED is emitted on avatar changes.
-    #[async_std::test]
-    async fn test_selfavatar_changed_event() -> Result<()> {
-        // Alice has two devices.
-        let alice1 = TestContext::new_alice().await;
-        let alice2 = TestContext::new_alice().await;
-
-        // Bob has one device.
-        let bob = TestContext::new_bob().await;
-
-        assert_eq!(alice1.get_config(Config::Selfavatar).await?, None);
-
-        let avatar_src = alice1.get_blobdir().join("avatar.png");
-        File::create(&avatar_src)
-            .await?
-            .write_all(test_utils::AVATAR_900x900_BYTES)
-            .await?;
-
-        alice1
-            .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await?;
-
-        alice1
-            .evtracker
-            .get_matching(|e| matches!(e, EventType::SelfavatarChanged))
-            .await;
-
-        // Bob sends a message so that Alice can encrypt to him.
-        let chat = bob
-            .create_chat_with_contact("Alice", "alice@example.org")
-            .await;
-
-        send_text_msg(&bob, chat.id, "Reply".to_string()).await?;
-        let sent_msg = bob.pop_sent_msg().await;
-        alice1.recv_msg(&sent_msg).await;
-        alice2.recv_msg(&sent_msg).await;
-
-        // Alice sends a message.
-        let alice1_chat_id = alice1.get_last_msg().await.chat_id;
-        alice1_chat_id.accept(&alice1).await?;
-        send_text_msg(&alice1, alice1_chat_id, "Hello".to_string()).await?;
-        let sent_msg = alice1.pop_sent_msg().await;
-
-        // The message is encrypted.
-        let message = Message::load_from_db(&alice1, sent_msg.sender_msg_id).await?;
-        assert!(message.get_showpadlock());
-
-        // Alice's second device receives a copy of the outgoing message.
-        alice2.recv_msg(&sent_msg).await;
-
-        // Alice's second device applies the selfavatar.
-        assert!(alice2.get_config(Config::Selfavatar).await?.is_some());
-        alice2
-            .evtracker
-            .get_matching(|e| matches!(e, EventType::SelfavatarChanged))
-            .await;
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_last_seen() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        let (contact_id, _) =
-            Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated)
-                .await?;
-        let contact = Contact::load_from_db(&alice, contact_id).await?;
-        assert_eq!(contact.last_seen(), 0);
-
-        let mime = br#"Subject: Hello
-Message-ID: message@example.net
-To: Alice <alice@example.org>
-From: Bob <bob@example.net>
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-Chat-Version: 1.0
-Date: Sun, 22 Mar 2020 22:37:55 +0000
-
-Hi."#;
-        dc_receive_imf(&alice, mime, "Inbox", false).await?;
-        let msg = alice.get_last_msg().await;
-
-        let timestamp = msg.get_timestamp();
-        assert!(timestamp > 0);
-        let contact = Contact::load_from_db(&alice, contact_id).await?;
-        assert_eq!(contact.last_seen(), timestamp);
-
-        Ok(())
-    }
 }

src/context.rs

@@ -42,9 +42,12 @@ impl Deref for Context {
 
 #[derive(Debug)]
 pub struct InnerContext {
+    /// Database file path
+    pub(crate) dbfile: PathBuf,
     /// Blob directory path
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
+    pub(crate) os_name: Option<String>,
     pub(crate) bob: Bob,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
@@ -73,16 +76,11 @@ pub struct InnerContext {
     pub(crate) id: u32,
 
     creation_time: SystemTime,
-
-    /// The text of the last error logged and emitted as an event.
-    /// If the ui wants to display an error after a failure,
-    /// `last_error` should be used to avoid races with the event thread.
-    pub(crate) last_error: RwLock<String>,
 }
 
 #[derive(Debug)]
 pub struct RunningState {
-    ongoing_running: bool,
+    pub ongoing_running: bool,
     shall_stop_ongoing: bool,
     cancel_sender: Option<Sender<()>>,
 }
@@ -104,19 +102,10 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 }
 
 impl Context {
-    /// Creates new context and opens the database.
-    pub async fn new(dbfile: PathBuf, id: u32) -> Result<Context> {
-        let context = Self::new_closed(dbfile, id).await?;
+    /// Creates new context.
+    pub async fn new(os_name: String, dbfile: PathBuf, id: u32) -> Result<Context> {
+        // pretty_env_logger::try_init_timed().ok();
 
-        // Open the database if is not encrypted.
-        if context.check_passphrase("".to_string()).await? {
-            context.sql.open(&context, "".to_string()).await?;
-        }
-        Ok(context)
-    }
-
-    /// Creates new context without opening the database.
-    pub async fn new_closed(dbfile: PathBuf, id: u32) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
@@ -124,38 +113,11 @@ impl Context {
         if !blobdir.exists().await {
             async_std::fs::create_dir_all(&blobdir).await?;
         }
-        let context = Context::with_blobdir(dbfile, blobdir, id).await?;
-        Ok(context)
-    }
-
-    /// Opens the database with the given passphrase.
-    ///
-    /// Returns true if passphrase is correct, false is passphrase is not correct. Fails on other
-    /// errors.
-    pub async fn open(&self, passphrase: String) -> Result<bool> {
-        if self.sql.check_passphrase(passphrase.clone()).await? {
-            self.sql.open(self, passphrase).await?;
-            Ok(true)
-        } else {
-            Ok(false)
-        }
-    }
-
-    /// Returns true if database is open.
-    pub async fn is_open(&self) -> bool {
-        self.sql.is_open().await
-    }
-
-    /// Tests the database passphrase.
-    ///
-    /// Returns true if passphrase is correct.
-    ///
-    /// Fails if database is already open.
-    pub(crate) async fn check_passphrase(&self, passphrase: String) -> Result<bool> {
-        self.sql.check_passphrase(passphrase).await
+        Context::with_blobdir(os_name, dbfile, blobdir, id).await
     }
 
     pub(crate) async fn with_blobdir(
+        os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
         id: u32,
@@ -169,8 +131,10 @@ impl Context {
         let inner = InnerContext {
             id,
             blobdir,
+            dbfile,
+            os_name: Some(os_name),
             running_state: RwLock::new(Default::default()),
-            sql: Sql::new(dbfile),
+            sql: Sql::new(),
             bob: Default::default(),
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
@@ -183,12 +147,12 @@ impl Context {
             quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
-            last_error: RwLock::new("".to_string()),
         };
 
         let ctx = Context {
             inner: Arc::new(inner),
         };
+        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }
@@ -226,7 +190,7 @@ impl Context {
 
     /// Returns database file path.
     pub fn get_dbfile(&self) -> &Path {
-        self.sql.dbfile.as_path()
+        self.dbfile.as_path()
     }
 
     /// Returns blob directory path.
@@ -257,7 +221,7 @@ impl Context {
 
     // Ongoing process allocation/free/check
 
-    pub(crate) async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
+    pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
         if self.has_ongoing().await {
             bail!("There is already another ongoing process running.");
         }
@@ -273,7 +237,7 @@ impl Context {
         Ok(receiver)
     }
 
-    pub(crate) async fn free_ongoing(&self) {
+    pub async fn free_ongoing(&self) {
         let s_a = &self.running_state;
         let mut s = s_a.write().await;
 
@@ -282,7 +246,7 @@ impl Context {
         s.cancel_sender.take();
     }
 
-    pub(crate) async fn has_ongoing(&self) -> bool {
+    pub async fn has_ongoing(&self) -> bool {
         let s_a = &self.running_state;
         let s = s_a.read().await;
 
@@ -307,7 +271,7 @@ impl Context {
         };
     }
 
-    pub(crate) async fn shall_stop_ongoing(&self) -> bool {
+    pub async fn shall_stop_ongoing(&self) -> bool {
         self.running_state.read().await.shall_stop_ongoing
     }
 
@@ -339,7 +303,6 @@ impl Context {
         let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await?;
         let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await?;
         let bcc_self = self.get_config_int(Config::BccSelf).await?;
-        let send_sync_msgs = self.get_config_int(Config::SendSyncMsgs).await?;
 
         let prv_key_cnt = self
             .sql
@@ -355,7 +318,9 @@ impl Context {
             Err(err) => format!("<key failure: {}>", err),
         };
 
+        let inbox_watch = self.get_config_int(Config::InboxWatch).await?;
         let sentbox_watch = self.get_config_int(Config::SentboxWatch).await?;
+        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await?;
         let mvbox_move = self.get_config_int(Config::MvboxMove).await?;
         let sentbox_move = self.get_config_int(Config::SentboxMove).await?;
         let folders_configured = self
@@ -383,13 +348,6 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
-        res.insert(
-            "database_encrypted",
-            self.sql
-                .is_encrypted()
-                .await
-                .map_or_else(|| "closed".to_string(), |b| b.to_string()),
-        );
         res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
@@ -413,13 +371,9 @@ impl Context {
             "show_emails",
             self.get_config_int(Config::ShowEmails).await?.to_string(),
         );
-        res.insert(
-            "download_limit",
-            self.get_config_int(Config::DownloadLimit)
-                .await?
-                .to_string(),
-        );
+        res.insert("inbox_watch", inbox_watch.to_string());
         res.insert("sentbox_watch", sentbox_watch.to_string());
+        res.insert("mvbox_watch", mvbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
         res.insert("sentbox_move", sentbox_move.to_string());
         res.insert("folders_configured", folders_configured.to_string());
@@ -432,7 +386,6 @@ impl Context {
             self.get_config_int(Config::KeyGenType).await?.to_string(),
         );
         res.insert("bcc_self", bcc_self.to_string());
-        res.insert("send_sync_msgs", send_sync_msgs.to_string());
         res.insert("private_key_count", prv_key_cnt.to_string());
         res.insert("public_key_count", pub_key_cnt.to_string());
         res.insert("fingerprint", fingerprint_str);
@@ -614,14 +567,14 @@ impl Context {
         Ok(spam.as_deref() == Some(folder_name))
     }
 
-    pub(crate) fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
+    pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         dbfile.with_file_name(blob_fname)
     }
 
-    pub(crate) fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
+    pub fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
         let mut wal_fname = OsString::new();
         wal_fname.push(dbfile.file_name().unwrap_or_default());
         wal_fname.push("-wal");
@@ -678,21 +631,16 @@ mod tests {
     use crate::dc_tools::dc_create_outgoing_rfc724_mid;
     use crate::message::Message;
     use crate::test_utils::TestContext;
-    use anyhow::Context as _;
     use std::time::Duration;
     use strum::IntoEnumIterator;
-    use tempfile::tempdir;
 
     #[async_std::test]
-    async fn test_wrong_db() -> Result<()> {
-        let tmp = tempfile::tempdir()?;
+    async fn test_wrong_db() {
+        let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        std::fs::write(&dbfile, b"123")?;
-        let res = Context::new(dbfile.into(), 1).await?;
-
-        // Broken database is indistinguishable from encrypted one.
-        assert_eq!(res.is_open().await, false);
-        Ok(())
+        std::fs::write(&dbfile, b"123").unwrap();
+        let res = Context::new("FakeOs".into(), dbfile.into(), 1).await;
+        assert!(res.is_err());
     }
 
     #[async_std::test]
@@ -709,7 +657,7 @@ mod tests {
             .unwrap();
         let msg = format!(
             "From: {}\n\
-             To: alice@example.org\n\
+             To: alice@example.com\n\
              Message-ID: <{}>\n\
              Chat-Version: 1.0\n\
              Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
@@ -719,7 +667,7 @@ mod tests {
             dc_create_outgoing_rfc724_mid(None, contact.get_addr())
         );
         println!("{}", msg);
-        dc_receive_imf(t, msg.as_bytes(), "INBOX", false)
+        dc_receive_imf(t, msg.as_bytes(), "INBOX", 1, false)
             .await
             .unwrap();
     }
@@ -842,7 +790,9 @@ mod tests {
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new(dbfile.into(), 1).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -853,7 +803,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new(dbfile.into(), 1).await;
+        let res = Context::new("FakeOS".into(), dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -863,7 +813,9 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new(dbfile.into(), 1).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -873,7 +825,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(dbfile.into(), blobdir, 1).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir, 1).await;
         assert!(res.is_err());
     }
 
@@ -882,7 +834,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(dbfile.into(), blobdir.into(), 1).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -1044,29 +996,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_check_passphrase() -> Result<()> {
-        let dir = tempdir()?;
-        let dbfile = dir.path().join("db.sqlite");
-
-        let id = 1;
-        let context = Context::new_closed(dbfile.clone().into(), id)
-            .await
-            .context("failed to create context")?;
-        assert_eq!(context.open("foo".to_string()).await?, true);
-        assert_eq!(context.is_open().await, true);
-        drop(context);
-
-        let id = 2;
-        let context = Context::new(dbfile.into(), id)
-            .await
-            .context("failed to create context")?;
-        assert_eq!(context.is_open().await, false);
-        assert_eq!(context.check_passphrase("bar".to_string()).await?, false);
-        assert_eq!(context.open("false".to_string()).await?, false);
-        assert_eq!(context.open("foo".to_string()).await?, true);
-
-        Ok(())
-    }
 }

src/dc_tools.rs

@@ -5,7 +5,6 @@ use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
 use std::io::Cursor;
-use std::str::from_utf8;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
@@ -13,11 +12,8 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use anyhow::Error;
+use anyhow::{bail, Error};
 use chrono::{Local, TimeZone};
-use mailparse::dateparse;
-use mailparse::headers::Headers;
-use mailparse::MailHeaderMap;
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
@@ -88,9 +84,9 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
 //   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
 const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 
-/// Returns the current smeared timestamp,
-///
-/// The returned timestamp MUST NOT be sent out.
+// returns the currently smeared timestamp,
+// may be used to check if call to dc_create_smeared_timestamp() is needed or not.
+// the returned timestamp MUST NOT be used to be sent out or saved in the database!
 pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
     let mut now = time();
     let ts = *context.last_smeared_timestamp.read().await;
@@ -101,7 +97,7 @@ pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
     now
 }
 
-/// Returns a timestamp that is guaranteed to be unique.
+// returns a timestamp that is guaranteed to be unique.
 pub(crate) async fn dc_create_smeared_timestamp(context: &Context) -> i64 {
     let now = time();
     let mut ret = now;
@@ -195,26 +191,44 @@ async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time
 }
 
 /* Message-ID tools */
-
-/// Generate an ID. The generated ID should be as short and as unique as possible:
-/// - short, because it may also used as part of Message-ID headers or in QR codes
-/// - unique as two IDs generated on two devices should not be the same. However, collisions are not world-wide but only by the few contacts.
-/// IDs generated by this function are 66 bit wide and are returned as 11 base64 characters.
-///
-/// Additional information when used as a message-id or group-id:
-/// - for OUTGOING messages this ID is written to the header as `Chat-Group-ID:` and is added to the message ID as Gr.<grpid>.<random>@<random>
-/// - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header
-/// - the group-id should be a string with the characters [a-zA-Z0-9\-_]
 pub(crate) fn dc_create_id() -> String {
-    // ThreadRng implements CryptoRng trait and is supposed to be cryptographically secure.
+    /* generate an id. the generated ID should be as short and as unique as possible:
+    - short, because it may also used as part of Message-ID headers or in QR codes
+    - unique as two IDs generated on two devices should not be the same. However, collisions are not world-wide but only by the few contacts.
+    IDs generated by this function are 66 bit wide and are returned as 11 base64 characters.
+    If possible, RNG of OpenSSL is used.
+
+    Additional information when used as a message-id or group-id:
+    - for OUTGOING messages this ID is written to the header as `Chat-Group-ID:` and is added to the message ID as Gr.<grpid>.<random>@<random>
+    - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header
+    - the group-id should be a string with the characters [a-zA-Z0-9\-_] */
+
     let mut rng = thread_rng();
+    let buf: [u32; 3] = [rng.gen(), rng.gen(), rng.gen()];
 
-    // Generate 72 random bits.
-    let mut arr = [0u8; 9];
-    rng.fill(&mut arr[..]);
+    encode_66bits_as_base64(buf[0usize], buf[1usize], buf[2usize])
+}
 
-    // Take 11 base64 characters containing 66 random bits.
-    base64::encode(&arr).chars().take(11).collect()
+/// Encode 66 bits as a base64 string.
+/// This is useful for ID generating with short strings as we save 5 character
+/// in each id compared to 64 bit hex encoding. For a typical group ID, these
+/// are 10 characters (grpid+msgid):
+///    hex:    64 bit, 4 bits/character, length = 64/4 = 16 characters
+///    base64: 64 bit, 6 bits/character, length = 64/6 = 11 characters (plus 2 additional bits)
+/// Only the lower 2 bits of `fill` are used.
+fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
+    use byteorder::{BigEndian, WriteBytesExt};
+
+    let mut wrapped_writer = Vec::new();
+    {
+        let mut enc = base64::write::EncoderWriter::new(&mut wrapped_writer, base64::URL_SAFE);
+        enc.write_u32::<BigEndian>(v1).unwrap();
+        enc.write_u32::<BigEndian>(v2).unwrap();
+        enc.write_u8(((fill & 0x3) as u8) << 6).unwrap();
+        enc.finish().unwrap();
+    }
+    assert_eq!(wrapped_writer.pop(), Some(b'A')); // Remove last "A"
+    String::from_utf8(wrapped_writer).unwrap()
 }
 
 /// Function generates a Message-ID that can be used for a new outgoing message.
@@ -336,6 +350,63 @@ pub async fn dc_delete_files_in_dir(context: &Context, path: impl AsRef<Path>) {
     }
 }
 
+pub(crate) async fn dc_copy_file(
+    context: &Context,
+    src_path: impl AsRef<Path>,
+    dest_path: impl AsRef<Path>,
+) -> bool {
+    let src_abs = dc_get_abs_path(context, &src_path);
+    let mut src_file = match fs::File::open(&src_abs).await {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for read '{}': {}",
+                src_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    let dest_abs = dc_get_abs_path(context, &dest_path);
+    let mut dest_file = match fs::OpenOptions::new()
+        .create_new(true)
+        .write(true)
+        .open(&dest_abs)
+        .await
+    {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for write '{}': {}",
+                dest_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    match io::copy(&mut src_file, &mut dest_file).await {
+        Ok(_) => true,
+        Err(err) => {
+            error!(
+                context,
+                "Cannot copy \"{}\" to \"{}\": {}",
+                src_abs.display(),
+                dest_abs.display(),
+                err
+            );
+            {
+                // Attempt to remove the failed file, swallow errors resulting from that.
+                fs::remove_file(dest_abs).await.ok();
+            }
+            false
+        }
+    }
+}
+
 pub(crate) async fn dc_create_folder(
     context: &Context,
     path: impl AsRef<Path>,
@@ -433,6 +504,33 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
+/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
+/// it can be renamed to dest_path. This guarantees that the backup is complete.
+pub(crate) async fn get_next_backup_path(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<(PathBuf, PathBuf), Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
+        .format("delta-chat-backup-%Y-%m-%d")
+        .to_string();
+
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut tempfile = folder.clone();
+        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
+
+        let mut destfile = folder.clone();
+        destfile.push(format!("{}-{:02}.tar", stem, i));
+
+        if !tempfile.exists().await && !destfile.exists().await {
+            return Ok((tempfile, destfile));
+        }
+    }
+    bail!("could not create backup file, disk full?");
+}
+
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
@@ -572,144 +670,13 @@ pub fn remove_subject_prefix(last_subject: &str) -> String {
         .to_string()
 }
 
-// Types and methods to create hop-info for message-info
-
-fn extract_address_from_receive_header<'a>(header: &'a str, start: &str) -> Option<&'a str> {
-    let header_len = header.len();
-    header.find(start).and_then(|mut begin| {
-        begin += start.len();
-        let end = header
-            .get(begin..)?
-            .find(|c: char| c.is_whitespace())
-            .unwrap_or(header_len);
-        header.get(begin..begin + end)
-    })
-}
-
-pub(crate) fn parse_receive_header(header: &str) -> String {
-    let header = header.replace(&['\r', '\n'][..], "");
-    let mut hop_info = String::from("Hop: ");
-
-    if let Some(from) = extract_address_from_receive_header(&header, "from ") {
-        hop_info += &format!("From: {}; ", from.trim());
-    }
-
-    if let Some(by) = extract_address_from_receive_header(&header, "by ") {
-        hop_info += &format!("By: {}; ", by.trim());
-    }
-
-    if let Ok(date) = dateparse(&header) {
-        // In tests, use the UTC timezone so that the test is reproducible
-        #[cfg(test)]
-        let date_obj = chrono::Utc.timestamp(date, 0);
-        #[cfg(not(test))]
-        let date_obj = Local.timestamp(date, 0);
-
-        hop_info += &format!("Date: {}", date_obj.to_rfc2822());
-    };
-
-    hop_info
-}
-
-/// parses "receive"-headers
-pub(crate) fn parse_receive_headers(headers: &Headers) -> String {
-    headers
-        .get_all_headers("Received")
-        .iter()
-        .rev()
-        .filter_map(|header_map_item| from_utf8(header_map_item.get_value_raw()).ok())
-        .map(parse_receive_header)
-        .collect::<Vec<_>>()
-        .join("\n")
-}
-
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
 
-    use crate::{
-        config::Config, dc_receive_imf::dc_receive_imf, message::get_msg_info,
-        test_utils::TestContext,
-    };
-
-    #[test]
-    fn test_parse_receive_headers() {
-        // Test `parse_receive_headers()` with some more-or-less random emails from the test-data
-        let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
-        let expected =
-            "Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000\n\
-             Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000";
-        check_parse_receive_headers(raw, expected);
-
-        let raw = include_bytes!("../test-data/message/wrong-html.eml");
-        let expected =
-            "Hop: From: oxbsltgw18.schlund.de; By: mrelayeu.kundenserver.de; Date: Thu, 06 Aug 2020 16:40:31 +0000\n\
-             Hop: From: mout.kundenserver.de; By: dd37930.kasserver.com; Date: Thu, 06 Aug 2020 16:40:32 +0000";
-        check_parse_receive_headers(raw, expected);
-
-        let raw = include_bytes!("../test-data/message/posteo_ndn.eml");
-        let expected =
-            "Hop: By: mout01.posteo.de; Date: Tue, 09 Jun 2020 18:44:22 +0000\n\
-             Hop: From: mout01.posteo.de; By: mx04.posteo.de; Date: Tue, 09 Jun 2020 18:44:22 +0000\n\
-             Hop: From: mx04.posteo.de; By: mailin06.posteo.de; Date: Tue, 09 Jun 2020 18:44:23 +0000\n\
-             Hop: From: mailin06.posteo.de; By: proxy02.posteo.de; Date: Tue, 09 Jun 2020 18:44:23 +0000\n\
-             Hop: From: proxy02.posteo.de; By: proxy02.posteo.name; Date: Tue, 09 Jun 2020 18:44:23 +0000\n\
-             Hop: From: proxy02.posteo.name; By: dovecot03.posteo.local; Date: Tue, 09 Jun 2020 18:44:24 +0000";
-        check_parse_receive_headers(raw, expected);
-    }
-
-    fn check_parse_receive_headers(raw: &[u8], expected: &str) {
-        let mail = mailparse::parse_mail(raw).unwrap();
-        let hop_info = parse_receive_headers(&mail.get_headers());
-        assert_eq!(hop_info, expected)
-    }
-
-    #[async_std::test]
-    async fn test_parse_receive_headers_integration() {
-        let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
-        let expected = r"State: Fresh
-
-hi
-
-Message-ID: 2dfdbde7@example.org
-
-Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000
-Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000";
-        check_parse_receive_headers_integration(raw, expected).await;
-
-        let raw = include_bytes!("../test-data/message/encrypted_with_received_headers.eml");
-        let expected = "State: Fresh, Encrypted
-
-Re: Message from alice@example.org
-
-hi back\r\n\
-\r\n\
--- \r\n\
-Sent with my Delta Chat Messenger: https://delta.chat
-
-Message-ID: Mr.adQpEwndXLH.LPDdlFVJ7wG@example.net
-
-Hop: From: [127.0.0.1]; By: mail.example.org; Date: Mon, 27 Dec 2021 11:21:21 +0000
-Hop: From: mout.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
-Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000";
-        check_parse_receive_headers_integration(raw, expected).await;
-    }
-
-    async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
-        let t = TestContext::new_alice().await;
-        t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
-        dc_receive_imf(&t, raw, "INBOX", false).await.unwrap();
-        let msg = t.get_last_msg().await;
-        let msg_info = get_msg_info(&t, msg.id).await.unwrap();
-
-        // Ignore the first rows of the msg_info because they contain a
-        // received time that depends on the test time which makes it impossible to
-        // compare with a static string
-        let capped_result = &msg_info[msg_info.find("State").unwrap()..];
-        assert_eq!(expected, capped_result);
-    }
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_rust_ftoa() {
@@ -762,6 +729,26 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         assert_eq!(buf.len(), 11);
     }
 
+    #[test]
+    fn test_encode_66bits_as_base64() {
+        assert_eq!(
+            encode_66bits_as_base64(0x01234567, 0x89abcdef, 0),
+            "ASNFZ4mrze8"
+        );
+        assert_eq!(
+            encode_66bits_as_base64(0x01234567, 0x89abcdef, 1),
+            "ASNFZ4mrze9"
+        );
+        assert_eq!(
+            encode_66bits_as_base64(0x01234567, 0x89abcdef, 2),
+            "ASNFZ4mrze-"
+        );
+        assert_eq!(
+            encode_66bits_as_base64(0x01234567, 0x89abcdef, 3),
+            "ASNFZ4mrze_"
+        );
+    }
+
     #[test]
     fn test_dc_extract_grpid_from_rfc724_mid() {
         // Should return None if we pass invalid mid
@@ -848,8 +835,8 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
+    use crate::chat;
     use crate::chatlist::Chatlist;
-    use crate::{chat, test_utils};
     use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
     use proptest::prelude::*;
 
@@ -857,18 +844,22 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         #[test]
         fn test_dc_truncate(
             buf: String,
-            approx_chars in 0..100usize
+            approx_chars in 0..10000usize
         ) {
             let res = dc_truncate(&buf, approx_chars);
             let el_len = 5;
             let l = res.chars().count();
-            assert!(
-                l <= approx_chars + el_len,
-                "buf: '{}' - res: '{}' - len {}, approx {}",
-                &buf, &res, res.len(), approx_chars
-            );
+            if approx_chars > 0 {
+                assert!(
+                    l <= approx_chars + el_len,
+                    "buf: '{}' - res: '{}' - len {}, approx {}",
+                    &buf, &res, res.len(), approx_chars
+                );
+            } else {
+                assert_eq!(&res, &buf);
+            }
 
-            if buf.chars().count() > approx_chars + el_len {
+            if approx_chars > 0 && buf.chars().count() > approx_chars + el_len {
                 let l = res.len();
                 assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
             }
@@ -903,7 +894,20 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
 
         assert!(dc_file_exist!(context, &abs_path).await);
 
+        assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
+
+        // attempting to copy a second time should fail
+        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
+
+        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada").await, 7);
+
+        let buf = dc_read_file(context, "$BLOBDIR/dada").await.unwrap();
+
+        assert_eq!(buf.len(), 7);
+        assert_eq!(&buf, b"content");
+
         assert!(dc_delete_file(context, "$BLOBDIR/foobar").await);
+        assert!(dc_delete_file(context, "$BLOBDIR/dada").await);
         assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder")
             .await
             .is_ok());
@@ -982,7 +986,8 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
 
     #[test]
     fn test_get_filemeta() {
-        let (w, h) = dc_get_filemeta(test_utils::AVATAR_900x900_BYTES).unwrap();
+        let data = include_bytes!("../test-data/image/avatar900x900.png");
+        let (w, h) = dc_get_filemeta(data).unwrap();
         assert_eq!(w, 900);
         assert_eq!(h, 900);
 
@@ -1029,7 +1034,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         maybe_warn_on_bad_time(&t, timestamp_past, get_provider_update_timestamp()).await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
+        let device_chat_id = chats.get_chat_id(0);
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1067,7 +1072,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        assert_eq!(device_chat_id, chats.get_chat_id(0).unwrap());
+        assert_eq!(device_chat_id, chats.get_chat_id(0));
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1099,7 +1104,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
+        let device_chat_id = chats.get_chat_id(0);
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1121,7 +1126,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
+        let device_chat_id = chats.get_chat_id(0);
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1138,7 +1143,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
+        let device_chat_id = chats.get_chat_id(0);
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();

src/dehtml.rs

@@ -36,9 +36,9 @@ impl Dehtml {
             ""
         }
     }
-    fn append_prefix(&self, line_end: &str) -> String {
+    fn append_prefix(&self, line_end: impl AsRef<str>) -> String {
         // line_end is e.g. "\n\n". We add "> " if necessary.
-        line_end.to_string() + self.line_prefix()
+        line_end.as_ref().to_owned() + self.line_prefix()
     }
     fn get_add_text(&self) -> AddText {
         if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {

src/download.rs

@@ -1,400 +0,0 @@
-//! # Download large messages manually.
-
-use anyhow::{anyhow, Result};
-use deltachat_derive::{FromSql, ToSql};
-use serde::{Deserialize, Serialize};
-
-use crate::config::Config;
-use crate::constants::Viewtype;
-use crate::context::Context;
-use crate::dc_tools::time;
-use crate::imap::{Imap, ImapActionResult};
-use crate::job::{self, Action, Job, Status};
-use crate::message::{Message, MsgId};
-use crate::mimeparser::{MimeMessage, Part};
-use crate::param::Params;
-use crate::{job_try, stock_str, EventType};
-use std::cmp::max;
-
-/// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`.
-///
-/// Some messages as non-delivery-reports (NDN) or read-receipts (MDN)
-/// need to be downloaded completely to handle them correctly,
-/// eg. to assign them to the correct chat.
-/// As these messages are typically small,
-/// they're catched by `MIN_DOWNLOAD_LIMIT`.
-const MIN_DOWNLOAD_LIMIT: u32 = 32768;
-
-/// If a message is downloaded only partially
-/// and `delete_server_after` is set to small timeouts (eg. "at once"),
-/// the user might have no chance to actually download that message.
-/// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case.
-pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
-
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
-#[repr(u32)]
-pub enum DownloadState {
-    Done = 0,
-    Available = 10,
-    Failure = 20,
-    InProgress = 1000,
-}
-
-impl Default for DownloadState {
-    fn default() -> Self {
-        DownloadState::Done
-    }
-}
-
-impl Context {
-    // Returns validated download limit or `None` for "no limit".
-    pub(crate) async fn download_limit(&self) -> Result<Option<u32>> {
-        let download_limit = self.get_config_int(Config::DownloadLimit).await?;
-        if download_limit <= 0 {
-            Ok(None)
-        } else {
-            Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32)))
-        }
-    }
-}
-
-impl MsgId {
-    /// Schedules full message download for partially downloaded message.
-    pub async fn download_full(self, context: &Context) -> Result<()> {
-        let msg = Message::load_from_db(context, self).await?;
-        match msg.download_state() {
-            DownloadState::Done => return Err(anyhow!("Nothing to download.")),
-            DownloadState::InProgress => return Err(anyhow!("Download already in progress.")),
-            DownloadState::Available | DownloadState::Failure => {
-                self.update_download_state(context, DownloadState::InProgress)
-                    .await?;
-                job::add(
-                    context,
-                    Job::new(Action::DownloadMsg, self.to_u32(), Params::new(), 0),
-                )
-                .await?;
-            }
-        }
-        Ok(())
-    }
-
-    pub(crate) async fn update_download_state(
-        self,
-        context: &Context,
-        download_state: DownloadState,
-    ) -> Result<()> {
-        let msg = Message::load_from_db(context, self).await?;
-        context
-            .sql
-            .execute(
-                "UPDATE msgs SET download_state=? WHERE id=?;",
-                paramsv![download_state, self],
-            )
-            .await?;
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: msg.chat_id,
-            msg_id: self,
-        });
-        Ok(())
-    }
-}
-
-impl Message {
-    /// Returns the download state of the message.
-    pub fn download_state(&self) -> DownloadState {
-        self.download_state
-    }
-}
-
-impl Job {
-    /// Actually download a message.
-    /// Called in response to `Action::DownloadMsg`.
-    pub(crate) async fn download_msg(&self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.prepare(context).await {
-            warn!(context, "download: could not connect: {:?}", err);
-            return Status::RetryNow;
-        }
-
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let row = job_try!(
-            context
-                .sql
-                .query_row_optional(
-                    "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''",
-                    paramsv![msg.rfc724_mid],
-                    |row| {
-                        let server_uid: u32 = row.get(0)?;
-                        let server_folder: String = row.get(1)?;
-                        Ok((server_uid, server_folder))
-                    }
-                )
-                .await
-        );
-
-        if let Some((server_uid, server_folder)) = row {
-            match imap
-                .fetch_single_msg(context, &server_folder, server_uid)
-                .await
-            {
-                ImapActionResult::RetryLater | ImapActionResult::Failed => {
-                    job_try!(
-                        msg.id
-                            .update_download_state(context, DownloadState::Failure)
-                            .await
-                    );
-                    Status::Finished(Err(anyhow!("Call download_full() again to try over.")))
-                }
-                ImapActionResult::Success => {
-                    // update_download_state() not needed as receive_imf() already
-                    // set the state and emitted the event.
-                    Status::Finished(Ok(()))
-                }
-            }
-        } else {
-            // No IMAP record found, we don't know the UID and folder.
-            job_try!(
-                msg.id
-                    .update_download_state(context, DownloadState::Failure)
-                    .await
-            );
-            Status::Finished(Err(anyhow!("Call download_full() again to try over.")))
-        }
-    }
-}
-
-impl Imap {
-    /// Download a single message and pipe it to receive_imf().
-    ///
-    /// receive_imf() is not directly aware that this is a result of a call to download_msg(),
-    /// however, implicitly knows that as the existing message is flagged as being partly.
-    async fn fetch_single_msg(
-        &mut self,
-        context: &Context,
-        folder: &str,
-        uid: u32,
-    ) -> ImapActionResult {
-        if let Some(imapresult) = self
-            .prepare_imap_operation_on_msg(context, folder, uid)
-            .await
-        {
-            return imapresult;
-        }
-
-        // we are connected, and the folder is selected
-        info!(context, "Downloading message {}/{} fully...", folder, uid);
-
-        let (last_uid, _received) = match self
-            .fetch_many_msgs(context, folder, vec![uid], false, false)
-            .await
-        {
-            Ok(res) => res,
-            Err(_) => return ImapActionResult::Failed,
-        };
-        if last_uid.is_none() {
-            ImapActionResult::Failed
-        } else {
-            ImapActionResult::Success
-        }
-    }
-}
-
-impl MimeMessage {
-    /// Creates a placeholder part and add that to `parts`.
-    ///
-    /// To create the placeholder, only the outermost header can be used,
-    /// the mime-structure itself is not available.
-    ///
-    /// The placeholder part currently contains a text with size and availability of the message;
-    /// in the future, we may do more advanced things as previews here.
-    pub(crate) async fn create_stub_from_partial_download(
-        &mut self,
-        context: &Context,
-        org_bytes: u32,
-    ) -> Result<()> {
-        let mut text = format!(
-            "[{}]",
-            stock_str::partial_download_msg_body(context, org_bytes).await
-        );
-        if let Some(delete_server_after) = context.get_config_delete_server_after().await? {
-            let until = stock_str::download_availability(
-                context,
-                time() + max(delete_server_after, MIN_DELETE_SERVER_AFTER),
-            )
-            .await;
-            text += format!(" [{}]", until).as_str();
-        };
-
-        info!(context, "Partial download: {}", text);
-
-        self.parts.push(Part {
-            typ: Viewtype::Text,
-            msg: text,
-            ..Default::default()
-        });
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::chat::send_msg;
-    use crate::constants::Viewtype;
-    use crate::dc_receive_imf::dc_receive_imf_inner;
-    use crate::ephemeral::Timer;
-    use crate::test_utils::TestContext;
-    use num_traits::FromPrimitive;
-
-    #[test]
-    fn test_downloadstate_values() {
-        // values may be written to disk and must not change
-        assert_eq!(DownloadState::Done, DownloadState::default());
-        assert_eq!(DownloadState::Done, DownloadState::from_i32(0).unwrap());
-        assert_eq!(
-            DownloadState::Available,
-            DownloadState::from_i32(10).unwrap()
-        );
-        assert_eq!(DownloadState::Failure, DownloadState::from_i32(20).unwrap());
-        assert_eq!(
-            DownloadState::InProgress,
-            DownloadState::from_i32(1000).unwrap()
-        );
-    }
-
-    #[async_std::test]
-    async fn test_download_limit() -> Result<()> {
-        let t = TestContext::new_alice().await;
-
-        assert_eq!(t.download_limit().await?, None);
-
-        t.set_config(Config::DownloadLimit, Some("200000")).await?;
-        assert_eq!(t.download_limit().await?, Some(200000));
-
-        t.set_config(Config::DownloadLimit, Some("20000")).await?;
-        assert_eq!(t.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
-
-        t.set_config(Config::DownloadLimit, None).await?;
-        assert_eq!(t.download_limit().await?, None);
-
-        for val in &["0", "-1", "-100", "", "foo"] {
-            t.set_config(Config::DownloadLimit, Some(val)).await?;
-            assert_eq!(t.download_limit().await?, None);
-        }
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_update_download_state() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat = t.create_chat_with_contact("Bob", "bob@example.org").await;
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("Hi Bob".to_owned()));
-        let msg_id = send_msg(&t, chat.id, &mut msg).await?;
-        let msg = Message::load_from_db(&t, msg_id).await?;
-        assert_eq!(msg.download_state(), DownloadState::Done);
-
-        for s in &[
-            DownloadState::Available,
-            DownloadState::InProgress,
-            DownloadState::Failure,
-            DownloadState::Done,
-        ] {
-            msg_id.update_download_state(&t, *s).await?;
-            let msg = Message::load_from_db(&t, msg_id).await?;
-            assert_eq!(msg.download_state(), *s);
-        }
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_partial_receive_imf() -> Result<()> {
-        let t = TestContext::new_alice().await;
-
-        let header =
-            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-             From: bob@example.com\n\
-             To: alice@example.org\n\
-             Subject: foo\n\
-             Message-ID: <Mr.12345678901@example.com>\n\
-             Chat-Version: 1.0\n\
-             Date: Sun, 22 Mar 2020 22:37:57 +0000\
-             Content-Type: text/plain";
-
-        dc_receive_imf_inner(&t, header.as_bytes(), "INBOX", false, Some(100000), false).await?;
-        let msg = t.get_last_msg().await;
-        assert_eq!(msg.download_state(), DownloadState::Available);
-        assert_eq!(msg.get_subject(), "foo");
-        assert!(msg
-            .get_text()
-            .unwrap()
-            .contains(&stock_str::partial_download_msg_body(&t, 100000).await));
-
-        dc_receive_imf_inner(
-            &t,
-            format!("{}\n\n100k text...", header).as_bytes(),
-            "INBOX",
-            false,
-            None,
-            false,
-        )
-        .await?;
-        let msg = t.get_last_msg().await;
-        assert_eq!(msg.download_state(), DownloadState::Done);
-        assert_eq!(msg.get_subject(), "foo");
-        assert_eq!(msg.get_text(), Some("100k text...".to_string()));
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_partial_download_and_ephemeral() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat_id = t
-            .create_chat_with_contact("bob", "bob@example.org")
-            .await
-            .id;
-        chat_id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
-            .await?;
-
-        // download message from bob partially, this must not change the ephemeral timer
-        dc_receive_imf_inner(
-            &t,
-            b"From: Bob <bob@example.org>\n\
-                    To: Alice <alice@example.org>\n\
-                    Chat-Version: 1.0\n\
-                    Subject: subject\n\
-                    Message-ID: <first@example.org>\n\
-                    Date: Sun, 14 Nov 2021 00:10:00 +0000\
-                    Content-Type: text/plain",
-            "INBOX",
-            false,
-            Some(100000),
-            false,
-        )
-        .await?;
-        assert_eq!(
-            chat_id.get_ephemeral_timer(&t).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        Ok(())
-    }
-}

src/e2ee.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashSet;
 
-use anyhow::{bail, format_err, Context as _, Result};
+use anyhow::{bail, ensure, format_err, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
@@ -121,9 +121,9 @@ impl EncryptHelper {
             .into_iter()
             .filter_map(|(state, addr)| state.map(|s| (s, addr)))
         {
-            let key = peerstate
-                .take_key(min_verified)
-                .with_context(|| format!("proper enc-key for {} missing, cannot encrypt", addr))?;
+            let key = peerstate.take_key(min_verified).ok_or_else(|| {
+                format_err!("proper enc-key for {} missing, cannot encrypt", addr)
+            })?;
             keyring.add(key);
         }
         keyring.add(self.public_key.clone());
@@ -161,24 +161,21 @@ pub async fn try_decrypt(
     let mut peerstate = Peerstate::from_addr(context, &from).await?;
 
     // Apply Autocrypt header
-    match Aheader::from_headers(&from, &mail.headers) {
-        Ok(Some(ref header)) => {
-            if let Some(ref mut peerstate) = peerstate {
-                peerstate.apply_header(header, message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            } else {
-                let p = Peerstate::from_header(header, message_time);
-                p.save_to_db(&context.sql, true).await?;
-                peerstate = Some(p);
-            }
+    if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
+        if let Some(ref mut peerstate) = peerstate {
+            peerstate.apply_header(header, message_time);
+            peerstate.save_to_db(&context.sql, false).await?;
+        } else {
+            let p = Peerstate::from_header(header, message_time);
+            p.save_to_db(&context.sql, true).await?;
+            peerstate = Some(p);
         }
-        Ok(None) => {}
-        Err(err) => warn!(context, "Failed to parse Autocrypt header: {}", err),
     }
 
     // Possibly perform decryption
     let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
     let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
+    let mut signatures = HashSet::default();
 
     if let Some(ref mut peerstate) = peerstate {
         peerstate
@@ -191,17 +188,14 @@ pub async fn try_decrypt(
         }
     }
 
-    let (out_mail, signatures) = match decrypt_if_autocrypt_message(
+    let out_mail = decrypt_if_autocrypt_message(
         context,
         mail,
         private_keyring,
         public_keyring_for_validate,
+        &mut signatures,
     )
-    .await?
-    {
-        Some((out_mail, signatures)) => (Some(out_mail), signatures),
-        None => (None, Default::default()),
-    };
+    .await?;
 
     if let Some(mut peerstate) = peerstate {
         // If message is not encrypted and it is not a read receipt, degrade encryption.
@@ -277,7 +271,8 @@ async fn decrypt_if_autocrypt_message(
     mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
+) -> Result<Option<Vec<u8>>> {
     let encrypted_data_part = match get_autocrypt_mime(mail).or_else(|| get_mixed_up_mime(mail)) {
         None => {
             // not an autocrypt mime message, abort and ignore
@@ -291,60 +286,36 @@ async fn decrypt_if_autocrypt_message(
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
+        ret_valid_signatures,
     )
     .await
 }
 
-/// Validates signatures of Multipart/Signed message part, as defined in RFC 1847.
-///
-/// Returns `None` if the part is not a Multipart/Signed part, otherwise retruns the set of key
-/// fingerprints for which there is a valid signature.
-async fn validate_detached_signature(
-    mail: &ParsedMail<'_>,
-    public_keyring_for_validate: &Keyring<SignedPublicKey>,
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    if mail.ctype.mimetype != "multipart/signed" {
-        return Ok(None);
-    }
-
-    if let [first_part, second_part] = &mail.subparts[..] {
-        // First part is the content, second part is the signature.
-        let content = first_part.raw_bytes;
-        let signature = second_part.get_body_raw()?;
-        let ret_valid_signatures =
-            pgp::pk_validate(content, &signature, public_keyring_for_validate).await?;
-
-        Ok(Some((content.to_vec(), ret_valid_signatures)))
-    } else {
-        Ok(None)
-    }
-}
-
 /// Returns Ok(None) if nothing encrypted was found.
 async fn decrypt_part(
     mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
+) -> Result<Option<Vec<u8>>> {
     let data = mail.get_body_raw()?;
 
     if has_decrypted_pgp_armor(&data) {
-        let (plain, ret_valid_signatures) =
-            pgp::pk_decrypt(data, private_keyring, &public_keyring_for_validate).await?;
+        // we should only have one decryption happening
+        ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
 
-        // Check for detached signatures.
-        // If decrypted part is a multipart/signed, then there is a detached signature.
-        let decrypted_part = mailparse::parse_mail(&plain)?;
-        if let Some((content, valid_detached_signatures)) =
-            validate_detached_signature(&decrypted_part, &public_keyring_for_validate).await?
-        {
-            return Ok(Some((content, valid_detached_signatures)));
-        } else {
-            // If the message was wrongly or not signed, still return the plain text.
-            // The caller has to check the signatures then.
+        let plain = pgp::pk_decrypt(
+            data,
+            private_keyring,
+            public_keyring_for_validate,
+            Some(ret_valid_signatures),
+        )
+        .await?;
 
-            return Ok(Some((plain, ret_valid_signatures)));
-        }
+        // If the message was wrongly or not signed, still return the plain text.
+        // The caller has to check the signatures then.
+
+        return Ok(Some(plain));
     }
 
     Ok(None)
@@ -390,10 +361,12 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
     let self_addr = context
         .get_config(Config::ConfiguredAddr)
         .await?
-        .context(concat!(
-            "Failed to get self address, ",
-            "cannot ensure secret key if not configured."
-        ))?;
+        .ok_or_else(|| {
+            format_err!(concat!(
+                "Failed to get self address, ",
+                "cannot ensure secret key if not configured."
+            ))
+        })?;
     SignedPublicKey::load_self(context).await?;
     Ok(self_addr)
 }
@@ -414,11 +387,9 @@ mod tests {
 
         #[async_std::test]
         async fn test_prexisting() {
-            let t = TestContext::new_alice().await;
-            assert_eq!(
-                ensure_secret_key_exists(&t).await.unwrap(),
-                "alice@example.org"
-            );
+            let t = TestContext::new().await;
+            let test_addr = t.configure_alice().await;
+            assert_eq!(ensure_secret_key_exists(&t).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
@@ -489,7 +460,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert!(!msg.was_encrypted());
 
         // Parsing a message is enough to update peerstate
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
@@ -520,28 +491,28 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
         let msg = bob.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
         // Alice sends plaintext message with Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
-        msg.force_plaintext();
+        msg.param.set_int(Param::ForcePlaintext, 1);
         chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
         chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
         let sent = alice.pop_sent_msg().await;
 
         let msg = bob.parse_msg(&sent).await;
         assert!(!msg.was_encrypted());
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
         // Alice sends plaintext message without Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
-        msg.force_plaintext();
+        msg.param.set_int(Param::ForcePlaintext, 1);
         msg.param.set_int(Param::SkipAutocrypt, 1);
         chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
         chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
@@ -549,7 +520,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
         let msg = bob.parse_msg(&sent).await;
         assert!(!msg.was_encrypted());
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Reset);

src/ephemeral.rs

@@ -52,7 +52,7 @@
 //! `MsgsChanged` event is emitted when a message deletion is due, to
 //! make UI reload displayed messages and cause actual deletion.
 //!
-//! Server deletion happens by updating the `imap` table based on
+//! Server deletion happens by generating IMAP deletion jobs based on
 //! the database entries which are expired either according to their
 //! ephemeral message timers or global `delete_server_after` setting.
 
@@ -71,12 +71,11 @@ use crate::constants::{
 };
 use crate::context::Context;
 use crate::dc_tools::time;
-use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
+use crate::job;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::stock_str;
-use std::cmp::max;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -262,7 +261,7 @@ pub(crate) async fn stock_ephemeral_timer_changed(
 
 impl MsgId {
     /// Returns ephemeral message timer value for the message.
-    pub(crate) async fn ephemeral_timer(self, context: &Context) -> Result<Timer> {
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> anyhow::Result<Timer> {
         let res = match context
             .sql
             .query_get_value(
@@ -278,7 +277,7 @@ impl MsgId {
     }
 
     /// Starts ephemeral message timer for the message if it is not started yet.
-    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> Result<()> {
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> anyhow::Result<()> {
         if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
             let ephemeral_timestamp = time().saturating_add(duration.into());
 
@@ -433,36 +432,36 @@ pub async fn schedule_ephemeral_task(context: &Context) {
     }
 }
 
-/// Schedules expired IMAP messages for deletion.
-pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<()> {
+/// Returns ID of any expired message that should be deleted from the server.
+///
+/// It looks up the trash chat too, to find messages that are already
+/// deleted locally, but not deleted on the server.
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
     let now = time();
 
-    let (threshold_timestamp, threshold_timestamp_extended) =
-        match context.get_config_delete_server_after().await? {
-            None => (0, 0),
-            Some(delete_server_after) => (
-                now - delete_server_after,
-                now - max(delete_server_after, MIN_DELETE_SERVER_AFTER),
-            ),
-        };
+    let threshold_timestamp = match context.get_config_delete_server_after().await? {
+        None => 0,
+        Some(delete_server_after) => now - delete_server_after,
+    };
 
     context
         .sql
-        .execute(
-            "UPDATE imap
-             SET target=''
-             WHERE EXISTS (
-               SELECT * FROM msgs
-               WHERE rfc724_mid=imap.rfc724_mid
-               AND ((download_state = 0 AND timestamp < ?) OR
-                    (download_state != 0 AND timestamp < ?) OR
-                    (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?))
-             )",
-            paramsv![threshold_timestamp, threshold_timestamp_extended, now],
+        .query_row_optional(
+            "SELECT id FROM msgs \
+         WHERE ( \
+         timestamp < ? \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
+         ) \
+         AND server_uid != 0 \
+         AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
+         LIMIT 1",
+            paramsv![threshold_timestamp, now, job::Action::DeleteMsgOnImap],
+            |row| {
+                let msg_id: MsgId = row.get(0)?;
+                Ok(msg_id)
+            },
         )
-        .await?;
-
-    Ok(())
+        .await
 }
 
 /// Start ephemeral timers for seen messages if they are not started
@@ -497,10 +496,10 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
 
 #[cfg(test)]
 mod tests {
+    use crate::param::Params;
+    use async_std::task::sleep;
+
     use super::*;
-    use crate::config::Config;
-    use crate::dc_receive_imf::dc_receive_imf;
-    use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
         chat::{self, Chat, ChatItem},
@@ -636,83 +635,8 @@ mod tests {
         );
     }
 
-    /// Test enabling and disabling ephemeral timer remotely.
     #[async_std::test]
-    async fn test_ephemeral_enable_disable() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-
-        let chat_alice = alice.create_chat(&bob).await.id;
-        let chat_bob = bob.create_chat(&alice).await.id;
-
-        chat_alice
-            .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
-            .await?;
-        let sent = alice.pop_sent_msg().await;
-        bob.recv_msg(&sent).await;
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        chat_alice
-            .set_ephemeral_timer(&alice.ctx, Timer::Disabled)
-            .await?;
-        let sent = alice.pop_sent_msg().await;
-        bob.recv_msg(&sent).await;
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Disabled
-        );
-
-        Ok(())
-    }
-
-    /// Test that timer is enabled even if the message explicitly enabling the timer is lost.
-    #[async_std::test]
-    async fn test_ephemeral_enable_lost() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-
-        let chat_alice = alice.create_chat(&bob).await.id;
-        let chat_bob = bob.create_chat(&alice).await.id;
-
-        // Alice enables the timer.
-        chat_alice
-            .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
-            .await?;
-        assert_eq!(
-            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-        // The message enabling the timer is lost.
-        let _sent = alice.pop_sent_msg().await;
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Disabled,
-        );
-
-        // Alice sends a text message.
-        let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
-
-        // Bob receives text message and enables the timer, even though explicit timer update was
-        // lost previously.
-        bob.recv_msg(&sent).await;
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        Ok(())
-    }
-
-    /// Test that Alice replying to the chat without a timer at the same time as Bob enables the
-    /// timer does not result in disabling the timer on the Bob's side.
-    #[async_std::test]
-    async fn test_ephemeral_timer_rollback() -> Result<()> {
+    async fn test_ephemeral_timer() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
@@ -770,30 +694,18 @@ mod tests {
             Timer::Enabled { duration: 60 }
         );
 
-        // Bob disables the chat timer.
-        // Note that the last message in the Bob's chat is from Alice and has no timer,
-        // but the chat timer is enabled.
-        chat_bob
-            .set_ephemeral_timer(&bob.ctx, Timer::Disabled)
-            .await?;
-        alice.recv_msg(&bob.pop_sent_msg().await).await;
-        assert_eq!(
-            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
-            Timer::Disabled
-        );
-
         Ok(())
     }
 
     #[async_std::test]
-    async fn test_ephemeral_delete_msgs() -> Result<()> {
+    async fn test_ephemeral_delete_msgs() {
         let t = TestContext::new_alice().await;
         let chat = t.get_self_chat().await;
 
         t.send_text(chat.id, "Saved message, which we delete manually")
             .await;
         let msg = t.get_last_msg_in(chat.id).await;
-        msg.id.delete_from_db(&t).await?;
+        msg.id.delete_from_db(&t).await.unwrap();
         check_msg_was_deleted(&t, &chat, msg.id).await;
 
         chat.id
@@ -804,12 +716,36 @@ mod tests {
             .send_text(chat.id, "Saved message, disappearing after 1s")
             .await;
 
-        async_std::task::sleep(Duration::from_millis(1100)).await;
+        sleep(Duration::from_millis(1100)).await;
 
-        // Check that the msg was deleted locally.
+        // Check checks that the msg was deleted locally
         check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
 
-        Ok(())
+        // Check that the msg will be deleted on the server
+        // First of all, set a server_uid so that DC thinks that it's actually possible to delete
+        t.sql
+            .execute(
+                "UPDATE msgs SET server_uid=1 WHERE id=?",
+                paramsv![msg.sender_msg_id],
+            )
+            .await
+            .unwrap();
+        let job = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(
+            job,
+            Some(job::Job::new(
+                job::Action::DeleteMsgOnImap,
+                msg.sender_msg_id.to_u32(),
+                Params::new(),
+                0,
+            ))
+        );
+        // Let's assume that executing the job fails on first try and the job is saved to the db
+        job.unwrap().save(&t).await.unwrap();
+
+        // Make sure that we don't get yet another job when loading from db
+        let job2 = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(job2, None);
     }
 
     async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
@@ -835,202 +771,4 @@ mod tests {
             assert!(rawtxt.is_none_or_empty(), "{:?}", rawtxt);
         }
     }
-
-    #[async_std::test]
-    async fn test_delete_expired_imap_messages() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        const HOUR: i64 = 60 * 60;
-        let now = time();
-        for (id, timestamp, ephemeral_timestamp) in &[
-            (900, now - 2 * HOUR, 0),
-            (1000, now - 23 * HOUR - MIN_DELETE_SERVER_AFTER, 0),
-            (1010, now - 23 * HOUR, 0),
-            (1020, now - 21 * HOUR, 0),
-            (1030, now - 19 * HOUR, 0),
-            (2000, now - 18 * HOUR, now - HOUR),
-            (2020, now - 17 * HOUR, now + HOUR),
-        ] {
-            let message_id = id.to_string();
-            t.sql
-                   .execute(
-                       "INSERT INTO msgs (id, rfc724_mid, timestamp, ephemeral_timestamp) VALUES (?,?,?,?);",
-                       paramsv![id, message_id, timestamp, ephemeral_timestamp],
-                   )
-                   .await?;
-            t.sql
-                   .execute(
-                       "INSERT INTO imap (rfc724_mid, folder, uid, target) VALUES (?,'INBOX',?, 'INBOX');",
-                       paramsv![message_id, id],
-                   )
-                   .await?;
-        }
-
-        async fn test_marked_for_deletion(context: &Context, id: u32) -> Result<()> {
-            assert_eq!(
-                context
-                    .sql
-                    .count(
-                        "SELECT COUNT(*) FROM imap WHERE target='' AND rfc724_mid=?",
-                        paramsv![id.to_string()],
-                    )
-                    .await?,
-                1
-            );
-            Ok(())
-        }
-
-        async fn remove_uid(context: &Context, id: u32) -> Result<()> {
-            context
-                .sql
-                .execute(
-                    "DELETE FROM imap WHERE rfc724_mid=?",
-                    paramsv![id.to_string()],
-                )
-                .await?;
-            Ok(())
-        }
-
-        // This should mark message 2000 for deletion.
-        delete_expired_imap_messages(&t).await?;
-        test_marked_for_deletion(&t, 2000).await?;
-        remove_uid(&t, 2000).await?;
-        // No other messages are marked for deletion.
-        assert_eq!(
-            t.sql
-                .count("SELECT COUNT(*) FROM imap WHERE target=''", paramsv![],)
-                .await?,
-            0
-        );
-
-        t.set_config(Config::DeleteServerAfter, Some(&*(25 * HOUR).to_string()))
-            .await?;
-        delete_expired_imap_messages(&t).await?;
-        test_marked_for_deletion(&t, 1000).await?;
-
-        MsgId::new(1000)
-            .update_download_state(&t, DownloadState::Available)
-            .await?;
-        t.sql
-            .execute(
-                "UPDATE imap SET target=folder WHERE rfc724_mid='1000'",
-                paramsv![],
-            )
-            .await?;
-        delete_expired_imap_messages(&t).await?;
-        test_marked_for_deletion(&t, 1000).await?; // Delete downloadable anyway.
-        remove_uid(&t, 1000).await?;
-
-        t.set_config(Config::DeleteServerAfter, Some(&*(22 * HOUR).to_string()))
-            .await?;
-        delete_expired_imap_messages(&t).await?;
-        test_marked_for_deletion(&t, 1010).await?;
-        t.sql
-            .execute(
-                "UPDATE imap SET target=folder WHERE rfc724_mid='1010'",
-                paramsv![],
-            )
-            .await?;
-
-        MsgId::new(1010)
-            .update_download_state(&t, DownloadState::Available)
-            .await?;
-        delete_expired_imap_messages(&t).await?;
-        // Keep downloadable for now.
-        assert_eq!(
-            t.sql
-                .count("SELECT COUNT(*) FROM imap WHERE target=''", paramsv![],)
-                .await?,
-            0
-        );
-
-        Ok(())
-    }
-
-    // Regression test for a bug in the timer rollback protection.
-    #[async_std::test]
-    async fn test_ephemeral_timer_references() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        // Message with Message-ID <first@example.com> and no timer is received.
-        dc_receive_imf(
-            &alice,
-            b"From: Bob <bob@example.com>\n\
-                    To: Alice <alice@example.org>\n\
-                    Chat-Version: 1.0\n\
-                    Subject: Subject\n\
-                    Message-ID: <first@example.com>\n\
-                    Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
-                    \n\
-                    hello\n",
-            "INBOX",
-            false,
-        )
-        .await?;
-
-        let msg = alice.get_last_msg().await;
-        let chat_id = msg.chat_id;
-        assert_eq!(chat_id.get_ephemeral_timer(&alice).await?, Timer::Disabled);
-
-        // Message with Message-ID <second@example.com> is received.
-        dc_receive_imf(
-            &alice,
-            b"From: Bob <bob@example.com>\n\
-                    To: Alice <alice@example.org>\n\
-                    Chat-Version: 1.0\n\
-                    Subject: Subject\n\
-                    Message-ID: <second@example.com>\n\
-                    Date: Sun, 22 Mar 2020 00:11:00 +0000\n\
-                    Ephemeral-Timer: 60\n\
-                    \n\
-                    second message\n",
-            "INBOX",
-            false,
-        )
-        .await?;
-        assert_eq!(
-            chat_id.get_ephemeral_timer(&alice).await?,
-            Timer::Enabled { duration: 60 }
-        );
-        let msg = alice.get_last_msg().await;
-
-        // Message is deleted from the database when its timer expires.
-        msg.id.delete_from_db(&alice).await?;
-
-        // Message with Message-ID <third@example.com>, referencing <first@example.com> and
-        // <second@example.com>, is received.  The message <second@example.come> is not in the
-        // database anymore, so the timer should be applied unconditionally without rollback
-        // protection.
-        //
-        // Previously Delta Chat fallen back to using <first@example.com> in this case and
-        // compared received timer value to the timer value of the <first@examle.com>. Because
-        // their timer values are the same ("disabled"), Delta Chat assumed that the timer was not
-        // changed explicitly and the change should be ignored.
-        //
-        // The message also contains a quote of the first message to test that only References:
-        // header and not In-Reply-To: is consulted by the rollback protection.
-        dc_receive_imf(
-            &alice,
-            b"From: Bob <bob@example.com>\n\
-                    To: Alice <alice@example.org>\n\
-                    Chat-Version: 1.0\n\
-                    Subject: Subject\n\
-                    Message-ID: <third@example.com>\n\
-                    Date: Sun, 22 Mar 2020 00:12:00 +0000\n\
-                    References: <first@example.com> <second@example.com>\n\
-                    In-Reply-To: <first@example.com>\n\
-                    \n\
-                    > hello\n",
-            "INBOX",
-            false,
-        )
-        .await?;
-
-        let msg = alice.get_last_msg().await;
-        assert_eq!(
-            msg.chat_id.get_ephemeral_timer(&alice).await?,
-            Timer::Disabled
-        );
-
-        Ok(())
-    }
 }

src/events.rs

@@ -9,7 +9,6 @@ use strum::EnumProperty;
 use crate::chat::ChatId;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
-use crate::webxdc::StatusUpdateId;
 
 #[derive(Debug)]
 pub struct Events {
@@ -63,6 +62,10 @@ impl Events {
 pub struct EventEmitter(Receiver<Event>);
 
 impl EventEmitter {
+    pub(crate) fn into_inner(self) -> Receiver<Event> {
+        self.0
+    }
+
     /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
     pub fn recv_sync(&self) -> Option<Event> {
         async_std::task::block_on(self.recv())
@@ -200,8 +203,7 @@ pub enum EventType {
     /// - Chats created, deleted or archived
     /// - A draft has been set
     ///
-    /// `chat_id` is set if only a single chat is affected by the changes, otherwise 0.
-    /// `msg_id` is set if only a single message is affected by the changes, otherwise 0.
+    /// The `chat_id` and `msg_id` values will be 0 if more than one message is changed.
     #[strum(props(id = "2000"))]
     MsgsChanged { chat_id: ChatId, msg_id: MsgId },
 
@@ -324,13 +326,4 @@ pub enum EventType {
     /// dc_get_connectivity_html() for details.
     #[strum(props(id = "2100"))]
     ConnectivityChanged,
-
-    #[strum(props(id = "2110"))]
-    SelfavatarChanged,
-
-    #[strum(props(id = "2120"))]
-    WebxdcStatusUpdate {
-        msg_id: MsgId,
-        status_update_id: StatusUpdateId,
-    },
 }

src/headerdef.rs

@@ -1,8 +1,9 @@
 //! # List of email headers.
 
+use crate::strum::AsStaticRef;
 use mailparse::{MailHeader, MailHeaderMap};
 
-#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, IntoStaticStr)]
+#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
 #[strum(serialize_all = "kebab_case")]
 pub enum HeaderDef {
     MessageId,
@@ -33,7 +34,6 @@ pub enum HeaderDef {
     XMozillaDraftInfo,
 
     ListId,
-    ListPost,
     References,
     InReplyTo,
     Precedence,
@@ -67,9 +67,9 @@ pub enum HeaderDef {
 }
 
 impl HeaderDef {
-    /// Returns the corresponding header string.
+    /// Returns the corresponding Event id.
     pub fn get_headername(&self) -> &'static str {
-        self.into()
+        self.as_static()
     }
 }
 

src/html.rs

@@ -440,7 +440,9 @@ test some special html-characters as < > and & but also " and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
         assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
@@ -466,7 +468,7 @@ test some special html-characters as < > and & but also " and &#x
 
         // bob: check that bob also got the html-part of the forwarded message
         let bob = TestContext::new_bob().await;
-        let chat = bob.create_chat_with_contact("", "alice@example.org").await;
+        let chat = bob.create_chat_with_contact("", "alice@example.com").await;
         bob.recv_msg(&alice.pop_sent_msg().await).await;
         let msg = bob.get_last_msg_in(chat.get_id()).await;
         assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
@@ -489,7 +491,9 @@ test some special html-characters as < > and & but also " and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
 
         // forward the message to saved-messages,
@@ -556,6 +560,7 @@ test some special html-characters as < > and & but also " and &#x
             &t,
             include_bytes!("../test-data/message/cp1252-html.eml"),
             "INBOX",
+            0,
             false,
         )
         .await?;

src/imap/idle.rs

@@ -1,9 +1,8 @@
 use super::Imap;
 
-use anyhow::{bail, Context as _, Result};
+use anyhow::{bail, format_err, Result};
 use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
-use imap_proto::types::{AttributeValue, Response};
 use std::time::{Duration, SystemTime};
 
 use crate::{context::Context, scheduler::InterruptInfo};
@@ -32,7 +31,7 @@ impl Imap {
         let timeout = Duration::from_secs(23 * 60);
         let mut info = Default::default();
 
-        if self.server_sent_unsolicited_exists(context)? {
+        if self.server_sent_unsolicited_exists(context) {
             return Ok(info);
         }
 
@@ -72,13 +71,6 @@ impl Imap {
             match fut.await {
                 Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
                     info!(context, "Idle has NewData {:?}", x);
-                    if let Response::Fetch(_message, attrs) = x.parsed() {
-                        for attr in attrs {
-                            if let AttributeValue::ModSeq(modseq) = attr {
-                                self.update_modseq(*modseq);
-                            }
-                        }
-                    }
                 }
                 Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
                     info!(context, "Idle-wait timeout or interruption");
@@ -98,8 +90,8 @@ impl Imap {
             let session = handle
                 .done()
                 .timeout(Duration::from_secs(15))
-                .await?
-                .context("IMAP IDLE protocol timed out")?;
+                .await
+                .map_err(|err| format_err!("IMAP IDLE protocol timed out: {}", err))??;
             self.session = Some(Session { inner: session });
         } else {
             warn!(context, "Attempted to idle without a session");
@@ -158,7 +150,7 @@ impl Imap {
                     }
                     if self.config.can_idle {
                         // we only fake-idled because network was gone during IDLE, probably
-                        break InterruptInfo::new(false);
+                        break InterruptInfo::new(false, None);
                     }
                     info!(context, "fake_idle is connected");
                     // we are connected, let's see if fetching messages results
@@ -170,7 +162,7 @@ impl Imap {
                         Ok(res) => {
                             info!(context, "fetch_new_messages returned {:?}", res);
                             if res {
-                                break InterruptInfo::new(false);
+                                break InterruptInfo::new(false, None);
                             }
                         }
                         Err(err) => {

src/imap/scan_folders.rs

@@ -10,7 +10,7 @@ use async_std::prelude::*;
 use super::{get_folder_meaning, get_folder_meaning_by_name};
 
 impl Imap {
-    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<()> {
+    pub async fn scan_folders(&mut self, context: &Context) -> Result<()> {
         // First of all, debounce to once per minute:
         let mut last_scan = context.last_full_folder_scan.lock().await;
         if let Some(last_scan) = *last_scan {
@@ -29,7 +29,7 @@ impl Imap {
         let session = self.session.as_mut();
         let session = session.context("scan_folders(): IMAP No Connection established")?;
         let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
-        let watched_folders = get_watched_folders(context).await?;
+        let watched_folders = get_watched_folders(context).await;
 
         let mut folder_configs = BTreeMap::new();
 
@@ -71,15 +71,15 @@ impl Imap {
             // Don't scan folders that are watched anyway
             if !watched_folders.contains(&folder.name().to_string()) && !is_drafts {
                 // Drain leftover unsolicited EXISTS messages
-                self.server_sent_unsolicited_exists(context)?;
+                self.server_sent_unsolicited_exists(context);
 
                 loop {
-                    self.fetch_move_delete(context, folder.name())
+                    self.fetch_new_messages(context, folder.name(), false)
                         .await
                         .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
 
                     // If the server sent an unsocicited EXISTS during the fetch, we need to fetch again
-                    if !self.server_sent_unsolicited_exists(context)? {
+                    if !self.server_sent_unsolicited_exists(context) {
                         break;
                     }
                 }
@@ -102,21 +102,19 @@ impl Imap {
     }
 }
 
-pub(crate) async fn get_watched_folders(context: &Context) -> Result<Vec<String>> {
+pub(crate) async fn get_watched_folders(context: &Context) -> Vec<String> {
     let mut res = Vec::new();
-    if let Some(inbox_folder) = context.get_config(Config::ConfiguredInboxFolder).await? {
-        res.push(inbox_folder);
-    }
     let folder_watched_configured = &[
         (Config::SentboxWatch, Config::ConfiguredSentboxFolder),
-        (Config::MvboxMove, Config::ConfiguredMvboxFolder),
+        (Config::MvboxWatch, Config::ConfiguredMvboxFolder),
+        (Config::InboxWatch, Config::ConfiguredInboxFolder),
     ];
     for (watched, configured) in folder_watched_configured {
-        if context.get_config_bool(*watched).await? {
-            if let Some(folder) = context.get_config(*configured).await? {
+        if context.get_config_bool(*watched).await.unwrap_or_default() {
+            if let Ok(Some(folder)) = context.get_config(*configured).await {
                 res.push(folder);
             }
         }
     }
-    Ok(res)
+    res
 }

src/imap/select_folder.rs

@@ -15,9 +15,6 @@ pub enum Error {
     #[error("IMAP Folder name invalid: {0}")]
     BadFolderName(String),
 
-    #[error("IMAP folder does not exist: {0}")]
-    NoFolder(String),
-
     #[error("IMAP close/expunge failed")]
     CloseExpungeFailed(#[from] async_imap::error::Error),
 
@@ -93,11 +90,7 @@ impl Imap {
         // select new folder
         if let Some(folder) = folder {
             if let Some(ref mut session) = &mut self.session {
-                let res = if self.config.can_condstore {
-                    session.select_condstore(folder).await
-                } else {
-                    session.select(folder).await
-                };
+                let res = session.select(folder).await;
 
                 // <https://tools.ietf.org/html/rfc3501#section-6.3.1>
                 // says that if the server reports select failure we are in
@@ -117,9 +110,6 @@ impl Imap {
                     Err(async_imap::error::Error::Validate(_)) => {
                         Err(Error::BadFolderName(folder.to_string()))
                     }
-                    Err(async_imap::error::Error::No(_)) => {
-                        Err(Error::NoFolder(folder.to_string()))
-                    }
                     Err(err) => {
                         self.config.selected_folder = None;
                         self.trigger_reconnect(context).await;
@@ -133,28 +123,6 @@ impl Imap {
             Ok(NewlySelected::No)
         }
     }
-
-    /// Selects a folder. Tries to create it once and select again if the folder does not exist.
-    pub(super) async fn select_or_create_folder(
-        &mut self,
-        context: &Context,
-        folder: &str,
-    ) -> Result<NewlySelected> {
-        match self.select_folder(context, Some(folder)).await {
-            Ok(newly_selected) => Ok(newly_selected),
-            Err(err) => match err {
-                Error::NoFolder(_) => {
-                    if let Some(ref mut session) = self.session {
-                        session.create(folder).await?;
-                    } else {
-                        return Err(Error::NoSession);
-                    }
-                    self.select_folder(context, Some(folder)).await
-                }
-                _ => Err(err),
-            },
-        }
-    }
 }
 #[derive(PartialEq, Debug, Copy, Clone, Eq)]
 pub(super) enum NewlySelected {

src/imex.rs

@@ -19,8 +19,8 @@ use crate::config::Config;
 use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::context::Context;
 use crate::dc_tools::{
-    dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
-    dc_open_file_std, dc_read_file, dc_write_file, time, EmailAddress,
+    dc_copy_file, dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
+    dc_open_file_std, dc_read_file, dc_write_file, get_next_backup_path, time, EmailAddress,
 };
 use crate::e2ee;
 use crate::events::EventType;
@@ -30,7 +30,7 @@ use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::pgp;
-use crate::sql;
+use crate::sql::{self, Sql};
 use crate::stock_str;
 
 // Name of the database file in the backup.
@@ -41,24 +41,24 @@ const BLOBS_BACKUP_NAME: &str = "blobs_backup";
 #[repr(u32)]
 pub enum ImexMode {
     /// Export all private keys and all public keys of the user to the
-    /// directory given as `path`.  The default key is written to the files `public-key-default.asc`
+    /// directory given as `param1`.  The default key is written to the files `public-key-default.asc`
     /// and `private-key-default.asc`, if there are more keys, they are written to files as
     /// `public-key-<id>.asc` and `private-key-<id>.asc`
     ExportSelfKeys = 1,
 
-    /// Import private keys found in the directory given as `path`.
+    /// Import private keys found in the directory given as `param1`.
     /// The last imported key is made the default keys unless its name contains the string `legacy`.
     /// Public keys are not imported.
     ImportSelfKeys = 2,
 
-    /// Export a backup to the directory given as `path` with the given `passphrase`.
+    /// Export a backup to the directory given as `param1`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
     /// The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
     /// the format is `delta-chat-<day>-<number>.tar`
     ExportBackup = 11,
 
-    /// `path` is the file (not: directory) to import. The file is normally
+    /// `param1` is the file (not: directory) to import. The file is normally
     /// created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup
     /// is only possible as long as the context is not configured or used in another way.
     ImportBackup = 12,
@@ -78,16 +78,11 @@ pub enum ImexMode {
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
-pub async fn imex(
-    context: &Context,
-    what: ImexMode,
-    path: &Path,
-    passphrase: Option<String>,
-) -> Result<()> {
+pub async fn imex(context: &Context, what: ImexMode, param1: &Path) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
 
     let res = async {
-        let success = imex_inner(context, what, path, passphrase).await;
+        let success = imex_inner(context, what, param1).await;
         match success {
             Ok(()) => {
                 info!(context, "IMEX successfully completed");
@@ -120,10 +115,15 @@ async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
         dc_delete_file(context, context.get_dbfile()).await;
         dc_delete_files_in_dir(context, context.get_blobdir()).await;
     }
+    if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
+        if let Err(e) = context.sql.open(context, context.get_dbfile(), false).await {
+            warn!(context, "Re-opening db after imex failed: {}", e);
+        }
+    }
 }
 
 /// Returns the filename of the backup found (otherwise an error)
-pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result<String> {
+pub async fn has_backup(context: &Context, dir_name: &Path) -> Result<String> {
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_name = "".to_string();
     let mut newest_backup_path: Option<PathBuf> = None;
@@ -145,6 +145,59 @@ pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result<String> {
         }
     }
 
+    match newest_backup_path {
+        Some(path) => Ok(path.to_string_lossy().into_owned()),
+        None => has_backup_old(context, dir_name).await,
+        // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`.
+    }
+}
+
+/// Returns the filename of the backup found (otherwise an error)
+pub async fn has_backup_old(context: &Context, dir_name: &Path) -> Result<String> {
+    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
+    let mut newest_backup_time = 0;
+    let mut newest_backup_name = "".to_string();
+    let mut newest_backup_path: Option<PathBuf> = None;
+    while let Some(dirent) = dir_iter.next().await {
+        if let Ok(dirent) = dirent {
+            let path = dirent.path();
+            let name = dirent.file_name();
+            let name = name.to_string_lossy();
+            if name.starts_with("delta-chat") && name.ends_with(".bak") {
+                let sql = Sql::new();
+                match sql.open(context, &path, true).await {
+                    Ok(_) => {
+                        let curr_backup_time = sql
+                            .get_raw_config_int("backup_time")
+                            .await?
+                            .unwrap_or_default();
+                        if curr_backup_time > newest_backup_time {
+                            newest_backup_path = Some(path);
+                            newest_backup_time = curr_backup_time;
+                        }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close().await;
+                    }
+                    Err(e) => {
+                        warn!(
+                            context,
+                            "Found backup file {} which could not be opened: {}", name, e
+                        );
+                        // On some Android devices we can't open sql files that are not in our private directory
+                        // (see <https://github.com/deltachat/deltachat-android/issues/1768>). So, compare names
+                        // to still find the newest backup.
+                        let name: String = name.into();
+                        if newest_backup_time == 0
+                            && (newest_backup_name.is_empty() || name > newest_backup_name)
+                        {
+                            newest_backup_path = Some(path);
+                            newest_backup_name = name;
+                        }
+                    }
+                }
+            }
+        }
+    }
     match newest_backup_path {
         Some(path) => Ok(path.to_string_lossy().into_owned()),
         None => bail!("no backup found in {}", dir_name.display()),
@@ -185,7 +238,7 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.force_plaintext();
+    msg.param.set_int(Param::ForcePlaintext, 1);
     msg.param.set_int(Param::SkipAutocrypt, 1);
 
     let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
@@ -396,12 +449,7 @@ fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-async fn imex_inner(
-    context: &Context,
-    what: ImexMode,
-    path: &Path,
-    passphrase: Option<String>,
-) -> Result<()> {
+async fn imex_inner(context: &Context, what: ImexMode, path: &Path) -> Result<()> {
     info!(context, "Import/export dir: {}", path.display());
     ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
@@ -419,27 +467,19 @@ async fn imex_inner(
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
 
-        ImexMode::ExportBackup => {
-            export_backup(context, path, passphrase.unwrap_or_default()).await
-        }
-        ImexMode::ImportBackup => {
-            import_backup(context, path, passphrase.unwrap_or_default()).await?;
-            context.sql.run_migrations(context).await
-        }
+        ImexMode::ExportBackup => export_backup(context, path).await,
+        // import_backup() will call import_backup_old() if this is an old backup.
+        ImexMode::ImportBackup => import_backup(context, path).await,
     }
 }
 
-/// Imports backup into the currently open database.
-///
-/// The contents of the currently open database will be lost.
-///
-/// `passphrase` is the passphrase used to open backup database. If backup is unencrypted, pass
-/// empty string here.
-async fn import_backup(
-    context: &Context,
-    backup_to_import: &Path,
-    passphrase: String,
-) -> Result<()> {
+/// Import Backup
+async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()> {
+    if backup_to_import.to_string_lossy().ends_with(".bak") {
+        // Backwards compability
+        return import_backup_old(context, backup_to_import).await;
+    }
+
     info!(
         context,
         "Import \"{}\" to \"{}\".",
@@ -455,6 +495,12 @@ async fn import_backup(
         !context.scheduler.read().await.is_running(),
         "cannot import backup, IO already running"
     );
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
+    ensure!(
+        !context.get_dbfile().exists().await,
+        "Cannot delete old database."
+    );
 
     let backup_file = File::open(backup_to_import).await?;
     let file_size = backup_file.metadata().await?.len();
@@ -476,15 +522,11 @@ async fn import_backup(
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
             // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
             f.unpack_in(context.get_blobdir()).await?;
-            let unpacked_database = context.get_blobdir().join(DBFILE_BACKUP_NAME);
-            context
-                .sql
-                .import(&unpacked_database, passphrase.clone())
-                .await
-                .context("cannot import unpacked database")?;
-            fs::remove_file(unpacked_database)
-                .await
-                .context("cannot remove unpacked database")?;
+            fs::rename(
+                context.get_blobdir().join(DBFILE_BACKUP_NAME),
+                context.get_dbfile(),
+            )
+            .await?;
         } else {
             // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
             f.unpack_in(context.get_blobdir()).await?;
@@ -499,52 +541,136 @@ async fn import_backup(
         }
     }
 
+    context
+        .sql
+        .open(context, context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
+
     delete_and_reset_all_device_msgs(context).await?;
 
     Ok(())
 }
 
+async fn import_backup_old(context: &Context, backup_to_import: &Path) -> Result<()> {
+    info!(
+        context,
+        "Import \"{}\" to \"{}\".",
+        backup_to_import.display(),
+        context.get_dbfile().display()
+    );
+
+    ensure!(
+        !context.is_configured().await?,
+        "Cannot import backups to accounts in use."
+    );
+    ensure!(
+        !context.scheduler.read().await.is_running(),
+        "cannot import backup, IO already running"
+    );
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
+    ensure!(
+        !context.get_dbfile().exists().await,
+        "Cannot delete old database."
+    );
+
+    ensure!(
+        dc_copy_file(context, backup_to_import, context.get_dbfile()).await,
+        "could not copy file"
+    );
+    /* error already logged */
+    /* re-open copied database file */
+    context
+        .sql
+        .open(context, context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
+
+    delete_and_reset_all_device_msgs(context).await?;
+
+    let total_files_cnt = context
+        .sql
+        .count("SELECT COUNT(*) FROM backup_blobs;", paramsv![])
+        .await?;
+    info!(
+        context,
+        "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
+    );
+
+    // Load IDs only for now, without the file contents, to avoid
+    // consuming too much memory.
+    let file_ids = context
+        .sql
+        .query_map(
+            "SELECT id FROM backup_blobs ORDER BY id",
+            paramsv![],
+            |row| row.get(0),
+            |ids| {
+                ids.collect::<std::result::Result<Vec<i64>, _>>()
+                    .map_err(Into::into)
+            },
+        )
+        .await?;
+
+    let mut all_files_extracted = true;
+    for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
+        // Load a single blob into memory
+        let (file_name, file_blob) = context
+            .sql
+            .query_row(
+                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
+                paramsv![file_id],
+                |row| {
+                    let file_name: String = row.get(0)?;
+                    let file_blob: Vec<u8> = row.get(1)?;
+                    Ok((file_name, file_blob))
+                },
+            )
+            .await?;
+
+        if context.shall_stop_ongoing().await {
+            all_files_extracted = false;
+            break;
+        }
+        let mut permille = processed_files_cnt * 1000 / total_files_cnt;
+        if permille < 10 {
+            permille = 10
+        }
+        if permille > 990 {
+            permille = 990
+        }
+        context.emit_event(EventType::ImexProgress(permille));
+        if file_blob.is_empty() {
+            continue;
+        }
+
+        let path_filename = context.get_blobdir().join(file_name);
+        dc_write_file(context, &path_filename, &file_blob).await?;
+    }
+
+    if all_files_extracted {
+        // only delete backup_blobs if all files were successfully extracted
+        context
+            .sql
+            .execute("DROP TABLE backup_blobs;", paramsv![])
+            .await?;
+        context.sql.execute("VACUUM;", paramsv![]).await.ok();
+        Ok(())
+    } else {
+        bail!("received stop signal");
+    }
+}
+
 /*******************************************************************************
  * Export backup
  ******************************************************************************/
-
-/// Returns Ok((temp_db_path, temp_path, dest_path)) on success. Unencrypted database can be
-/// written to temp_db_path. The backup can then be written to temp_path. If the backup succeeded,
-/// it can be renamed to dest_path. This guarantees that the backup is complete.
-async fn get_next_backup_path(
-    folder: &Path,
-    backup_time: i64,
-) -> Result<(PathBuf, PathBuf, PathBuf)> {
-    let folder = PathBuf::from(folder);
-    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
-        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
-        .format("delta-chat-backup-%Y-%m-%d")
-        .to_string();
-
-    // 64 backup files per day should be enough for everyone
-    for i in 0..64 {
-        let mut tempdbfile = folder.clone();
-        tempdbfile.push(format!("{}-{:02}.db", stem, i));
-
-        let mut tempfile = folder.clone();
-        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
-
-        let mut destfile = folder.clone();
-        destfile.push(format!("{}-{:02}.tar", stem, i));
-
-        if !tempdbfile.exists().await && !tempfile.exists().await && !destfile.exists().await {
-            return Ok((tempdbfile, tempfile, destfile));
-        }
-    }
-    bail!("could not create backup file, disk full?");
-}
-
-async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Result<()> {
+#[allow(unused)]
+async fn export_backup(context: &Context, dir: &Path) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
-    let (temp_db_path, temp_path, dest_path) = get_next_backup_path(dir, now).await?;
-    let _d1 = DeleteOnDrop(temp_db_path.clone());
-    let _d2 = DeleteOnDrop(temp_path.clone());
+    let (temp_path, dest_path) = get_next_backup_path(dir, now).await?;
+    let _d = DeleteOnDrop(temp_path.clone());
 
     context
         .sql
@@ -556,14 +682,16 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
         .sql
         .execute("VACUUM;", paramsv![])
         .await
-        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e))
-        .ok();
+        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
 
     ensure!(
         !context.scheduler.read().await.is_running(),
         "cannot export backup, IO already running"
     );
 
+    // we close the database during the export
+    context.sql.close().await;
+
     info!(
         context,
         "Backup '{}' to '{}'.",
@@ -571,13 +699,10 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
         dest_path.display(),
     );
 
-    context
-        .sql
-        .export(&temp_db_path, passphrase)
-        .await
-        .with_context(|| format!("failed to backup plaintext database to {:?}", temp_db_path))?;
+    let res = export_backup_inner(context, &temp_path).await;
 
-    let res = export_backup_inner(context, &temp_db_path, &temp_path).await;
+    // we re-open the database after export is finished
+    context.sql.open(context, context.get_dbfile(), false).await;
 
     match &res {
         Ok(_) => {
@@ -596,21 +721,18 @@ impl Drop for DeleteOnDrop {
     fn drop(&mut self) {
         let file = self.0.clone();
         // Not using dc_delete_file() here because it would send a DeletedBlobFile event
-        async_std::task::block_on(fs::remove_file(file)).ok();
+        async_std::task::block_on(async move { fs::remove_file(file).await.ok() });
     }
 }
 
-async fn export_backup_inner(
-    context: &Context,
-    temp_db_path: &Path,
-    temp_path: &Path,
-) -> Result<()> {
+async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
     let file = File::create(temp_path).await?;
 
     let mut builder = async_tar::Builder::new(file);
 
+    // append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument.
     builder
-        .append_path_with_name(temp_db_path, DBFILE_BACKUP_NAME)
+        .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
         .await?;
 
     let read_dir: Vec<_> = fs::read_dir(context.get_blobdir()).await?.collect().await;
@@ -814,7 +936,9 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file() {
-        let t = TestContext::new_alice().await;
+        let t = TestContext::new().await;
+
+        t.configure_alice().await;
         let msg = render_setup_file(&t, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
@@ -831,10 +955,11 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
-        let t = TestContext::new_alice().await;
+        let t = TestContext::new().await;
         t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
+        t.configure_alice().await;
         let msg = render_setup_file(&t, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
@@ -887,14 +1012,16 @@ mod tests {
 
     #[async_std::test]
     async fn test_export_and_import_key() {
-        let context = TestContext::new_alice().await;
+        let context = TestContext::new().await;
+        context.configure_alice().await;
         let blobdir = context.ctx.get_blobdir();
-        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir, None).await {
+        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir).await {
             panic!("got error on export: {:?}", err);
         }
 
-        let context2 = TestContext::new_alice().await;
-        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir, None).await {
+        let context2 = TestContext::new().await;
+        context2.configure_alice().await;
+        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir).await {
             panic!("got error on import: {:?}", err);
         }
     }

src/job.rs

@@ -3,24 +3,30 @@
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
 use std::fmt;
+use std::future::Future;
 
-use anyhow::{bail, format_err, Context as _, Error, Result};
+use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
+use async_smtp::smtp::response::{Category, Code, Detail};
 use deltachat_derive::{FromSql, ToSql};
+use itertools::Itertools;
 use rand::{thread_rng, Rng};
 
+use crate::blob::BlobObject;
+use crate::chat::{self, ChatId};
 use crate::config::Config;
 use crate::contact::{normalize_name, Contact, Modifier, Origin};
 use crate::context::Context;
-use crate::dc_tools::time;
+use crate::dc_tools::{dc_delete_file, dc_read_file, time};
+use crate::ephemeral::load_imap_deletion_msgid;
 use crate::events::EventType;
 use crate::imap::{Imap, ImapActionResult};
 use crate::location;
 use crate::log::LogExt;
-use crate::message::{Message, MsgId};
+use crate::message::{self, Message, MessageState, MsgId};
 use crate::mimefactory::MimeFactory;
 use crate::param::{Param, Params};
 use crate::scheduler::InterruptInfo;
-use crate::smtp::{smtp_send, Smtp};
+use crate::smtp::Smtp;
 use crate::sql;
 
 // results in ~3 weeks for the last backoff timespan
@@ -91,11 +97,10 @@ pub enum Action {
     // this is user initiated so it should have a fairly high priority
     UpdateRecentQuota = 140,
 
-    // This job will download partially downloaded messages completely
-    // and is added when download_full() is called.
-    // Most messages are downloaded automatically on fetch
-    // and do not go through this job.
-    DownloadMsg = 250,
+    // Moving message is prioritized lower than deletion so we don't
+    // bother moving message if it is already scheduled for deletion.
+    MoveMsg = 200,
+    DeleteMsgOnImap = 210,
 
     // UID synchronization is high-priority to make sure correct UIDs
     // are used by message moving/deletion.
@@ -105,6 +110,7 @@ pub enum Action {
     MaybeSendLocations = 5005, // low priority ...
     MaybeSendLocationsEnded = 5007,
     SendMdn = 5010,
+    SendMsgToSmtp = 5901, // ... high priority
 }
 
 impl Default for Action {
@@ -122,14 +128,16 @@ impl From<Action> for Thread {
 
             Housekeeping => Thread::Imap,
             FetchExistingMsgs => Thread::Imap,
+            DeleteMsgOnImap => Thread::Imap,
             ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
+            MoveMsg => Thread::Imap,
             UpdateRecentQuota => Thread::Imap,
-            DownloadMsg => Thread::Imap,
 
             MaybeSendLocations => Thread::Smtp,
             MaybeSendLocationsEnded => Thread::Smtp,
             SendMdn => Thread::Smtp,
+            SendMsgToSmtp => Thread::Smtp,
         }
     }
 }
@@ -220,6 +228,210 @@ impl Job {
         Ok(())
     }
 
+    async fn smtp_send<F, Fut>(
+        &mut self,
+        context: &Context,
+        recipients: Vec<async_smtp::EmailAddress>,
+        message: Vec<u8>,
+        job_id: u32,
+        smtp: &mut Smtp,
+        success_cb: F,
+    ) -> Status
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = Result<()>>,
+    {
+        // hold the smtp lock during sending of a job and
+        // its ok/error response processing. Note that if a message
+        // was sent we need to mark it in the database ASAP as we
+        // otherwise might send it twice.
+        if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
+            info!(context, "smtp-sending out mime message:");
+            println!("{}", String::from_utf8_lossy(&message));
+        }
+
+        smtp.connectivity.set_working(context).await;
+
+        let send_result = smtp.send(context, recipients, message, job_id).await;
+        smtp.last_send_error = send_result.as_ref().err().map(|e| e.to_string());
+
+        let status = match send_result {
+            Err(crate::smtp::send::Error::SmtpSend(err)) => {
+                // Remote error, retry later.
+                warn!(context, "SMTP failed to send: {:?}", &err);
+
+                let res = match err {
+                    async_smtp::smtp::error::Error::Permanent(ref response) => {
+                        // Workaround for incorrectly configured servers returning permanent errors
+                        // instead of temporary ones.
+                        let maybe_transient = match response.code {
+                            // Sometimes servers send a permanent error when actually it is a temporary error
+                            // For documentation see <https://tools.ietf.org/html/rfc3463>
+                            Code {
+                                category: Category::MailSystem,
+                                detail: Detail::Zero,
+                                ..
+                            } => {
+                                // Ignore status code 5.5.0, see <https://support.delta.chat/t/every-other-message-gets-stuck/877/2>
+                                // Maybe incorrectly configured Postfix milter with "reject" instead of "tempfail", which returns
+                                // "550 5.5.0 Service unavailable" instead of "451 4.7.1 Service unavailable - try again later".
+                                //
+                                // Other enhanced status codes, such as Postfix
+                                // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
+                                // are not ignored.
+                                response.first_word() == Some(&"5.5.0".to_string())
+                            }
+                            _ => false,
+                        };
+
+                        if maybe_transient {
+                            Status::RetryLater
+                        } else {
+                            // If we do not retry, add an info message to the chat.
+                            // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
+                            // should definitely go here, because user has to open the link to
+                            // resume message sending.
+                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                        }
+                    }
+                    async_smtp::smtp::error::Error::Transient(ref response) => {
+                        // We got a transient 4xx response from SMTP server.
+                        // Give some time until the server-side error maybe goes away.
+
+                        if let Some(first_word) = response.first_word() {
+                            if first_word.ends_with(".1.1")
+                                || first_word.ends_with(".1.2")
+                                || first_word.ends_with(".1.3")
+                            {
+                                // Sometimes we receive transient errors that should be permanent.
+                                // Any extended smtp status codes like x.1.1, x.1.2 or x.1.3 that we
+                                // receive as a transient error are misconfigurations of the smtp server.
+                                // See <https://tools.ietf.org/html/rfc3463#section-3.2>
+                                info!(context, "Smtp-job #{} Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", self.job_id, first_word);
+                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } else {
+                                Status::RetryLater
+                            }
+                        } else {
+                            Status::RetryLater
+                        }
+                    }
+                    _ => {
+                        if smtp.has_maybe_stale_connection().await {
+                            info!(context, "stale connection? immediately reconnecting");
+                            Status::RetryNow
+                        } else {
+                            Status::RetryLater
+                        }
+                    }
+                };
+
+                // this clears last_success info
+                smtp.disconnect().await;
+
+                res
+            }
+            Err(crate::smtp::send::Error::Envelope(err)) => {
+                // Local error, job is invalid, do not retry.
+                smtp.disconnect().await;
+                warn!(context, "SMTP job is invalid: {}", err);
+                Status::Finished(Err(err.into()))
+            }
+            Err(crate::smtp::send::Error::NoTransport) => {
+                // Should never happen.
+                // It does not even make sense to disconnect here.
+                error!(context, "SMTP job failed because SMTP has no transport");
+                Status::Finished(Err(format_err!("SMTP has not transport")))
+            }
+            Err(crate::smtp::send::Error::Other(err)) => {
+                // Local error, job is invalid, do not retry.
+                smtp.disconnect().await;
+                warn!(context, "unable to load job: {}", err);
+                Status::Finished(Err(err))
+            }
+            Ok(()) => {
+                job_try!(success_cb().await);
+                Status::Finished(Ok(()))
+            }
+        };
+
+        if let Status::Finished(Err(err)) = &status {
+            // We couldn't send the message, so mark it as failed
+            let msg_id = MsgId::new(self.foreign_id);
+            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+        }
+        status
+    }
+
+    pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+        //  SMTP server, if not yet done
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            smtp.last_send_error = Some(format!("SMTP connection failure: {:#}", err));
+            return Status::RetryLater;
+        }
+
+        let filename = job_try!(job_try!(self
+            .param
+            .get_path(Param::File, context)
+            .map_err(|_| format_err!("Can't get filename")))
+        .ok_or_else(|| format_err!("Can't get filename")));
+        let body = job_try!(dc_read_file(context, &filename).await);
+        let recipients = job_try!(self.param.get(Param::Recipients).ok_or_else(|| {
+            warn!(context, "Missing recipients for job {}", self.job_id);
+            format_err!("Missing recipients")
+        }));
+
+        let recipients_list = recipients
+            .split('\x1e')
+            .filter_map(
+                |addr| match async_smtp::EmailAddress::new(addr.to_string()) {
+                    Ok(addr) => Some(addr),
+                    Err(err) => {
+                        warn!(context, "invalid recipient: {} {:?}", addr, err);
+                        None
+                    }
+                },
+            )
+            .collect::<Vec<_>>();
+
+        /* if there is a msg-id and it does not exist in the db, cancel sending.
+        this happends if dc_delete_msgs() was called
+        before the generated mime was sent out */
+        if 0 != self.foreign_id {
+            match message::exists(context, MsgId::new(self.foreign_id)).await {
+                Ok(exists) => {
+                    if !exists {
+                        return Status::Finished(Err(format_err!(
+                            "Not sending Message {} as it was deleted",
+                            self.foreign_id
+                        )));
+                    }
+                }
+                Err(err) => {
+                    warn!(context, "failed to check message existence: {:?}", err);
+                    smtp.last_send_error =
+                        Some(format!("failed to check message existence: {:#}", err));
+                    return Status::RetryLater;
+                }
+            }
+        };
+
+        let foreign_id = self.foreign_id;
+        self.smtp_send(context, recipients_list, body, self.job_id, smtp, || {
+            async move {
+                // smtp success, update db ASAP, then delete smtp file
+                if 0 != foreign_id {
+                    set_delivered(context, MsgId::new(foreign_id)).await?;
+                }
+                // now also delete the generated file
+                dc_delete_file(context, filename).await;
+                Ok(())
+            }
+        })
+        .await
+    }
+
     /// Get `SendMdn` jobs with foreign_id equal to `contact_id` excluding the `job_id` job.
     async fn get_additional_mdn_jobs(
         &self,
@@ -318,13 +530,157 @@ impl Job {
             return Status::RetryLater;
         }
 
-        let status = smtp_send(context, &recipients, &body, smtp, msg_id, 0).await;
-        if matches!(status, Status::Finished(Ok(_))) {
-            // Remove additional SendMdn jobs we have aggregated into this one.
-            job_try!(kill_ids(context, &additional_job_ids).await);
+        self.smtp_send(context, recipients, body, self.job_id, smtp, || {
+            async move {
+                // Remove additional SendMdn jobs we have aggregated into this one.
+                kill_ids(context, &additional_job_ids).await?;
+                Ok(())
+            }
+        })
+        .await
+    }
+
+    async fn move_msg(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.prepare(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
         }
 
-        status
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let server_folder = &job_try!(msg
+            .server_folder
+            .context("Can't move message out of folder if we don't know the current folder"));
+
+        let move_res = msg.id.needs_move(context, server_folder).await;
+        let dest_folder = match move_res {
+            Err(e) => {
+                warn!(context, "could not load dest folder: {}", e);
+                return Status::RetryLater;
+            }
+            Ok(None) => {
+                warn!(
+                    context,
+                    "msg {} does not need to be moved from {}", msg.id, server_folder
+                );
+                return Status::Finished(Ok(()));
+            }
+            Ok(Some(config)) => match context.get_config(config).await {
+                Ok(folder) => folder,
+                Err(err) => {
+                    warn!(context, "failed to load config: {}", err);
+                    return Status::RetryLater;
+                }
+            },
+        };
+
+        if let Some(dest_folder) = dest_folder {
+            match imap
+                .mv(context, server_folder, msg.server_uid, &dest_folder)
+                .await
+            {
+                ImapActionResult::RetryLater => Status::RetryLater,
+                ImapActionResult::Success => {
+                    // Rust-Imap provides no target uid on mv, so just set it to 0, update again when precheck_imf() is called for the moved message
+                    message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, 0).await;
+                    Status::Finished(Ok(()))
+                }
+                ImapActionResult::Failed => {
+                    Status::Finished(Err(format_err!("IMAP action failed")))
+                }
+                ImapActionResult::AlreadyDone => Status::Finished(Ok(())),
+            }
+        } else {
+            Status::Finished(Err(format_err!("No mvbox folder configured")))
+        }
+    }
+
+    /// Deletes a message on the server.
+    ///
+    /// `foreign_id` is a MsgId.
+    ///
+    /// If the message is in the trash chat or hidden, this job
+    /// removes database record, otherwise it only clears the
+    /// `server_uid` column.  If there are no more records pointing to
+    /// the same message on the server, the job actually removes the
+    /// message on the server.
+    async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.prepare(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+
+        if !msg.rfc724_mid.is_empty() {
+            let cnt = message::rfc724_mid_cnt(context, &msg.rfc724_mid).await;
+            info!(
+                context,
+                "Running delete job for message {} which has {} entries in the database",
+                &msg.rfc724_mid,
+                cnt
+            );
+            if cnt > 1 {
+                info!(
+                    context,
+                    "The message is deleted from the server when all parts are deleted.",
+                );
+            } else if cnt == 0 {
+                warn!(
+                    context,
+                    "The message {} has no UID on the server to delete", &msg.rfc724_mid
+                );
+            } else {
+                /* if this is the last existing part of the message,
+                we delete the message from the server */
+                let mid = msg.rfc724_mid;
+                let server_folder = msg.server_folder.as_ref().unwrap();
+                let res = if msg.server_uid == 0 {
+                    // Message is already deleted on IMAP server.
+                    ImapActionResult::AlreadyDone
+                } else {
+                    imap.delete_msg(context, &mid, server_folder, msg.server_uid)
+                        .await
+                };
+                match res {
+                    ImapActionResult::AlreadyDone | ImapActionResult::Success => {}
+                    ImapActionResult::RetryLater | ImapActionResult::Failed => {
+                        // If job has failed, for example due to some
+                        // IMAP bug, we postpone it instead of failing
+                        // immediately. This will prevent adding it
+                        // immediately again if user has enabled
+                        // automatic message deletion. Without this,
+                        // we might waste a lot of traffic constantly
+                        // retrying message deletion.
+                        return Status::RetryLater;
+                    }
+                }
+            }
+            if msg.chat_id.is_trash() || msg.hidden {
+                // Messages are stored in trash chat only to keep
+                // their server UID and Message-ID. Once message is
+                // deleted from the server, database record can be
+                // removed as well.
+                //
+                // Hidden messages are similar to trashed, but are
+                // related to some chat. We also delete their
+                // database records.
+                job_try!(msg.id.delete_from_db(context).await)
+            } else {
+                // Remove server UID from the database record.
+                //
+                // We have either just removed the message from the
+                // server, in which case UID is not valid anymore, or
+                // we have more refernces to the same server UID, so
+                // we remove UID to reduce the number of messages
+                // pointing to the corresponding UID. Once the counter
+                // reaches zero, we will remove the message.
+                job_try!(msg.id.unlink(context).await);
+            }
+            Status::Finished(Ok(()))
+        } else {
+            /* eg. device messages have no Message-ID */
+            Status::Finished(Ok(()))
+        }
     }
 
     /// Read the recipients from old emails sent by the user and add them as contacts.
@@ -352,7 +708,7 @@ impl Job {
                 Config::ConfiguredSentboxFolder,
             ] {
                 if let Some(folder) = job_try!(context.get_config(*config).await) {
-                    if let Err(e) = imap.fetch_new_messages(context, &folder, true).await {
+                    if let Err(e) = imap.fetch_new_messages(context, folder, true).await {
                         // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
                         warn!(context, "Could not fetch messages, retrying: {:#}", e);
                         return Status::RetryLater;
@@ -365,7 +721,15 @@ impl Job {
         Status::Finished(Ok(()))
     }
 
-    /// Synchronizes UIDs for sentbox, inbox and mvbox.
+    /// Synchronizes UIDs for sentbox, inbox and mvbox, in this order.
+    ///
+    /// If a copy of the message is present in multiple folders, mvbox
+    /// is preferred to inbox, which is in turn preferred to
+    /// sentbox. This is because in the database it is impossible to
+    /// store multiple UIDs for one message, so we prefer to
+    /// automatically delete messages in the folders managed by Delta
+    /// Chat in contrast to the Sent folder, which is normally managed
+    /// by the user via webmail or another email client.
     async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.prepare(context).await {
             warn!(context, "could not connect: {:?}", err);
@@ -397,58 +761,55 @@ impl Job {
         }
 
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let row = job_try!(
-            context
-                .sql
-                .query_row_optional(
-                    "SELECT uid, folder FROM imap
-                    WHERE rfc724_mid=? AND folder=target
-                    ORDER BY uid ASC
-                    LIMIT 1",
-                    paramsv![msg.rfc724_mid],
-                    |row| {
-                        let uid: u32 = row.get(0)?;
-                        let folder: String = row.get(1)?;
-                        Ok((uid, folder))
-                    }
-                )
-                .await
-        );
-        if let Some((server_uid, server_folder)) = row {
-            let result = imap.set_seen(context, &server_folder, server_uid).await;
-            match result {
-                ImapActionResult::RetryLater => return Status::RetryLater,
-                ImapActionResult::Success | ImapActionResult::Failed => {}
-            }
+
+        let folder = msg.server_folder.as_ref().unwrap();
+
+        let result = if msg.server_uid == 0 {
+            // The message is moved or deleted by us.
+            //
+            // Do not call set_seen with zero UID, as it will return
+            // ImapActionResult::RetryLater, but we do not want to
+            // retry. If the message was moved, we will create another
+            // job to mark the message as seen later. If it was
+            // deleted, there is nothing to do.
+            info!(context, "Can't mark message as seen: No UID");
+            ImapActionResult::Failed
         } else {
-            info!(
-                context,
-                "Can't mark the message {} as seen on IMAP because there is no known UID",
-                msg.rfc724_mid
-            );
-        }
+            imap.set_seen(context, folder, msg.server_uid).await
+        };
 
-        // XXX we send MDN even in case of failure to mark the messages as seen, e.g. if it was
-        // already deleted on the server by another device. The job will not be retried so locally
-        // there is no risk of double-sending MDNs.
-        //
-        // Read receipts for system messages are never sent. These messages have no place to
-        // display received read receipt anyway.  And since their text is locally generated,
-        // quoting them is dangerous as it may contain contact names. E.g., for original message
-        // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
-        // be a display name stored in address book rather than the name sent in the From field by
-        // the user.
-        if msg.param.get_bool(Param::WantsMdn).unwrap_or_default() && !msg.is_system_message() {
-            let mdns_enabled = job_try!(context.get_config_bool(Config::MdnsEnabled).await);
-            if mdns_enabled {
-                if let Err(err) = send_mdn(context, &msg).await {
-                    warn!(context, "could not send out mdn for {}: {}", msg.id, err);
-                    return Status::Finished(Err(err));
+        match result {
+            ImapActionResult::RetryLater => Status::RetryLater,
+            ImapActionResult::AlreadyDone => Status::Finished(Ok(())),
+            ImapActionResult::Success | ImapActionResult::Failed => {
+                // XXX the message might just have been moved
+                // we want to send out an MDN anyway
+                // The job will not be retried so locally
+                // there is no risk of double-sending MDNs.
+                //
+                // Read receipts for system messages are never
+                // sent. These messages have no place to display
+                // received read receipt anyway.  And since their text
+                // is locally generated, quoting them is dangerous as
+                // it may contain contact names. E.g., for original
+                // message "Group left by me", a read receipt will
+                // quote "Group left by <name>", and the name can be a
+                // display name stored in address book rather than
+                // the name sent in the From field by the user.
+                if msg.param.get_bool(Param::WantsMdn).unwrap_or_default()
+                    && !msg.is_system_message()
+                {
+                    let mdns_enabled = job_try!(context.get_config_bool(Config::MdnsEnabled).await);
+                    if mdns_enabled {
+                        if let Err(err) = send_mdn(context, &msg).await {
+                            warn!(context, "could not send out mdn for {}: {}", msg.id, err);
+                            return Status::Finished(Err(err));
+                        }
+                    }
                 }
+                Status::Finished(Ok(()))
             }
         }
-
-        Status::Finished(Ok(()))
     }
 }
 
@@ -465,7 +826,7 @@ pub async fn kill_action(context: &Context, action: Action) -> Result<()> {
 async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
     let q = format!(
         "DELETE FROM jobs WHERE id IN({})",
-        job_ids.iter().map(|_| "?").collect::<Vec<&str>>().join(",")
+        job_ids.iter().map(|_| "?").join(",")
     );
     context
         .sql
@@ -474,15 +835,26 @@ async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
     Ok(())
 }
 
-pub async fn action_exists(context: &Context, action: Action) -> Result<bool> {
-    let exists = context
+pub async fn action_exists(context: &Context, action: Action) -> bool {
+    context
         .sql
         .exists(
             "SELECT COUNT(*) FROM jobs WHERE action=?;",
             paramsv![action],
         )
-        .await?;
-    Ok(exists)
+        .await
+        .unwrap_or_default()
+}
+
+async fn set_delivered(context: &Context, msg_id: MsgId) -> Result<()> {
+    message::update_msg_state(context, msg_id, MessageState::OutDelivered).await;
+    let chat_id: ChatId = context
+        .sql
+        .query_get_value("SELECT chat_id FROM msgs WHERE id=?", paramsv![msg_id])
+        .await?
+        .unwrap_or_default();
+    context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
+    Ok(())
 }
 
 async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
@@ -503,7 +875,7 @@ async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, fold
                 let display_name_normalized = contact
                     .display_name
                     .as_ref()
-                    .map(|s| normalize_name(s))
+                    .map(normalize_name)
                     .unwrap_or_default();
 
                 match Contact::add_or_lookup(
@@ -530,11 +902,138 @@ async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, fold
     };
 }
 
+/// Constructs a job for sending a message.
+///
+/// Returns `None` if no messages need to be sent out.
+///
+/// In order to be processed, must be `add`ded.
+pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job>> {
+    let mut msg = Message::load_from_db(context, msg_id).await?;
+    msg.try_calc_and_set_dimensions(context).await.ok();
+
+    /* create message */
+    let needs_encryption = msg.param.get_bool(Param::GuaranteeE2ee).unwrap_or_default();
+
+    let attach_selfavatar = match chat::shall_attach_selfavatar(context, msg.chat_id).await {
+        Ok(attach_selfavatar) => attach_selfavatar,
+        Err(err) => {
+            warn!(context, "job: cannot get selfavatar-state: {}", err);
+            false
+        }
+    };
+
+    let mimefactory = MimeFactory::from_msg(context, &msg, attach_selfavatar).await?;
+
+    let mut recipients = mimefactory.recipients();
+
+    let from = context
+        .get_config(Config::ConfiguredAddr)
+        .await?
+        .unwrap_or_default();
+    let lowercase_from = from.to_lowercase();
+
+    // Send BCC to self if it is enabled and we are not going to
+    // delete it immediately.
+    if context.get_config_bool(Config::BccSelf).await?
+        && context.get_config_delete_server_after().await? != Some(0)
+        && !recipients
+            .iter()
+            .any(|x| x.to_lowercase() == lowercase_from)
+    {
+        recipients.push(from);
+    }
+
+    if recipients.is_empty() {
+        // may happen eg. for groups with only SELF and bcc_self disabled
+        info!(
+            context,
+            "message {} has no recipient, skipping smtp-send", msg_id
+        );
+        set_delivered(context, msg_id).await?;
+        return Ok(None);
+    }
+
+    let rendered_msg = match mimefactory.render(context).await {
+        Ok(res) => Ok(res),
+        Err(err) => {
+            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+            Err(err)
+        }
+    }?;
+
+    if needs_encryption && !rendered_msg.is_encrypted {
+        /* unrecoverable */
+        message::set_msg_failed(
+            context,
+            msg_id,
+            Some("End-to-end-encryption unavailable unexpectedly."),
+        )
+        .await;
+        bail!(
+            "e2e encryption unavailable {} - {:?}",
+            msg_id,
+            needs_encryption
+        );
+    }
+
+    if rendered_msg.is_gossiped {
+        chat::set_gossiped_timestamp(context, msg.chat_id, time()).await?;
+    }
+
+    if 0 != rendered_msg.last_added_location_id {
+        if let Err(err) = location::set_kml_sent_timestamp(context, msg.chat_id, time()).await {
+            error!(context, "Failed to set kml sent_timestamp: {:?}", err);
+        }
+        if !msg.hidden {
+            if let Err(err) =
+                location::set_msg_location_id(context, msg.id, rendered_msg.last_added_location_id)
+                    .await
+            {
+                error!(context, "Failed to set msg_location_id: {:?}", err);
+            }
+        }
+    }
+
+    if attach_selfavatar {
+        if let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, time()).await {
+            error!(context, "Failed to set selfavatar timestamp: {:?}", err);
+        }
+    }
+
+    if rendered_msg.is_encrypted && !needs_encryption {
+        msg.param.set_int(Param::GuaranteeE2ee, 1);
+        msg.update_param(context).await;
+    }
+
+    ensure!(!recipients.is_empty(), "no recipients for smtp job set");
+    let mut param = Params::new();
+    let bytes = &rendered_msg.message;
+    let blob = BlobObject::create(context, &rendered_msg.rfc724_mid, bytes).await?;
+
+    let recipients = recipients.join("\x1e");
+    param.set(Param::File, blob.as_name());
+    param.set(Param::Recipients, &recipients);
+
+    msg.subject = rendered_msg.subject.clone();
+    msg.update_subject(context).await;
+
+    let job = create(Action::SendMsgToSmtp, msg_id.to_u32(), param, 0)?;
+
+    Ok(Some(job))
+}
+
 pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
     Smtp(&'a mut Smtp),
 }
 
+pub(crate) async fn load_imap_deletion_job(context: &Context) -> Result<Option<Job>> {
+    let res = load_imap_deletion_msgid(context)
+        .await?
+        .map(|msg_id| Job::new(Action::DeleteMsgOnImap, msg_id.to_u32(), Params::new(), 0));
+    Ok(res)
+}
+
 impl<'a> fmt::Display for Connection<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
@@ -578,7 +1077,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
                     "{} thread increases job {} tries to {}", &connection, job, tries
                 );
                 job.tries = tries;
-                let time_offset = get_backoff_time_offset(tries, job.action);
+                let time_offset = get_backoff_time_offset(tries);
                 job.desired_timestamp = time() + time_offset;
                 info!(
                     context,
@@ -637,13 +1136,16 @@ async fn perform_job_action(
 
     let try_res = match job.action {
         Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))),
+        Action::SendMsgToSmtp => job.send_msg_to_smtp(context, connection.smtp()).await,
         Action::SendMdn => job.send_mdn(context, connection.smtp()).await,
         Action::MaybeSendLocations => location::job_maybe_send_locations(context, job).await,
         Action::MaybeSendLocationsEnded => {
             location::job_maybe_send_locations_ended(context, job).await
         }
+        Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
+        Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
         Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::Housekeeping => {
             sql::housekeeping(context).await.ok_or_log(context);
@@ -653,7 +1155,6 @@ async fn perform_job_action(
             Ok(status) => status,
             Err(err) => Status::Finished(Err(err)),
         },
-        Action::DownloadMsg => job.download_msg(context, connection.inbox()).await,
     };
 
     info!(context, "Finished immediate try {} of job {}", tries, job);
@@ -661,31 +1162,22 @@ async fn perform_job_action(
     try_res
 }
 
-fn get_backoff_time_offset(tries: u32, action: Action) -> i64 {
-    match action {
-        // Just try every 10s to update the quota
-        // If all retries are exhausted, a new job will be created when the quota information is needed
-        Action::UpdateRecentQuota => 10,
-
-        _ => {
-            // Exponential backoff
-            let n = 2_i32.pow(tries - 1) * 60;
-            let mut rng = thread_rng();
-            let r: i32 = rng.gen();
-            let mut seconds = r % (n + 1);
-            if seconds < 1 {
-                seconds = 1;
-            }
-            seconds as i64
-        }
+fn get_backoff_time_offset(tries: u32) -> i64 {
+    let n = 2_i32.pow(tries - 1) * 60;
+    let mut rng = thread_rng();
+    let r: i32 = rng.gen();
+    let mut seconds = r % (n + 1);
+    if seconds < 1 {
+        seconds = 1;
     }
+    seconds as i64
 }
 
 async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     let mut param = Params::new();
     param.set(Param::MsgId, msg.id.to_u32().to_string());
 
-    add(context, Job::new(Action::SendMdn, msg.from_id, param, 0)).await?;
+    add(context, Job::new(Action::SendMdn, msg.from_id, param, 0)).await;
 
     Ok(())
 }
@@ -696,35 +1188,54 @@ pub(crate) async fn schedule_resync(context: &Context) -> Result<()> {
         context,
         Job::new(Action::ResyncFolders, 0, Params::new(), 0),
     )
-    .await?;
+    .await;
     Ok(())
 }
 
+/// Creates a job.
+pub fn create(action: Action, foreign_id: u32, param: Params, delay_seconds: i64) -> Result<Job> {
+    ensure!(
+        action != Action::Unknown,
+        "Invalid action passed to job_add"
+    );
+
+    Ok(Job::new(action, foreign_id, param, delay_seconds))
+}
+
 /// Adds a job to the database, scheduling it.
-pub async fn add(context: &Context, job: Job) -> Result<()> {
+pub async fn add(context: &Context, job: Job) {
     let action = job.action;
     let delay_seconds = job.delay_seconds();
-    job.save(context).await.context("failed to save job")?;
+    job.save(context).await.unwrap_or_else(|err| {
+        error!(context, "failed to save job: {}", err);
+    });
 
     if delay_seconds == 0 {
         match action {
             Action::Unknown => unreachable!(),
             Action::Housekeeping
+            | Action::DeleteMsgOnImap
             | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
             | Action::FetchExistingMsgs
-            | Action::UpdateRecentQuota
-            | Action::DownloadMsg => {
+            | Action::MoveMsg
+            | Action::UpdateRecentQuota => {
                 info!(context, "interrupt: imap");
-                context.interrupt_inbox(InterruptInfo::new(false)).await;
+                context
+                    .interrupt_inbox(InterruptInfo::new(false, None))
+                    .await;
             }
-            Action::MaybeSendLocations | Action::MaybeSendLocationsEnded | Action::SendMdn => {
+            Action::MaybeSendLocations
+            | Action::MaybeSendLocationsEnded
+            | Action::SendMdn
+            | Action::SendMsgToSmtp => {
                 info!(context, "interrupt: smtp");
-                context.interrupt_smtp(InterruptInfo::new(false)).await;
+                context
+                    .interrupt_smtp(InterruptInfo::new(false, None))
+                    .await;
             }
         }
     }
-    Ok(())
 }
 
 async fn load_housekeeping_job(context: &Context) -> Result<Option<Job>> {
@@ -755,9 +1266,20 @@ pub(crate) async fn load_next(
     let query;
     let params;
     let t = time();
+    let m;
     let thread_i = thread as i64;
 
-    if !info.probe_network {
+    if let Some(msg_id) = info.msg_id {
+        query = r#"
+SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
+FROM jobs
+WHERE thread=? AND foreign_id=?
+ORDER BY action DESC, added_timestamp
+LIMIT 1;
+"#;
+        m = msg_id;
+        params = paramsv![thread_i, m];
+    } else if !info.probe_network {
         // processing for first-try and after backoff-timeouts:
         // process jobs in the order they were added.
         query = r#"
@@ -827,6 +1349,12 @@ LIMIT 1;
         }
         Thread::Imap => {
             if let Some(job) = job {
+                if job.action < Action::DeleteMsgOnImap {
+                    Ok(load_imap_deletion_job(context).await?.or(Some(job)))
+                } else {
+                    Ok(Some(job))
+                }
+            } else if let Some(job) = load_imap_deletion_job(context).await? {
                 Ok(Some(job))
             } else {
                 Ok(load_housekeeping_job(context).await?)
@@ -852,12 +1380,8 @@ mod tests {
                  VALUES (?, ?, ?, ?, ?, ?);",
                 paramsv![
                     now,
-                    Thread::from(Action::DownloadMsg),
-                    if valid {
-                        Action::DownloadMsg as i32
-                    } else {
-                        -1
-                    },
+                    Thread::from(Action::MoveMsg),
+                    if valid { Action::MoveMsg as i32 } else { -1 },
                     foreign_id,
                     Params::new().to_string(),
                     now
@@ -876,8 +1400,8 @@ mod tests {
         insert_job(&t, 1, false).await; // This can not be loaded into Job struct.
         let jobs = load_next(
             &t,
-            Thread::from(Action::DownloadMsg),
-            &InterruptInfo::new(false),
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
         )
         .await?;
         // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
@@ -886,8 +1410,8 @@ mod tests {
         insert_job(&t, 1, true).await;
         let jobs = load_next(
             &t,
-            Thread::from(Action::DownloadMsg),
-            &InterruptInfo::new(false),
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
         )
         .await?;
         assert!(jobs.is_some());
@@ -902,8 +1426,8 @@ mod tests {
 
         let jobs = load_next(
             &t,
-            Thread::from(Action::DownloadMsg),
-            &InterruptInfo::new(false),
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
         )
         .await?;
         assert!(jobs.is_some());

src/key.rs

@@ -4,7 +4,7 @@ use std::collections::BTreeMap;
 use std::fmt;
 use std::io::Cursor;
 
-use anyhow::{format_err, Context as _, Result};
+use anyhow::{format_err, Result};
 use async_trait::async_trait;
 use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
@@ -50,7 +50,8 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     /// the ASCII-armored representation.
     fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
         let bytes = data.as_bytes();
-        Self::KeyType::from_armor_single(Cursor::new(bytes)).context("rPGP error")
+        Self::KeyType::from_armor_single(Cursor::new(bytes))
+            .map_err(|err| format_err!("rPGP error: {}", err))
     }
 
     /// Load the users' default key from the database.
@@ -201,7 +202,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     let addr = context
         .get_config(Config::ConfiguredAddr)
         .await?
-        .context("no address configured")?;
+        .ok_or_else(|| format_err!("No address configured"))?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -288,13 +289,13 @@ pub async fn store_self_keypair(
             paramsv![public_key, secret_key],
         )
         .await
-        .context("failed to remove old use of key")?;
+        .map_err(|err| err.context("failed to remove old use of key"))?;
     if default == KeyPairUse::Default {
         context
             .sql
             .execute("UPDATE keypairs SET is_default=0;", paramsv![])
             .await
-            .context("failed to clear default")?;
+            .map_err(|err| err.context("failed to clear default"))?;
     }
     let is_default = match default {
         KeyPairUse::Default => true as i32,
@@ -312,7 +313,7 @@ pub async fn store_self_keypair(
             paramsv![addr, is_default, public_key, secret_key, t],
         )
         .await
-        .context("failed to insert keypair")?;
+        .map_err(|err| err.context("failed to insert keypair"))?;
 
     Ok(())
 }
@@ -509,7 +510,8 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
-        let t = TestContext::new_alice().await;
+        let t = TestContext::new().await;
+        t.configure_alice().await;
         let pubkey = SignedPublicKey::load_self(&t).await.unwrap();
         assert_eq!(alice.public, pubkey);
         let seckey = SignedSecretKey::load_self(&t).await.unwrap();
@@ -519,7 +521,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_generate_public() {
         let t = TestContext::new().await;
-        t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
         let key = SignedPublicKey::load_self(&t).await;
@@ -529,7 +531,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_generate_secret() {
         let t = TestContext::new().await;
-        t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
         let key = SignedSecretKey::load_self(&t).await;
@@ -541,7 +543,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         use std::thread;
 
         let t = TestContext::new().await;
-        t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
             .unwrap();
         let thr0 = {
@@ -587,6 +589,27 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(nrows().await, 1);
     }
 
+    // Convenient way to create a new key if you need one, run with
+    // `cargo test key::tests::gen_key`.
+    // #[test]
+    // fn gen_key() {
+    //     let name = "fiona";
+    //     let keypair = crate::pgp::create_keypair(
+    //         EmailAddress::new(&format!("{}@example.net", name)).unwrap(),
+    //     )
+    //     .unwrap();
+    //     std::fs::write(
+    //         format!("test-data/key/{}-public.asc", name),
+    //         keypair.public.to_base64(),
+    //     )
+    //     .unwrap();
+    //     std::fs::write(
+    //         format!("test-data/key/{}-secret.asc", name),
+    //         keypair.secret.to_base64(),
+    //     )
+    //     .unwrap();
+    // }
+
     #[test]
     fn test_fingerprint_from_str() {
         let res = Fingerprint::new(vec![

src/keyring.rs

@@ -79,7 +79,8 @@ mod tests {
     #[async_std::test]
     async fn test_keyring_load_self() {
         // new_self() implies load_self()
-        let t = TestContext::new_alice().await;
+        let t = TestContext::new().await;
+        t.configure_alice().await;
         let alice = alice_keypair();
 
         let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t).await.unwrap();

src/lib.rs

@@ -12,8 +12,7 @@
 #![allow(
     clippy::match_bool,
     clippy::eval_order_dependence,
-    clippy::bool_assert_comparison,
-    clippy::manual_split_once
+    clippy::bool_assert_comparison
 )]
 
 #[macro_use]
@@ -56,7 +55,6 @@ mod configure;
 pub mod constants;
 pub mod contact;
 pub mod context;
-pub mod download;
 mod e2ee;
 pub mod ephemeral;
 mod imap;
@@ -69,6 +67,7 @@ pub mod key;
 mod keyring;
 pub mod location;
 mod login_param;
+pub mod lot;
 pub mod message;
 mod mimefactory;
 pub mod mimeparser;
@@ -78,16 +77,13 @@ pub mod peerstate;
 pub mod pgp;
 pub mod provider;
 pub mod qr;
-pub mod qr_code_generator;
 pub mod quota;
 pub mod securejoin;
 mod simplify;
 mod smtp;
 pub mod stock_str;
-mod sync;
 mod token;
 mod update_helper;
-pub mod webxdc;
 #[macro_use]
 mod dehtml;
 mod color;

src/location.rs

@@ -223,11 +223,11 @@ pub async fn send_locations_to_chat(
             .unwrap_or_default();
     } else if 0 == seconds && is_sending_locations_before {
         let stock_str = stock_str::msg_location_disabled(context).await;
-        chat::add_info_msg(context, chat_id, &stock_str, now).await?;
+        chat::add_info_msg(context, chat_id, stock_str, now).await;
     }
     context.emit_event(EventType::ChatModified(chat_id));
     if 0 != seconds {
-        schedule_maybe_send_locations(context, false).await?;
+        schedule_maybe_send_locations(context, false).await;
         job::add(
             context,
             job::Job::new(
@@ -237,20 +237,19 @@ pub async fn send_locations_to_chat(
                 seconds + 1,
             ),
         )
-        .await?;
+        .await;
     }
     Ok(())
 }
 
-async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) -> Result<()> {
-    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await? {
+async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) {
+    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await {
         job::add(
             context,
             job::Job::new(job::Action::MaybeSendLocations, 0, Params::new(), 60),
         )
-        .await?;
+        .await;
     };
-    Ok(())
 }
 
 /// Returns whether `chat_id` or any chat is sending locations.
@@ -325,7 +324,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
         if continue_streaming {
             context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
-        schedule_maybe_send_locations(context, false).await.ok();
+        schedule_maybe_send_locations(context, false).await;
     }
 
     continue_streaming
@@ -552,20 +551,17 @@ pub async fn set_msg_location_id(context: &Context, msg_id: MsgId, location_id:
     Ok(())
 }
 
-/// Saves given locations to the database.
-///
-/// Returns the database row ID of the location with the highest timestamp.
-pub(crate) async fn save(
+pub async fn save(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
     locations: &[Location],
     independent: bool,
-) -> Result<Option<u32>> {
+) -> Result<u32> {
     ensure!(!chat_id.is_special(), "Invalid chat id");
 
     let mut newest_timestamp = 0;
-    let mut newest_location_id = None;
+    let mut newest_location_id = 0;
 
     let stmt_insert = "INSERT INTO locations\
              (timestamp, from_id, chat_id, latitude, longitude, accuracy, independent) \
@@ -603,12 +599,12 @@ pub(crate) async fn save(
                 drop(stmt_test);
                 drop(stmt_insert);
                 newest_timestamp = timestamp;
-                newest_location_id = Some(u32::try_from(conn.last_insert_rowid())?);
+                newest_location_id = conn.last_insert_rowid();
             }
         }
     }
 
-    Ok(newest_location_id)
+    Ok(u32::try_from(newest_location_id)?)
 }
 
 pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> job::Status {
@@ -701,7 +697,7 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
     }
 
     if continue_streaming {
-        job_try!(schedule_maybe_send_locations(context, true).await);
+        schedule_maybe_send_locations(context, true).await;
     }
     job::Status::Finished(Ok(()))
 }
@@ -747,7 +743,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
             );
 
             let stock_str = stock_str::msg_location_disabled(context).await;
-            job_try!(chat::add_info_msg(context, chat_id, &stock_str, now).await);
+            chat::add_info_msg(context, chat_id, stock_str, now).await;
             context.emit_event(EventType::ChatModified(chat_id));
         }
     }

src/log.rs

@@ -1,7 +1,6 @@
 //! # Logging.
 
 use crate::context::Context;
-use async_std::task::block_on;
 
 #[macro_export]
 macro_rules! info {
@@ -40,28 +39,10 @@ macro_rules! error {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        $ctx.set_last_error(&formatted);
         $ctx.emit_event($crate::EventType::Error(formatted));
     }};
 }
 
-impl Context {
-    /// Set last error string.
-    /// Implemented as blocking as used from macros in different, not always async blocks.
-    pub fn set_last_error(&self, error: &str) {
-        block_on(async move {
-            let mut last_error = self.last_error.write().await;
-            *last_error = error.to_string();
-        });
-    }
-
-    /// Get last error string.
-    pub async fn get_last_error(&self) -> String {
-        let last_error = &*self.last_error.read().await;
-        last_error.clone()
-    }
-}
-
 pub trait LogExt<T, E>
 where
     Self: std::marker::Sized,
@@ -153,31 +134,3 @@ impl<T, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
         self
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use crate::test_utils::TestContext;
-    use anyhow::Result;
-
-    #[async_std::test]
-    async fn test_get_last_error() -> Result<()> {
-        let t = TestContext::new().await;
-
-        assert_eq!(t.get_last_error().await, "");
-
-        error!(t, "foo-error");
-        assert_eq!(t.get_last_error().await, "foo-error");
-
-        warn!(t, "foo-warning");
-        assert_eq!(t.get_last_error().await, "foo-error");
-
-        info!(t, "foo-info");
-        assert_eq!(t.get_last_error().await, "foo-error");
-
-        error!(t, "bar-error");
-        error!(t, "baz-error");
-        assert_eq!(t.get_last_error().await, "baz-error");
-
-        Ok(())
-    }
-}

src/login_param.rs

@@ -11,10 +11,8 @@ use anyhow::Result;
 use async_std::io;
 use async_std::net::TcpStream;
 
-use async_native_tls::Certificate;
 pub use async_smtp::ServerAddress;
 use fast_socks5::client::Socks5Stream;
-use once_cell::sync::Lazy;
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(u32)]
@@ -370,18 +368,8 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-// this certificate is missing on older android devices (eg. lg with android6 from 2017)
-// certificate downloaded from https://letsencrypt.org/certificates/
-static LETSENCRYPT_ROOT: Lazy<Certificate> = Lazy::new(|| {
-    Certificate::from_der(include_bytes!(
-        "../assets/root-certificates/letsencrypt/isrgrootx1.der"
-    ))
-    .unwrap()
-});
-
 pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
-    let tls_builder =
-        async_native_tls::TlsConnector::new().add_root_certificate(LETSENCRYPT_ROOT.clone());
+    let tls_builder = async_native_tls::TlsConnector::new();
 
     if strict_tls {
         tls_builder
@@ -413,7 +401,7 @@ mod tests {
         let t = TestContext::new().await;
 
         let param = LoginParam {
-            addr: "alice@example.org".to_string(),
+            addr: "alice@example.com".to_string(),
             imap: ServerLoginParam {
                 server: "imap.example.com".to_string(),
                 user: "alice".to_string(),
@@ -424,7 +412,7 @@ mod tests {
             },
             smtp: ServerLoginParam {
                 server: "smtp.example.com".to_string(),
-                user: "alice@example.org".to_string(),
+                user: "alice@example.com".to_string(),
                 password: "bar".to_string(),
                 port: 456,
                 security: Socket::Ssl,
@@ -442,13 +430,4 @@ mod tests {
         assert_eq!(param, loaded);
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_build_tls() -> Result<()> {
-        // we are using some additional root certificates.
-        // make sure, they do not break construction of TlsConnector
-        let _ = dc_build_tls(true);
-        let _ = dc_build_tls(false);
-        Ok(())
-    }
 }

src/lot.rs

@@ -0,0 +1,180 @@
+//! # Legacy generic return values for C API.
+
+use deltachat_derive::{FromSql, ToSql};
+
+use crate::key::Fingerprint;
+use crate::message::MessageState;
+use crate::summary::{Summary, SummaryPrefix};
+
+/// An object containing a set of values.
+/// The meaning of the values is defined by the function returning the object.
+/// Lot objects are created
+/// eg. by chatlist.get_summary() or dc_msg_get_summary().
+///
+/// *Lot* is used in the meaning *heap* here.
+#[derive(Default, Debug, Clone)]
+pub struct Lot {
+    pub(crate) text1_meaning: Meaning,
+    pub(crate) text1: Option<String>,
+    pub(crate) text2: Option<String>,
+    pub(crate) timestamp: i64,
+    pub(crate) state: LotState,
+    pub(crate) id: u32,
+    pub(crate) fingerprint: Option<Fingerprint>,
+    pub(crate) invitenumber: Option<String>,
+    pub(crate) auth: Option<String>,
+}
+
+#[repr(u8)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
+pub enum Meaning {
+    None = 0,
+    Text1Draft = 1,
+    Text1Username = 2,
+    Text1Self = 3,
+}
+
+impl Default for Meaning {
+    fn default() -> Self {
+        Meaning::None
+    }
+}
+
+impl Lot {
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn get_text1(&self) -> Option<&str> {
+        self.text1.as_deref()
+    }
+
+    pub fn get_text2(&self) -> Option<&str> {
+        self.text2.as_deref()
+    }
+
+    pub fn get_text1_meaning(&self) -> Meaning {
+        self.text1_meaning
+    }
+
+    pub fn get_state(&self) -> LotState {
+        self.state
+    }
+
+    pub fn get_id(&self) -> u32 {
+        self.id
+    }
+
+    pub fn get_timestamp(&self) -> i64 {
+        self.timestamp
+    }
+}
+
+#[repr(u32)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
+pub enum LotState {
+    // Default
+    Undefined = 0,
+
+    // Qr States
+    /// id=contact
+    QrAskVerifyContact = 200,
+
+    /// text1=groupname
+    QrAskVerifyGroup = 202,
+
+    /// id=contact
+    QrFprOk = 210,
+
+    /// id=contact
+    QrFprMismatch = 220,
+
+    /// test1=formatted fingerprint
+    QrFprWithoutAddr = 230,
+
+    /// text1=domain
+    QrAccount = 250,
+
+    /// text1=domain, text2=instance pattern
+    QrWebrtcInstance = 260,
+
+    /// id=contact
+    QrAddr = 320,
+
+    /// text1=text
+    QrText = 330,
+
+    /// text1=URL
+    QrUrl = 332,
+
+    /// text1=error string
+    QrError = 400,
+
+    QrWithdrawVerifyContact = 500,
+
+    /// text1=groupname
+    QrWithdrawVerifyGroup = 502,
+
+    QrReviveVerifyContact = 510,
+
+    /// text1=groupname
+    QrReviveVerifyGroup = 512,
+
+    // Message States
+    MsgInFresh = 10,
+    MsgInNoticed = 13,
+    MsgInSeen = 16,
+    MsgOutPreparing = 18,
+    MsgOutDraft = 19,
+    MsgOutPending = 20,
+    MsgOutFailed = 24,
+    MsgOutDelivered = 26,
+    MsgOutMdnRcvd = 28,
+}
+
+impl Default for LotState {
+    fn default() -> Self {
+        LotState::Undefined
+    }
+}
+
+impl From<MessageState> for LotState {
+    fn from(s: MessageState) -> Self {
+        use MessageState::*;
+        match s {
+            Undefined => LotState::Undefined,
+            InFresh => LotState::MsgInFresh,
+            InNoticed => LotState::MsgInNoticed,
+            InSeen => LotState::MsgInSeen,
+            OutPreparing => LotState::MsgOutPreparing,
+            OutDraft => LotState::MsgOutDraft,
+            OutPending => LotState::MsgOutPending,
+            OutFailed => LotState::MsgOutFailed,
+            OutDelivered => LotState::MsgOutDelivered,
+            OutMdnRcvd => LotState::MsgOutMdnRcvd,
+        }
+    }
+}
+
+impl From<Summary> for Lot {
+    fn from(summary: Summary) -> Self {
+        let (text1, text1_meaning) = match summary.prefix {
+            None => (None, Meaning::None),
+            Some(SummaryPrefix::Draft(text)) => (Some(text), Meaning::Text1Draft),
+            Some(SummaryPrefix::Username(username)) => (Some(username), Meaning::Text1Username),
+            Some(SummaryPrefix::Me(text)) => (Some(text), Meaning::Text1Self),
+        };
+        Self {
+            text1_meaning,
+            text1,
+            text2: Some(summary.text),
+            timestamp: summary.timestamp,
+            state: summary.state.into(),
+            ..Default::default()
+        }
+    }
+}

src/message.rs

@@ -1,6 +1,6 @@
 //! # Messages and their identifiers.
 
-use std::collections::BTreeSet;
+use std::collections::BTreeMap;
 use std::convert::TryInto;
 
 use anyhow::{ensure, format_err, Context as _, Result};
@@ -10,6 +10,7 @@ use rusqlite::types::ValueRef;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
+use crate::config::Config;
 use crate::constants::{
     Blocked, Chattype, VideochatType, Viewtype, DC_CHAT_ID_TRASH, DC_CONTACT_ID_INFO,
     DC_CONTACT_ID_SELF, DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
@@ -20,7 +21,6 @@ use crate::dc_tools::{
     dc_create_smeared_timestamp, dc_get_filebytes, dc_get_filemeta, dc_gm2local_offset,
     dc_read_file, dc_timestamp_to_str, dc_truncate, time,
 };
-use crate::download::DownloadState;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::events::EventType;
 use crate::job::{self, Action};
@@ -28,9 +28,8 @@ use crate::log::LogExt;
 use crate::mimeparser::{parse_message_id, FailureReport, SystemMessage};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
-use crate::scheduler::InterruptInfo;
 use crate::stock_str;
-use crate::summary::Summary;
+use crate::summary::{get_summarytext_by_raw, Summary};
 
 /// Message ID, including reserved IDs.
 ///
@@ -83,6 +82,65 @@ impl MsgId {
         Ok(result)
     }
 
+    /// Returns Some if the message needs to be moved from `folder`.
+    /// If yes, returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder`,
+    /// depending on where the message should be moved
+    pub async fn needs_move(self, context: &Context, folder: &str) -> Result<Option<Config>> {
+        use Config::*;
+        if context.is_mvbox(folder).await? {
+            return Ok(None);
+        }
+
+        let msg = Message::load_from_db(context, self).await?;
+
+        if context.is_spam_folder(folder).await? {
+            let msg_unblocked = msg.chat_id != DC_CHAT_ID_TRASH && msg.chat_blocked == Blocked::Not;
+
+            return if msg_unblocked {
+                if self.needs_move_to_mvbox(context, &msg).await? {
+                    Ok(Some(ConfiguredMvboxFolder))
+                } else {
+                    Ok(Some(ConfiguredInboxFolder))
+                }
+            } else {
+                // Blocked or contact request message in the spam folder, leave it there
+                Ok(None)
+            };
+        }
+
+        if self.needs_move_to_mvbox(context, &msg).await? {
+            Ok(Some(ConfiguredMvboxFolder))
+        } else if msg.state.is_outgoing()
+                && msg.is_dc_message == MessengerMessage::Yes
+                && !msg.is_setupmessage()
+                && msg.to_id != DC_CONTACT_ID_SELF // Leave self-chat-messages in the inbox, not sure about this
+                && context.is_inbox(folder).await?
+                && context.get_config_bool(SentboxMove).await?
+                && context.get_config(ConfiguredSentboxFolder).await?.is_some()
+        {
+            Ok(Some(ConfiguredSentboxFolder))
+        } else {
+            Ok(None)
+        }
+    }
+
+    async fn needs_move_to_mvbox(self, context: &Context, msg: &Message) -> Result<bool> {
+        if !context.get_config_bool(Config::MvboxMove).await? {
+            return Ok(false);
+        }
+
+        if msg.is_setupmessage() {
+            // do not move setup messages;
+            // there may be a non-delta device that wants to handle it
+            return Ok(false);
+        }
+
+        match msg.is_dc_message {
+            MessengerMessage::No => Ok(false),
+            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
+        }
+    }
+
     /// Put message into trash chat and delete message text.
     ///
     /// It means the message is deleted locally, but not on the server.
@@ -113,25 +171,14 @@ WHERE id=?;
         Ok(())
     }
 
-    /// Deletes a message, corresponding MDNs and unsent SMTP messages from the database.
+    /// Deletes a message and corresponding MDNs from the database.
     pub async fn delete_from_db(self, context: &Context) -> Result<()> {
         // We don't use transactions yet, so remove MDNs first to make
         // sure they are not left while the message is deleted.
-        context
-            .sql
-            .execute("DELETE FROM smtp WHERE msg_id=?", paramsv![self])
-            .await?;
         context
             .sql
             .execute("DELETE FROM msgs_mdns WHERE msg_id=?;", paramsv![self])
             .await?;
-        context
-            .sql
-            .execute(
-                "DELETE FROM msgs_status_updates WHERE msg_id=?;",
-                paramsv![self],
-            )
-            .await?;
         context
             .sql
             .execute("DELETE FROM msgs WHERE id=?;", paramsv![self])
@@ -139,17 +186,21 @@ WHERE id=?;
         Ok(())
     }
 
-    pub(crate) async fn set_delivered(self, context: &Context) -> Result<()> {
-        update_msg_state(context, self, MessageState::OutDelivered).await?;
-        let chat_id: ChatId = context
+    /// Removes IMAP server UID and folder from the database record.
+    ///
+    /// It is used to avoid trying to remove the message from the
+    /// server multiple times when there are multiple message records
+    /// pointing to the same server UID.
+    pub(crate) async fn unlink(self, context: &Context) -> Result<()> {
+        context
             .sql
-            .query_get_value("SELECT chat_id FROM msgs WHERE id=?", paramsv![self])
-            .await?
-            .unwrap_or_default();
-        context.emit_event(EventType::MsgDelivered {
-            chat_id,
-            msg_id: self,
-        });
+            .execute(
+                "UPDATE msgs \
+             SET server_folder='', server_uid=0 \
+             WHERE id=?",
+                paramsv![self],
+            )
+            .await?;
         Ok(())
     }
 
@@ -180,7 +231,7 @@ impl rusqlite::types::ToSql for MsgId {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
         if self.0 <= DC_MSG_ID_LAST_SPECIAL {
             return Err(rusqlite::Error::ToSqlConversionFailure(
-                format_err!("Invalid MsgId {}", self.0).into(),
+                format_err!("Invalid MsgId").into(),
             ));
         }
         let val = rusqlite::types::Value::Integer(self.0 as i64);
@@ -245,7 +296,6 @@ pub struct Message {
     pub(crate) chat_id: ChatId,
     pub(crate) viewtype: Viewtype,
     pub(crate) state: MessageState,
-    pub(crate) download_state: DownloadState,
     pub(crate) hidden: bool,
     pub(crate) timestamp_sort: i64,
     pub(crate) timestamp_sent: i64,
@@ -256,6 +306,8 @@ pub struct Message {
     pub(crate) subject: String,
     pub(crate) rfc724_mid: String,
     pub(crate) in_reply_to: Option<String>,
+    pub(crate) server_folder: Option<String>,
+    pub(crate) server_uid: u32,
     pub(crate) is_dc_message: MessengerMessage,
     pub(crate) mime_modified: bool,
     pub(crate) chat_blocked: Blocked,
@@ -275,7 +327,7 @@ impl Message {
     pub async fn load_from_db(context: &Context, id: MsgId) -> Result<Message> {
         ensure!(
             !id.is_special(),
-            "Can not load special message ID {} from DB",
+            "Can not load special message ID {} from DB.",
             id
         );
         let msg = context
@@ -286,6 +338,8 @@ impl Message {
                     "    m.id AS id,",
                     "    rfc724_mid AS rfc724mid,",
                     "    m.mime_in_reply_to AS mime_in_reply_to,",
+                    "    m.server_folder AS server_folder,",
+                    "    m.server_uid AS server_uid,",
                     "    m.chat_id AS chat_id,",
                     "    m.from_id AS from_id,",
                     "    m.to_id AS to_id,",
@@ -296,7 +350,6 @@ impl Message {
                     "    m.ephemeral_timestamp AS ephemeral_timestamp,",
                     "    m.type AS type,",
                     "    m.state AS state,",
-                    "    m.download_state AS download_state,",
                     "    m.error AS error,",
                     "    m.msgrmsg AS msgrmsg,",
                     "    m.mime_modified AS mime_modified,",
@@ -336,6 +389,8 @@ impl Message {
                         in_reply_to: row
                             .get::<_, Option<String>>("mime_in_reply_to")?
                             .and_then(|in_reply_to| parse_message_id(&in_reply_to).ok()),
+                        server_folder: row.get::<_, Option<String>>("server_folder")?,
+                        server_uid: row.get("server_uid")?,
                         chat_id: row.get("chat_id")?,
                         from_id: row.get("from_id")?,
                         to_id: row.get("to_id")?,
@@ -346,7 +401,6 @@ impl Message {
                         ephemeral_timestamp: row.get("ephemeral_timestamp")?,
                         viewtype: row.get("type")?,
                         state: row.get("state")?,
-                        download_state: row.get("download_state")?,
                         error: Some(row.get::<_, String>("error")?)
                             .filter(|error| !error.is_empty()),
                         is_dc_message: row.get("msgrmsg")?,
@@ -534,7 +588,7 @@ impl Message {
     }
 
     /// Returns message summary for display in the search results.
-    pub async fn get_summary(&self, context: &Context, chat: Option<&Chat>) -> Result<Summary> {
+    pub async fn get_summary(&mut self, context: &Context, chat: Option<&Chat>) -> Result<Summary> {
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
@@ -546,7 +600,7 @@ impl Message {
 
         let contact = if self.from_id != DC_CONTACT_ID_SELF {
             match chat.typ {
-                Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
+                Chattype::Group | Chattype::Mailinglist => {
                     Some(Contact::get_by_id(context, self.from_id).await?)
                 }
                 Chattype::Single | Chattype::Undefined => None,
@@ -558,6 +612,18 @@ impl Message {
         Ok(Summary::new(context, self, chat, contact.as_ref()).await)
     }
 
+    pub async fn get_summarytext(&self, context: &Context, approx_characters: usize) -> String {
+        get_summarytext_by_raw(
+            self.viewtype,
+            self.text.as_ref(),
+            self.is_forwarded(),
+            &self.param,
+            approx_characters,
+            context,
+        )
+        .await
+    }
+
     // It's a little unfortunate that the UI has to first call dc_msg_get_override_sender_name() and then if it was NULL, call
     // dc_contact_get_display_name() but this was the best solution:
     // - We could load a Contact struct from the db here to call get_display_name() instead of returning None, but then we had a db
@@ -781,41 +847,32 @@ impl Message {
     ///
     /// The message itself is not required to exist in the database,
     /// it may even be deleted from the database by the time the message is prepared.
-    pub async fn set_quote(&mut self, context: &Context, quote: Option<&Message>) -> Result<()> {
-        if let Some(quote) = quote {
-            ensure!(
-                !quote.rfc724_mid.is_empty(),
-                "Message without Message-Id cannot be quoted"
-            );
-            self.in_reply_to = Some(quote.rfc724_mid.clone());
+    pub async fn set_quote(&mut self, context: &Context, quote: &Message) -> Result<()> {
+        ensure!(
+            !quote.rfc724_mid.is_empty(),
+            "Message without Message-Id cannot be quoted"
+        );
+        self.in_reply_to = Some(quote.rfc724_mid.clone());
 
-            if quote
-                .param
-                .get_bool(Param::GuaranteeE2ee)
-                .unwrap_or_default()
-            {
-                self.param.set(Param::GuaranteeE2ee, "1");
-            }
-
-            let text = quote.get_text().unwrap_or_default();
-            self.param.set(
-                Param::Quote,
-                if text.is_empty() {
-                    // Use summary, similar to "Image" to avoid sending empty quote.
-                    quote
-                        .get_summary(context, None)
-                        .await?
-                        .truncated_text(500)
-                        .to_string()
-                } else {
-                    text
-                },
-            );
-        } else {
-            self.in_reply_to = None;
-            self.param.remove(Param::Quote);
+        if quote
+            .param
+            .get_bool(Param::GuaranteeE2ee)
+            .unwrap_or_default()
+        {
+            self.param.set(Param::GuaranteeE2ee, "1");
         }
 
+        let text = quote.get_text().unwrap_or_default();
+        self.param.set(
+            Param::Quote,
+            if text.is_empty() {
+                // Use summary, similar to "Image" to avoid sending empty quote.
+                quote.get_summarytext(context, 500).await
+            } else {
+                text
+            },
+        );
+
         Ok(())
     }
 
@@ -824,32 +881,22 @@ impl Message {
     }
 
     pub async fn quoted_message(&self, context: &Context) -> Result<Option<Message>> {
-        if self.param.get(Param::Quote).is_some() && !self.is_forwarded() {
-            return self.parent(context).await;
-        }
-        Ok(None)
-    }
-
-    pub async fn parent(&self, context: &Context) -> Result<Option<Message>> {
-        if let Some(in_reply_to) = &self.in_reply_to {
-            if let Some(msg_id) = rfc724_mid_exists(context, in_reply_to).await? {
-                let msg = Message::load_from_db(context, msg_id).await?;
-                return if msg.chat_id.is_trash() {
-                    // If message is already moved to trash chat, pretend it does not exist.
-                    Ok(None)
-                } else {
-                    Ok(Some(msg))
-                };
+        if self.param.get(Param::Quote).is_some() {
+            if let Some(in_reply_to) = &self.in_reply_to {
+                if let Some((_, _, msg_id)) = rfc724_mid_exists(context, in_reply_to).await? {
+                    let msg = Message::load_from_db(context, msg_id).await?;
+                    return if msg.chat_id.is_trash() {
+                        // If message is already moved to trash chat, pretend it does not exist.
+                        Ok(None)
+                    } else {
+                        Ok(Some(msg))
+                    };
+                }
             }
         }
         Ok(None)
     }
 
-    /// Force the message to be sent in plain text.
-    pub fn force_plaintext(&mut self) {
-        self.param.set_int(Param::ForcePlaintext, 1);
-    }
-
     pub async fn update_param(&self, context: &Context) {
         context
             .sql
@@ -1119,13 +1166,11 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
     if !msg.rfc724_mid.is_empty() {
         ret += &format!("\nMessage-ID: {}", msg.rfc724_mid);
     }
-    let hop_info: Option<String> = context
-        .sql
-        .query_get_value("SELECT hop_info FROM msgs WHERE id=?;", paramsv![msg_id])
-        .await?;
-
-    ret += "\n\n";
-    ret += &hop_info.unwrap_or_else(|| "No Hop Info".to_owned());
+    if let Some(ref server_folder) = msg.server_folder {
+        if !server_folder.is_empty() {
+            ret += &format!("\nLast seen as: {}/{}", server_folder, msg.server_uid);
+        }
+    }
 
     Ok(ret)
 }
@@ -1191,7 +1236,6 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
         "webm" => (Viewtype::Video, "video/webm"),
         "webp" => (Viewtype::Image, "image/webp"), // iOS via SDWebImage, Android since 4.0
         "wmv" => (Viewtype::Video, "video/x-ms-wmv"),
-        "xdc" => (Viewtype::Webxdc, "application/webxdc+zip"),
         "xhtml" => (Viewtype::File, "application/xhtml+xml"),
         "xlsx" => (
             Viewtype::File,
@@ -1243,13 +1287,11 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
             .trash(context)
             .await
             .with_context(|| format!("Unable to trash message {}", msg_id))?;
-        context
-            .sql
-            .execute(
-                "UPDATE imap SET target='' WHERE rfc724_mid=?",
-                paramsv![msg.rfc724_mid],
-            )
-            .await?;
+        job::add(
+            context,
+            job::Job::new(Action::DeleteMsgOnImap, msg_id.to_u32(), Params::new(), 0),
+        )
+        .await;
     }
 
     if !msg_ids.is_empty() {
@@ -1262,11 +1304,8 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
             context,
             job::Job::new(Action::Housekeeping, 0, Params::new(), 10),
         )
-        .await?;
+        .await;
     }
-
-    // Interrupt Inbox loop to start message deletion.
-    context.interrupt_inbox(InterruptInfo::new(false)).await;
     Ok(())
 }
 
@@ -1319,7 +1358,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
     })
     .await?;
 
-    let mut updated_chat_ids = BTreeSet::new();
+    let mut updated_chat_ids = BTreeMap::new();
 
     for (id, curr_chat_id, curr_state, curr_blocked) in msgs.into_iter() {
         if let Err(err) = id.start_ephemeral_timer(context).await {
@@ -1333,38 +1372,34 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
         if curr_blocked == Blocked::Not
             && (curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed)
         {
-            update_msg_state(context, id, MessageState::InSeen).await?;
+            update_msg_state(context, id, MessageState::InSeen).await;
             info!(context, "Seen message {}.", id);
 
             job::add(
                 context,
                 job::Job::new(Action::MarkseenMsgOnImap, id.to_u32(), Params::new(), 0),
             )
-            .await?;
-            updated_chat_ids.insert(curr_chat_id);
+            .await;
+            updated_chat_ids.insert(curr_chat_id, true);
         }
     }
 
-    for updated_chat_id in updated_chat_ids {
-        context.emit_event(EventType::MsgsNoticed(updated_chat_id));
+    for updated_chat_id in updated_chat_ids.keys() {
+        context.emit_event(EventType::MsgsNoticed(*updated_chat_id));
     }
 
     Ok(())
 }
 
-pub(crate) async fn update_msg_state(
-    context: &Context,
-    msg_id: MsgId,
-    state: MessageState,
-) -> Result<()> {
+pub async fn update_msg_state(context: &Context, msg_id: MsgId, state: MessageState) -> bool {
     context
         .sql
         .execute(
             "UPDATE msgs SET state=? WHERE id=?;",
             paramsv![state, msg_id],
         )
-        .await?;
-    Ok(())
+        .await
+        .is_ok()
 }
 
 // as we do not cut inside words, this results in about 32-42 characters.
@@ -1448,6 +1483,7 @@ pub async fn handle_mdn(
                 "SELECT",
                 "    m.id AS msg_id,",
                 "    c.id AS chat_id,",
+                "    c.type AS type,",
                 "    m.state AS state",
                 " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
                 " WHERE rfc724_mid=? AND from_id=1",
@@ -1458,13 +1494,14 @@ pub async fn handle_mdn(
                 Ok((
                     row.get::<_, MsgId>("msg_id")?,
                     row.get::<_, ChatId>("chat_id")?,
+                    row.get::<_, Chattype>("type")?,
                     row.get::<_, MessageState>("state")?,
                 ))
             },
         )
         .await?;
 
-    let (msg_id, chat_id, msg_state) = if let Some(res) = res {
+    let (msg_id, chat_id, chat_type, msg_state) = if let Some(res) = res {
         res
     } else {
         info!(
@@ -1475,28 +1512,63 @@ pub async fn handle_mdn(
         return Ok(None);
     };
 
-    if !context
-        .sql
-        .exists(
-            "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=? AND contact_id=?;",
-            paramsv![msg_id, from_id as i32,],
-        )
-        .await?
-    {
-        context
-            .sql
-            .execute(
-                "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?);",
-                paramsv![msg_id, from_id as i32, timestamp_sent],
-            )
-            .await?;
-    }
-
+    let mut read_by_all = false;
     if msg_state == MessageState::OutPreparing
         || msg_state == MessageState::OutPending
         || msg_state == MessageState::OutDelivered
     {
-        update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await?;
+        let mdn_already_in_table = context
+            .sql
+            .exists(
+                "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=? AND contact_id=?;",
+                paramsv![msg_id, from_id as i32,],
+            )
+            .await?;
+
+        if !mdn_already_in_table {
+            context
+                .sql
+                .execute(
+                    "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?);",
+                    paramsv![msg_id, from_id as i32, timestamp_sent],
+                )
+                .await?;
+        }
+
+        // Normal chat? that's quite easy.
+        if chat_type == Chattype::Single {
+            update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await;
+            read_by_all = true;
+        } else {
+            // send event about new state
+            let ist_cnt = context
+                .sql
+                .count(
+                    "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=?;",
+                    paramsv![msg_id],
+                )
+                .await?;
+
+            // Groupsize:  Min. MDNs
+            // 1 S         n/a
+            // 2 SR        1
+            // 3 SRR       2
+            // 4 SRRR      2
+            // 5 SRRRR     3
+            // 6 SRRRRR    3
+            //
+            // (S=Sender, R=Recipient)
+
+            // for rounding, SELF is already included!
+            let soll_cnt = (chat::get_chat_contact_cnt(context, chat_id).await? + 1) / 2;
+            if ist_cnt >= soll_cnt {
+                update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await;
+                read_by_all = true;
+            } // else wait for more receipts
+        }
+    }
+
+    if read_by_all {
         Ok(Some((chat_id, msg_id)))
     } else {
         Ok(None)
@@ -1560,12 +1632,14 @@ async fn ndn_maybe_add_info_msg(
     chat_type: Chattype,
 ) -> Result<()> {
     match chat_type {
-        Chattype::Group | Chattype::Broadcast => {
+        Chattype::Group => {
             if let Some(failed_recipient) = &failed.failed_recipient {
                 let contact_id =
                     Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown)
                         .await?
-                        .context("contact ID not found")?;
+                        .ok_or_else(|| {
+                            format_err!("ndn_maybe_add_info_msg: Contact ID not found")
+                        })?;
                 let contact = Contact::load_from_db(context, contact_id).await?;
                 // Tell the user which of the recipients failed if we know that (because in
                 // a group, this might otherwise be unclear)
@@ -1573,10 +1647,10 @@ async fn ndn_maybe_add_info_msg(
                 chat::add_info_msg(
                     context,
                     chat_id,
-                    &text,
+                    text,
                     dc_create_smeared_timestamp(context).await,
                 )
-                .await?;
+                .await;
                 context.emit_event(EventType::ChatModified(chat_id));
             }
         }
@@ -1649,7 +1723,7 @@ pub async fn estimate_deletion_cnt(
              WHERE m.id > ?
                AND timestamp < ?
                AND chat_id != ?
-               AND EXISTS (SELECT * FROM imap WHERE rfc724_mid=m.rfc724_mid);",
+               AND server_uid != 0;",
                 paramsv![DC_MSG_ID_LAST_SPECIAL, threshold_timestamp, self_chat_id],
             )
             .await?
@@ -1675,10 +1749,32 @@ pub async fn estimate_deletion_cnt(
     Ok(cnt)
 }
 
+/// Counts number of database records pointing to specified
+/// Message-ID.
+///
+/// Unlinked messages are excluded.
+pub async fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> usize {
+    // check the number of messages with the same rfc724_mid
+    match context
+        .sql
+        .count(
+            "SELECT COUNT(*) FROM msgs WHERE rfc724_mid=? AND NOT server_uid = 0",
+            paramsv![rfc724_mid],
+        )
+        .await
+    {
+        Ok(res) => res,
+        Err(err) => {
+            error!(context, "dc_get_rfc724_mid_cnt() failed. {}", err);
+            0
+        }
+    }
+}
+
 pub(crate) async fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
-) -> Result<Option<MsgId>> {
+) -> Result<Option<(String, u32, MsgId)>> {
     let rfc724_mid = rfc724_mid.trim_start_matches('<').trim_end_matches('>');
     if rfc724_mid.is_empty() {
         warn!(context, "Empty rfc724_mid passed to rfc724_mid_exists");
@@ -1688,12 +1784,14 @@ pub(crate) async fn rfc724_mid_exists(
     let res = context
         .sql
         .query_row_optional(
-            "SELECT id FROM msgs WHERE rfc724_mid=?",
+            "SELECT server_folder, server_uid, id FROM msgs WHERE rfc724_mid=?",
             paramsv![rfc724_mid],
             |row| {
-                let msg_id: MsgId = row.get(0)?;
+                let server_folder = row.get::<_, Option<String>>(0)?.unwrap_or_default();
+                let server_uid = row.get(1)?;
+                let msg_id: MsgId = row.get(2)?;
 
-                Ok(msg_id)
+                Ok((server_folder, server_uid, msg_id))
             },
         )
         .await?;
@@ -1701,6 +1799,28 @@ pub(crate) async fn rfc724_mid_exists(
     Ok(res)
 }
 
+pub async fn update_server_uid(
+    context: &Context,
+    rfc724_mid: &str,
+    server_folder: &str,
+    server_uid: u32,
+) {
+    match context
+        .sql
+        .execute(
+            "UPDATE msgs SET server_folder=?, server_uid=? \
+             WHERE rfc724_mid=?",
+            paramsv![server_folder, server_uid, rfc724_mid],
+        )
+        .await
+    {
+        Ok(_) => {}
+        Err(err) => {
+            warn!(context, "msg: failed to update server_uid: {}", err);
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1717,14 +1837,212 @@ mod tests {
             guess_msgtype_from_suffix(Path::new("foo/bar-sth.mp3")),
             Some((Viewtype::Audio, "audio/mpeg"))
         );
-        assert_eq!(
-            guess_msgtype_from_suffix(Path::new("foo/file.html")),
-            Some((Viewtype::File, "text/html"))
-        );
-        assert_eq!(
-            guess_msgtype_from_suffix(Path::new("foo/file.xdc")),
-            Some((Viewtype::Webxdc, "application/webxdc+zip"))
-        );
+    }
+
+    // chat_msg means that the message was sent by Delta Chat
+    // The tuples are (folder, mvbox_move, chat_msg, expected_destination)
+    const COMBINATIONS_ACCEPTED_CHAT: &[(&str, bool, bool, &str)] = &[
+        ("INBOX", false, false, "INBOX"),
+        ("INBOX", false, true, "INBOX"),
+        ("INBOX", true, false, "INBOX"),
+        ("INBOX", true, true, "DeltaChat"),
+        ("Sent", false, false, "Sent"),
+        ("Sent", false, true, "Sent"),
+        ("Sent", true, false, "Sent"),
+        ("Sent", true, true, "DeltaChat"),
+        ("Spam", false, false, "INBOX"), // Move classical emails in accepted chats from Spam to Inbox, not 100% sure on this, we could also just never move non-chat-msgs
+        ("Spam", false, true, "INBOX"),
+        ("Spam", true, false, "INBOX"), // Move classical emails in accepted chats from Spam to Inbox, not 100% sure on this, we could also just never move non-chat-msgs
+        ("Spam", true, true, "DeltaChat"),
+    ];
+
+    // These are the same as above, but all messages in Spam stay in Spam
+    const COMBINATIONS_REQUEST: &[(&str, bool, bool, &str)] = &[
+        ("INBOX", false, false, "INBOX"),
+        ("INBOX", false, true, "INBOX"),
+        ("INBOX", true, false, "INBOX"),
+        ("INBOX", true, true, "DeltaChat"),
+        ("Sent", false, false, "Sent"),
+        ("Sent", false, true, "Sent"),
+        ("Sent", true, false, "Sent"),
+        ("Sent", true, true, "DeltaChat"),
+        ("Spam", false, false, "Spam"),
+        ("Spam", false, true, "Spam"),
+        ("Spam", true, false, "Spam"),
+        ("Spam", true, true, "Spam"),
+    ];
+
+    #[async_std::test]
+    async fn test_needs_move_incoming_accepted() {
+        for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
+            check_needs_move_combination(
+                folder,
+                *mvbox_move,
+                *chat_msg,
+                expected_destination,
+                true,
+                false,
+                false,
+                false,
+            )
+            .await;
+        }
+    }
+
+    #[async_std::test]
+    async fn test_needs_move_incoming_request() {
+        for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_REQUEST {
+            check_needs_move_combination(
+                folder,
+                *mvbox_move,
+                *chat_msg,
+                expected_destination,
+                false,
+                false,
+                false,
+                false,
+            )
+            .await;
+        }
+    }
+
+    #[async_std::test]
+    async fn test_needs_move_outgoing() {
+        for sentbox_move in &[true, false] {
+            // Test outgoing emails
+            for (folder, mvbox_move, chat_msg, mut expected_destination) in
+                COMBINATIONS_ACCEPTED_CHAT
+            {
+                if *folder == "INBOX" && !mvbox_move && *chat_msg && *sentbox_move {
+                    expected_destination = "Sent"
+                }
+                check_needs_move_combination(
+                    folder,
+                    *mvbox_move,
+                    *chat_msg,
+                    expected_destination,
+                    true,
+                    true,
+                    false,
+                    *sentbox_move,
+                )
+                .await;
+            }
+        }
+    }
+
+    #[async_std::test]
+    async fn test_needs_move_setupmsg() {
+        // Test setupmessages
+        for (folder, mvbox_move, chat_msg, _expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
+            check_needs_move_combination(
+                folder,
+                *mvbox_move,
+                *chat_msg,
+                if folder == &"Spam" { "INBOX" } else { folder }, // Never move setup messages, except if they are in "Spam"
+                false,
+                true,
+                true,
+                false,
+            )
+            .await;
+        }
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    async fn check_needs_move_combination(
+        folder: &str,
+        mvbox_move: bool,
+        chat_msg: bool,
+        expected_destination: &str,
+        accepted_chat: bool,
+        outgoing: bool,
+        setupmessage: bool,
+        sentbox_move: bool,
+    ) {
+        println!("Testing: For folder {}, mvbox_move {}, chat_msg {}, accepted {}, outgoing {}, setupmessage {}",
+                               folder, mvbox_move, chat_msg, accepted_chat, outgoing, setupmessage);
+
+        let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::ConfiguredSpamFolder, Some("Spam"))
+            .await
+            .unwrap();
+        t.ctx
+            .set_config(Config::ConfiguredMvboxFolder, Some("DeltaChat"))
+            .await
+            .unwrap();
+        t.ctx
+            .set_config(Config::ConfiguredSentboxFolder, Some("Sent"))
+            .await
+            .unwrap();
+        t.ctx
+            .set_config(Config::MvboxMove, Some(if mvbox_move { "1" } else { "0" }))
+            .await
+            .unwrap();
+        t.ctx
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+        t.ctx
+            .set_config_bool(Config::SentboxMove, sentbox_move)
+            .await
+            .unwrap();
+
+        if accepted_chat {
+            let contact_id = Contact::create(&t.ctx, "", "bob@example.net")
+                .await
+                .unwrap();
+            ChatId::create_for_contact(&t.ctx, contact_id)
+                .await
+                .unwrap();
+        }
+        let temp;
+        dc_receive_imf(
+            &t.ctx,
+            if setupmessage {
+                include_bytes!("../test-data/message/AutocryptSetupMessage.eml")
+            } else {
+                temp = format!(
+                    "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                    {}\
+                    Subject: foo\n\
+                    Message-ID: <abc@example.com>\n\
+                    {}\
+                    Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                    \n\
+                    hello\n",
+                    if outgoing {
+                        "From: alice@example.com\nTo: bob@example.net\n"
+                    } else {
+                        "From: bob@example.net\nTo: alice@example.com\n"
+                    },
+                    if chat_msg { "Chat-Version: 1.0\n" } else { "" },
+                );
+                temp.as_bytes()
+            },
+            folder,
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let exists = rfc724_mid_exists(&t, "abc@example.com").await.unwrap();
+        let (folder_1, _, msg_id) = exists.unwrap();
+        assert_eq!(folder, folder_1);
+        let actual = if let Some(config) = msg_id.needs_move(&t.ctx, folder).await.unwrap() {
+            t.ctx.get_config(config).await.unwrap()
+        } else {
+            None
+        };
+        let expected = if expected_destination == folder {
+            None
+        } else {
+            Some(expected_destination)
+        };
+        assert_eq!(expected, actual.as_deref(), "For folder {}, mvbox_move {}, chat_msg {}, accepted {}, outgoing {}, setupmessage {}: expected {:?}, got {:?}",
+                                                     folder, mvbox_move, chat_msg, accepted_chat, outgoing, setupmessage, expected, actual);
     }
 
     #[async_std::test]
@@ -1892,9 +2210,7 @@ mod tests {
         assert!(!msg.rfc724_mid.is_empty());
 
         let mut msg2 = Message::new(Viewtype::Text);
-        msg2.set_quote(ctx, Some(&msg))
-            .await
-            .expect("can't set quote");
+        msg2.set_quote(ctx, &msg).await.expect("can't set quote");
         assert!(msg2.quoted_text() == msg.get_text());
 
         let quoted_msg = msg2
@@ -1912,13 +2228,14 @@ mod tests {
         dc_receive_imf(
             &alice,
             b"From: Bob <bob@example.com>\n\
-                    To: alice@example.org\n\
+                    To: alice@example.com\n\
                     Chat-Version: 1.0\n\
                     Message-ID: <123@example.com>\n\
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello\n",
             "INBOX",
+            123,
             false,
         )
         .await
@@ -2025,9 +2342,9 @@ mod tests {
         let msg2 = alice.get_last_msg().await;
         let chats = Chatlist::try_load(&alice, 0, None, None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0)?, alice_chat.id);
-        assert_eq!(chats.get_chat_id(0)?, msg1.chat_id);
-        assert_eq!(chats.get_chat_id(0)?, msg2.chat_id);
+        assert_eq!(chats.get_chat_id(0), alice_chat.id);
+        assert_eq!(chats.get_chat_id(0), msg1.chat_id);
+        assert_eq!(chats.get_chat_id(0), msg2.chat_id);
         assert_eq!(alice_chat.id.get_fresh_msg_cnt(&alice).await?, 2);
         assert_eq!(alice.get_fresh_msgs().await?.len(), 2);
 
@@ -2091,7 +2408,7 @@ mod tests {
         let payload = alice.pop_sent_msg().await;
         assert_state(&alice, alice_msg.id, MessageState::OutDelivered).await;
 
-        update_msg_state(&alice, alice_msg.id, MessageState::OutMdnRcvd).await?;
+        update_msg_state(&alice, alice_msg.id, MessageState::OutMdnRcvd).await;
         assert_state(&alice, alice_msg.id, MessageState::OutMdnRcvd).await;
 
         set_msg_failed(&alice, alice_msg.id, Some("badly failed")).await;
@@ -2120,7 +2437,7 @@ mod tests {
         dc_receive_imf(
             &alice,
             b"From: Bob <bob@example.com>\n\
-                    To: alice@example.org\n\
+                    To: alice@example.com\n\
                     Chat-Version: 1.0\n\
                     Message-ID: <123@example.com>\n\
                     Auto-Submitted: auto-generated\n\
@@ -2128,6 +2445,7 @@ mod tests {
                     \n\
                     hello\n",
             "INBOX",
+            1,
             false,
         )
         .await?;
@@ -2139,13 +2457,14 @@ mod tests {
         dc_receive_imf(
             &alice,
             b"From: Bob <bob@example.com>\n\
-                    To: alice@example.org\n\
+                    To: alice@example.com\n\
                     Chat-Version: 1.0\n\
                     Message-ID: <456@example.com>\n\
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello again\n",
             "INBOX",
+            2,
             false,
         )
         .await?;

src/mimefactory.rs

@@ -2,7 +2,7 @@
 
 use std::convert::TryInto;
 
-use anyhow::{bail, ensure, Context as _, Result};
+use anyhow::{bail, ensure, format_err, Result};
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
 
@@ -68,31 +68,18 @@ pub struct MimeFactory<'a> {
     references: String,
     req_mdn: bool,
     last_added_location_id: u32,
-
-    /// If the created mime-structure contains sync-items,
-    /// the IDs of these items are listed here.
-    /// The IDs are returned via `RenderedEmail`
-    /// and must be deleted if the message is actually queued for sending.
-    sync_ids_to_delete: Option<String>,
-
     attach_selfavatar: bool,
 }
 
 /// Result of rendering a message, ready to be submitted to a send job.
 #[derive(Debug, Clone)]
 pub struct RenderedEmail {
-    pub message: String,
+    pub message: Vec<u8>,
     // pub envelope: Envelope,
     pub is_encrypted: bool,
     pub is_gossiped: bool,
     pub last_added_location_id: u32,
 
-    /// A comma-separated string of sync-IDs that are used by the rendered email
-    /// and must be deleted once the message is actually queued for sending
-    /// (deletion must be done by `delete_sync_ids()`).
-    /// If the rendered email is not queued for sending, the IDs must not be deleted.
-    pub sync_ids_to_delete: Option<String>,
-
     /// Message ID (Message in the sense of Email)
     pub rfc724_mid: String,
     pub subject: String,
@@ -154,12 +141,6 @@ impl<'a> MimeFactory<'a> {
 
         if chat.is_self_talk() {
             recipients.push((from_displayname.to_string(), from_addr.to_string()));
-        } else if chat.is_mailing_list() {
-            let list_post = chat
-                .param
-                .get(Param::ListPost)
-                .context("Can't write to mailinglist without ListPost param")?;
-            recipients.push(("".to_string(), list_post.to_string()));
         } else {
             context
                 .sql
@@ -207,6 +188,7 @@ impl<'a> MimeFactory<'a> {
             )
             .await?;
 
+        let default_str = stock_str::status_line(context).await;
         let factory = MimeFactory {
             from_addr,
             from_displayname,
@@ -214,7 +196,7 @@ impl<'a> MimeFactory<'a> {
             selfstatus: context
                 .get_config(Config::Selfstatus)
                 .await?
-                .unwrap_or_default(),
+                .unwrap_or(default_str),
             recipients,
             timestamp: msg.timestamp_sort,
             loaded: Loaded::Message { chat },
@@ -223,7 +205,6 @@ impl<'a> MimeFactory<'a> {
             references,
             req_mdn,
             last_added_location_id: 0,
-            sync_ids_to_delete: None,
             attach_selfavatar,
         };
         Ok(factory)
@@ -245,10 +226,11 @@ impl<'a> MimeFactory<'a> {
             .get_config(Config::Displayname)
             .await?
             .unwrap_or_default();
+        let default_str = stock_str::status_line(context).await;
         let selfstatus = context
             .get_config(Config::Selfstatus)
             .await?
-            .unwrap_or_default();
+            .unwrap_or(default_str);
         let timestamp = dc_create_smeared_timestamp(context).await;
 
         let res = MimeFactory::<'a> {
@@ -267,7 +249,6 @@ impl<'a> MimeFactory<'a> {
             references: String::default(),
             req_mdn: false,
             last_added_location_id: 0,
-            sync_ids_to_delete: None,
             attach_selfavatar: false,
         };
 
@@ -281,7 +262,7 @@ impl<'a> MimeFactory<'a> {
         let self_addr = context
             .get_config(Config::ConfiguredAddr)
             .await?
-            .context("not configured")?;
+            .ok_or_else(|| format_err!("Not configured"))?;
 
         let mut res = Vec::new();
         for (_, addr) in self
@@ -335,10 +316,6 @@ impl<'a> MimeFactory<'a> {
             Loaded::Message { chat } => {
                 if chat.is_protected() {
                     false
-                } else if chat.typ == Chattype::Broadcast {
-                    // encryption may disclose recipients;
-                    // this is probably a worse issue than not opportunistically (!) encrypting
-                    true
                 } else {
                     self.msg
                         .param
@@ -365,7 +342,7 @@ impl<'a> MimeFactory<'a> {
         match &self.loaded {
             Loaded::Message { chat } => {
                 // beside key- and member-changes, force re-gossip every 48 hours
-                let gossiped_timestamp = chat.id.get_gossiped_timestamp(context).await?;
+                let gossiped_timestamp = chat.get_gossiped_timestamp(context).await?;
                 if time() > gossiped_timestamp + (2 * 24 * 60 * 60) {
                     Ok(true)
                 } else {
@@ -416,6 +393,8 @@ impl<'a> MimeFactory<'a> {
                 }
 
                 if chat.typ == Chattype::Group && quoted_msg_subject.is_none_or_empty() {
+                    // If we have a `quoted_msg_subject`, we use the subject of the quoted message
+                    // instead of the group name
                     let re = if self.in_reply_to.is_empty() {
                         ""
                     } else {
@@ -424,22 +403,22 @@ impl<'a> MimeFactory<'a> {
                     return Ok(format!("{}{}", re, chat.name));
                 }
 
-                if chat.typ != Chattype::Broadcast {
-                    let parent_subject = if quoted_msg_subject.is_none_or_empty() {
-                        chat.param.get(Param::LastSubject)
-                    } else {
-                        quoted_msg_subject.as_deref()
-                    };
-                    if let Some(last_subject) = parent_subject {
-                        return Ok(format!("Re: {}", remove_subject_prefix(last_subject)));
-                    }
-                }
-
-                let self_name = match context.get_config(Config::Displayname).await? {
-                    Some(name) => name,
-                    None => context.get_config(Config::Addr).await?.unwrap_or_default(),
+                let parent_subject = if quoted_msg_subject.is_none_or_empty() {
+                    chat.param.get(Param::LastSubject)
+                } else {
+                    quoted_msg_subject.as_deref()
                 };
-                stock_str::subject_for_new_contact(context, self_name).await
+
+                if let Some(last_subject) = parent_subject {
+                    format!("Re: {}", remove_subject_prefix(last_subject))
+                } else {
+                    let self_name = match context.get_config(Config::Displayname).await? {
+                        Some(name) => name,
+                        None => context.get_config(Config::Addr).await?.unwrap_or_default(),
+                    };
+
+                    stock_str::subject_for_new_contact(context, self_name).await
+                }
             }
             Loaded::Mdn { .. } => stock_str::read_rcpt(context).await,
         };
@@ -462,45 +441,20 @@ impl<'a> MimeFactory<'a> {
             self.from_addr.clone(),
         );
 
-        let undisclosed_recipients = match &self.loaded {
-            Loaded::Message { chat } => chat.typ == Chattype::Broadcast,
-            Loaded::Mdn { .. } => false,
-        };
-
         let mut to = Vec::new();
-        if undisclosed_recipients {
-            to.push(Address::new_group(
-                "hidden-recipients".to_string(),
-                Vec::new(),
-            ));
-        } else {
-            let email_to_remove =
-                if self.msg.param.get_cmd() == SystemMessage::MemberRemovedFromGroup {
-                    self.msg.param.get(Param::Arg)
-                } else {
-                    None
-                };
-
-            for (name, addr) in self.recipients.iter() {
-                if let Some(email_to_remove) = email_to_remove {
-                    if email_to_remove == addr {
-                        continue;
-                    }
-                }
-
-                if name.is_empty() {
-                    to.push(Address::new_mailbox(addr.clone()));
-                } else {
-                    to.push(Address::new_mailbox_with_name(
-                        name.to_string(),
-                        addr.clone(),
-                    ));
-                }
+        for (name, addr) in self.recipients.iter() {
+            if name.is_empty() {
+                to.push(Address::new_mailbox(addr.clone()));
+            } else {
+                to.push(Address::new_mailbox_with_name(
+                    name.to_string(),
+                    addr.clone(),
+                ));
             }
+        }
 
-            if to.is_empty() {
-                to.push(from.clone());
-            }
+        if to.is_empty() {
+            to.push(from.clone());
         }
 
         headers
@@ -604,7 +558,6 @@ impl<'a> MimeFactory<'a> {
         headers
             .unprotected
             .push(Header::new_with_value("To".into(), to).unwrap());
-
         headers
             .unprotected
             .push(Header::new_with_value("From".into(), vec![from]).unwrap());
@@ -636,25 +589,12 @@ impl<'a> MimeFactory<'a> {
             main_part
         } else {
             // Multiple parts, render as multipart.
-            let part_holder = if self.msg.param.get_cmd() == SystemMessage::MultiDeviceSync {
-                PartBuilder::new().header((
-                    "Content-Type".to_string(),
-                    "multipart/report; report-type=multi-device-sync".to_string(),
-                ))
-            } else if self.msg.param.get_cmd() == SystemMessage::WebxdcStatusUpdate {
-                PartBuilder::new().header((
-                    "Content-Type".to_string(),
-                    "multipart/report; report-type=status-update".to_string(),
-                ))
-            } else {
-                PartBuilder::new().message_type(MimeMultipartType::Mixed)
-            };
-
-            parts
-                .into_iter()
-                .fold(part_holder.child(main_part.build()), |message, part| {
-                    message.child(part.build())
-                })
+            parts.into_iter().fold(
+                PartBuilder::new()
+                    .message_type(MimeMultipartType::Mixed)
+                    .child(main_part.build()),
+                |message, part| message.child(part.build()),
+            )
         };
 
         let outer_message = if is_encrypted {
@@ -770,12 +710,11 @@ impl<'a> MimeFactory<'a> {
         } = self;
 
         Ok(RenderedEmail {
-            message: outer_message.build().as_string(),
+            message: outer_message.build().as_string().into_bytes(),
             // envelope: Envelope::new,
             is_encrypted,
             is_gossiped,
             last_added_location_id,
-            sync_ids_to_delete: self.sync_ids_to_delete,
             rfc724_mid,
             subject: subject_str,
         })
@@ -920,9 +859,7 @@ impl<'a> MimeFactory<'a> {
                     "ephemeral-timer-changed".to_string(),
                 ));
             }
-            SystemMessage::LocationOnly
-            | SystemMessage::MultiDeviceSync
-            | SystemMessage::WebxdcStatusUpdate => {
+            SystemMessage::LocationOnly => {
                 // This should prevent automatic replies,
                 // such as non-delivery reports.
                 //
@@ -1152,25 +1089,6 @@ impl<'a> MimeFactory<'a> {
             }
         }
 
-        // we do not piggyback sync-files to other self-sent-messages
-        // to not risk files becoming too larger and being skipped by download-on-demand.
-        if command == SystemMessage::MultiDeviceSync && self.is_e2ee_guaranteed() {
-            let json = self.msg.param.get(Param::Arg).unwrap_or_default();
-            let ids = self.msg.param.get(Param::Arg2).unwrap_or_default();
-            parts.push(context.build_sync_part(json.to_string()).await);
-            self.sync_ids_to_delete = Some(ids.to_string());
-        } else if command == SystemMessage::WebxdcStatusUpdate {
-            let json = self.msg.param.get(Param::Arg).unwrap_or_default();
-            parts.push(context.build_status_update_part(json).await);
-        } else if self.msg.viewtype == Viewtype::Webxdc {
-            if let Some(json) = context
-                .render_webxdc_status_update_object(self.msg.id, None)
-                .await?
-            {
-                parts.push(context.build_status_update_part(&json).await);
-            }
-        }
-
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar).await? {
                 Some(path) => match build_selfavatar_file(context, &path) {
@@ -1225,11 +1143,7 @@ impl<'a> MimeFactory<'a> {
         {
             stock_str::encrypted_msg(context).await
         } else {
-            self.msg
-                .get_summary(context, None)
-                .await?
-                .truncated_text(32)
-                .to_string()
+            self.msg.get_summarytext(context, 32).await
         };
         let p2 = stock_str::read_rcpt_mail_body(context, p1).await;
         let message_text = format!("{}\r\n", format_flowed(&p2));
@@ -1300,7 +1214,7 @@ async fn build_body_file(
         .param
         .get_blob(Param::File, context, true)
         .await?
-        .context("msg has no filename")?;
+        .ok_or_else(|| format_err!("msg has no filename"))?;
     let suffix = blob.suffix().unwrap_or("dat");
 
     // Get file name to use for sending.  For privacy purposes, we do
@@ -1328,7 +1242,8 @@ async fn build_body_file(
             "video_{}.{}",
             chrono::Utc
                 .timestamp(msg.timestamp_sort, 0)
-                .format("%Y-%m-%d_%H-%M-%S"),
+                .format("%Y-%m-%d_%H-%M-%S")
+                .to_string(),
             &suffix
         ),
         _ => blob.as_file_name().to_string(),
@@ -1439,10 +1354,6 @@ mod tests {
     use async_std::prelude::*;
 
     use crate::chat::ChatId;
-    use crate::chat::{
-        add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg,
-        ProtectionStatus,
-    };
     use crate::chatlist::Chatlist;
     use crate::contact::Origin;
     use crate::dc_receive_imf::dc_receive_imf;
@@ -1450,7 +1361,7 @@ mod tests {
     use crate::test_utils::{get_chat_msg, TestContext};
 
     use async_std::fs::File;
-    use mailparse::{addrparse_header, MailHeaderMap};
+    use pretty_assertions::assert_eq;
 
     #[test]
     fn test_render_email_address() {
@@ -1548,7 +1459,7 @@ mod tests {
             msg_to_subject_str(
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: Bob <bob@example.com>\n\
-                To: alice@example.org\n\
+                To: alice@example.com\n\
                 Subject: Antw: Chat: hello\n\
                 Message-ID: <2222@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
@@ -1563,7 +1474,7 @@ mod tests {
             msg_to_subject_str(
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: Bob <bob@example.com>\n\
-                To: alice@example.org\n\
+                To: alice@example.com\n\
                 Subject: Infos: 42\n\
                 Message-ID: <2222@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
@@ -1582,7 +1493,7 @@ mod tests {
             msg_to_subject_str(
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: bob@example.com\n\
-                To: alice@example.org\n\
+                To: alice@example.com\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
                 Message-ID: <2223@example.com>\n\
@@ -1600,7 +1511,7 @@ mod tests {
         // 3. Send the first message to a new contact
         let t = TestContext::new_alice().await;
 
-        assert_eq!(first_subject_str(t).await, "Message from alice@example.org");
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
 
         let t = TestContext::new_alice().await;
         t.set_config(Config::Displayname, Some("Alice"))
@@ -1615,7 +1526,7 @@ mod tests {
         msg_to_subject_str(
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
             From: bob@example.com\n\
-            To: alice@example.org\n\
+            To: alice@example.com\n\
             Subject: äääää\n\
             Chat-Version: 1.0\n\
             Message-ID: <2893@example.com>\n\
@@ -1629,7 +1540,7 @@ mod tests {
         msg_to_subject_str(
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
             From: bob@example.com\n\
-            To: alice@example.org\n\
+            To: alice@example.com\n\
             Subject: aäääää\n\
             Chat-Version: 1.0\n\
             Message-ID: <2893@example.com>\n\
@@ -1648,7 +1559,7 @@ mod tests {
         dc_receive_imf(
             &t,
             b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-            From: alice@example.org\n\
+            From: alice@example.com\n\
             To: bob@example.com\n\
             Subject: Hello, Bob\n\
             Chat-Version: 1.0\n\
@@ -1657,6 +1568,7 @@ mod tests {
             \n\
             hello\n",
             "INBOX",
+            1,
             false,
         )
         .await
@@ -1664,7 +1576,7 @@ mod tests {
         let new_msg = incoming_msg_to_reply_msg(
             b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                  From: bob@example.com\n\
-                 To: alice@example.org\n\
+                 To: alice@example.com\n\
                  Subject: message opened\n\
                  Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
                  Chat-Version: 1.0\n\
@@ -1693,30 +1605,27 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_subject_in_group() -> Result<()> {
+    async fn test_subject_in_group() {
         async fn send_msg_get_subject(
             t: &TestContext,
             group_id: ChatId,
             quote: Option<&Message>,
-        ) -> Result<String> {
+        ) -> String {
             let mut new_msg = Message::new(Viewtype::Text);
             new_msg.set_text(Some("Hi".to_string()));
             if let Some(q) = quote {
-                new_msg.set_quote(t, Some(q)).await?;
+                new_msg.set_quote(t, q).await.unwrap();
             }
             let sent = t.send_msg(group_id, &mut new_msg).await;
             get_subject(t, sent).await
         }
-        async fn get_subject(
-            t: &TestContext,
-            sent: crate::test_utils::SentMessage,
-        ) -> Result<String> {
+        async fn get_subject(t: &TestContext, sent: crate::test_utils::SentMessage) -> String {
             let parsed_subject = t.parse_msg(&sent).await.get_subject().unwrap();
 
-            let sent_msg = Message::load_from_db(t, sent.sender_msg_id).await?;
+            let sent_msg = Message::load_from_db(t, sent.sender_msg_id).await.unwrap();
             assert_eq!(parsed_subject, sent_msg.subject);
 
-            Ok(parsed_subject)
+            parsed_subject
         }
 
         // 6. Test that in a group, replies also take the quoted message's subject, while non-replies use the group title as subject
@@ -1725,13 +1634,13 @@ mod tests {
             chat::create_group_chat(&t, chat::ProtectionStatus::Unprotected, "groupname") // TODO encodings, ä
                 .await
                 .unwrap();
-        let bob = Contact::create(&t, "", "bob@example.org").await?;
-        chat::add_contact_to_chat(&t, group_id, bob).await?;
+        let bob = Contact::create(&t, "", "bob@example.org").await.unwrap();
+        chat::add_contact_to_chat(&t, group_id, bob).await;
 
-        let subject = send_msg_get_subject(&t, group_id, None).await?;
+        let subject = send_msg_get_subject(&t, group_id, None).await;
         assert_eq!(subject, "groupname");
 
-        let subject = send_msg_get_subject(&t, group_id, None).await?;
+        let subject = send_msg_get_subject(&t, group_id, None).await;
         assert_eq!(subject, "Re: groupname");
 
         dc_receive_imf(
@@ -1739,7 +1648,7 @@ mod tests {
             format!(
                 "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: bob@example.com\n\
-                To: alice@example.org\n\
+                To: alice@example.com\n\
                 Subject: Different subject\n\
                 In-Reply-To: {}\n\
                 Message-ID: <2893@example.com>\n\
@@ -1750,28 +1659,31 @@ mod tests {
             )
             .as_bytes(),
             "INBOX",
+            5,
             false,
         )
-        .await?;
+        .await
+        .unwrap();
         let message_from_bob = t.get_last_msg().await;
 
-        let subject = send_msg_get_subject(&t, group_id, None).await?;
+        let subject = send_msg_get_subject(&t, group_id, None).await;
         assert_eq!(subject, "Re: groupname");
 
-        let subject = send_msg_get_subject(&t, group_id, Some(&message_from_bob)).await?;
+        let subject = send_msg_get_subject(&t, group_id, Some(&message_from_bob)).await;
         let outgoing_quoting_msg = t.get_last_msg().await;
         assert_eq!(subject, "Re: Different subject");
 
-        let subject = send_msg_get_subject(&t, group_id, None).await?;
+        let subject = send_msg_get_subject(&t, group_id, None).await;
         assert_eq!(subject, "Re: groupname");
 
-        let subject = send_msg_get_subject(&t, group_id, Some(&outgoing_quoting_msg)).await?;
+        let subject = send_msg_get_subject(&t, group_id, Some(&outgoing_quoting_msg)).await;
         assert_eq!(subject, "Re: Different subject");
 
-        chat::forward_msgs(&t, &[message_from_bob.id], group_id).await?;
-        let subject = get_subject(&t, t.pop_sent_msg().await).await?;
-        assert_eq!(subject, "Re: groupname");
-        Ok(())
+        chat::forward_msgs(&t, &[message_from_bob.id], group_id)
+            .await
+            .unwrap();
+        let subject = get_subject(&t, t.pop_sent_msg().await).await;
+        assert_eq!(subject, "Fwd: Different subject");
     }
 
     async fn first_subject_str(t: TestContext) -> String {
@@ -1793,7 +1705,7 @@ mod tests {
         mf.subject_str(&t).await.unwrap()
     }
 
-    // In `imf_raw`, From has to be bob@example.com, To has to be alice@example.org
+    // In `imf_raw`, From has to be bob@example.com, To has to be alice@example.com
     async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
         let subject_str = msg_to_subject_str_inner(imf_raw, false, false, false).await;
 
@@ -1854,13 +1766,14 @@ mod tests {
                 &t,
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                     From: Bob <bob@example.com>\n\
-                    To: alice@example.org\n\
+                    To: alice@example.com\n\
                     Subject: Some other, completely unrelated subject\n\
                     Message-ID: <3cl4@example.com>\n\
                     Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                     \n\
                     Some other, completely unrelated content\n",
                 "INBOX",
+                2,
                 false,
             )
             .await
@@ -1871,7 +1784,7 @@ mod tests {
         }
 
         if reply {
-            new_msg.set_quote(&t, Some(&incoming_msg)).await.unwrap();
+            new_msg.set_quote(&t, &incoming_msg).await.unwrap();
         }
 
         let mf = MimeFactory::from_msg(&t, &new_msg, false).await.unwrap();
@@ -1885,13 +1798,13 @@ mod tests {
             .await
             .unwrap();
 
-        dc_receive_imf(context, imf_raw, "INBOX", false)
+        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
             .await
             .unwrap();
 
         let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
 
-        let chat_id = chats.get_chat_id(0).unwrap();
+        let chat_id = chats.get_chat_id(0);
         chat_id.accept(context).await.unwrap();
 
         let mut new_msg = Message::new(Viewtype::Text);
@@ -1913,7 +1826,7 @@ mod tests {
         let msg = incoming_msg_to_reply_msg(
             b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: Charlie <charlie@example.com>\n\
-                To: alice@example.org\n\
+                To: alice@example.com\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
                 Message-ID: <2223@example.com>\n\
@@ -1931,7 +1844,7 @@ mod tests {
 
         let rendered_msg = mimefactory.render(context).await.unwrap();
 
-        let mail = mailparse::parse_mail(rendered_msg.message.as_bytes()).unwrap();
+        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
         assert_eq!(
             mail.headers
                 .iter()
@@ -1941,7 +1854,7 @@ mod tests {
             "1.0"
         );
 
-        let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes())
+        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
             .await
             .unwrap();
     }
@@ -2015,9 +1928,8 @@ mod tests {
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("this is the text!".to_string()));
 
-        let sent_msg = t.send_msg(chat.id, &mut msg).await;
-        let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
-
+        let payload = t.send_msg(chat.id, &mut msg).await.payload();
+        let mut payload = payload.splitn(3, "\r\n\r\n");
         let outer = payload.next().unwrap();
         let inner = payload.next().unwrap();
         let body = payload.next().unwrap();
@@ -2035,8 +1947,8 @@ mod tests {
 
         // if another message is sent, that one must not contain the avatar
         // and no artificial multipart/mixed nesting
-        let sent_msg = t.send_msg(chat.id, &mut msg).await;
-        let mut payload = sent_msg.payload().splitn(2, "\r\n\r\n");
+        let payload = t.send_msg(chat.id, &mut msg).await.payload();
+        let mut payload = payload.splitn(2, "\r\n\r\n");
         let outer = payload.next().unwrap();
         let body = payload.next().unwrap();
 
@@ -2053,35 +1965,4 @@ mod tests {
 
         Ok(())
     }
-
-    /// Test that removed member address does not go into the `To:` field.
-    #[async_std::test]
-    async fn test_remove_member_bcc() -> Result<()> {
-        // Alice creates a group with Bob and Claire and then removes Bob.
-        let alice = TestContext::new_alice().await;
-
-        let bob_id = Contact::create(&alice, "Bob", "bob@example.net").await?;
-        let claire_id = Contact::create(&alice, "Claire", "claire@foo.de").await?;
-
-        let alice_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "foo").await?;
-        add_contact_to_chat(&alice, alice_chat_id, bob_id).await?;
-        add_contact_to_chat(&alice, alice_chat_id, claire_id).await?;
-        send_text_msg(&alice, alice_chat_id, "Creating a group".to_string()).await?;
-
-        remove_contact_from_chat(&alice, alice_chat_id, claire_id).await?;
-        let remove = alice.pop_sent_msg().await;
-        let remove_payload = remove.payload();
-        let parsed = mailparse::parse_mail(remove_payload.as_bytes())?;
-        let to = parsed
-            .headers
-            .get_first_header("To")
-            .context("no To: header parsed")?;
-        let to = addrparse_header(to)?;
-        let mailbox = to
-            .extract_single_info()
-            .context("to: field does not contain exactly one address")?;
-        assert_eq!(mailbox.addr, "bob@example.net");
-
-        Ok(())
-    }
 }

src/mimeparser.rs

@@ -15,7 +15,7 @@ use crate::blob::BlobObject;
 use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
 use crate::contact::addr_normalize;
 use crate::context::Context;
-use crate::dc_tools::{dc_get_filemeta, dc_truncate, parse_receive_headers};
+use crate::dc_tools::{dc_get_filemeta, dc_truncate};
 use crate::dehtml::dehtml;
 use crate::e2ee;
 use crate::events::EventType;
@@ -28,7 +28,6 @@ use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::simplify::simplify;
 use crate::stock_str;
-use crate::sync::SyncItems;
 
 /// A parsed MIME message.
 ///
@@ -47,7 +46,6 @@ pub struct MimeMessage {
     /// Addresses are normalized and lowercased:
     pub recipients: Vec<SingleInfo>,
     pub from: Vec<SingleInfo>,
-    pub list_post: Option<String>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
 
@@ -58,15 +56,11 @@ pub struct MimeMessage {
     /// this set is empty.
     pub signatures: HashSet<Fingerprint>,
 
-    /// The set of mail recipient addresses for which gossip headers were applied, regardless of
-    /// whether they modified any peerstates.
-    pub gossiped_addr: HashSet<String>,
+    pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub is_system_message: SystemMessage,
     pub location_kml: Option<location::Kml>,
     pub message_kml: Option<location::Kml>,
-    pub(crate) sync_items: Option<SyncItems>,
-    pub(crate) webxdc_status_update: Option<String>,
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
     pub(crate) mdn_reports: Vec<Report>,
@@ -84,8 +78,6 @@ pub struct MimeMessage {
     /// This is non-empty only if the message was actually encrypted.  It is used
     /// for e.g. late-parsing HTML.
     pub decoded_data: Vec<u8>,
-
-    pub(crate) hop_info: String,
 }
 
 #[derive(Debug, PartialEq)]
@@ -132,14 +124,6 @@ pub enum SystemMessage {
     // Chat protection state changed
     ChatProtectionEnabled = 11,
     ChatProtectionDisabled = 12,
-
-    /// Self-sent-message that contains only json used for multi-device-sync;
-    /// if possible, we attach that to other messages as for locations.
-    MultiDeviceSync = 20,
-
-    // Sync message that contains a json payload
-    // sent to the other webxdc instances
-    WebxdcStatusUpdate = 30,
 }
 
 impl Default for SystemMessage {
@@ -152,18 +136,6 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
 
 impl MimeMessage {
     pub async fn from_bytes(context: &Context, body: &[u8]) -> Result<Self> {
-        MimeMessage::from_bytes_with_partial(context, body, None).await
-    }
-
-    /// Parse a mime message.
-    ///
-    /// If `partial` is set, it contains the full message size in bytes
-    /// and `body` contains the header only.
-    pub async fn from_bytes_with_partial(
-        context: &Context,
-        body: &[u8],
-        partial: Option<u32>,
-    ) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
 
         let message_time = mail
@@ -171,12 +143,10 @@ impl MimeMessage {
             .get_header_value(HeaderDef::Date)
             .and_then(|v| mailparse::dateparse(&v).ok())
             .unwrap_or_default();
-        let hop_info = parse_receive_headers(&mail.get_headers());
 
         let mut headers = Default::default();
         let mut recipients = Default::default();
         let mut from = Default::default();
-        let mut list_post = Default::default();
         let mut chat_disposition_notification_to = None;
 
         // Parse IMF headers.
@@ -185,7 +155,6 @@ impl MimeMessage {
             &mut headers,
             &mut recipients,
             &mut from,
-            &mut list_post,
             &mut chat_disposition_notification_to,
             &mail.headers,
         );
@@ -211,7 +180,7 @@ impl MimeMessage {
 
         // Memory location for a possible decrypted message.
         let mut mail_raw = Vec::new();
-        let mut gossiped_addr = Default::default();
+        let mut gossipped_addr = Default::default();
 
         let (mail, signatures, warn_empty_signature) =
             match e2ee::try_decrypt(context, &mail, message_time).await {
@@ -234,7 +203,7 @@ impl MimeMessage {
                         if !signatures.is_empty() {
                             let gossip_headers =
                                 decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
-                            gossiped_addr = update_gossip_peerstates(
+                            gossipped_addr = update_gossip_peerstates(
                                 context,
                                 message_time,
                                 &mail,
@@ -259,7 +228,6 @@ impl MimeMessage {
                             &mut headers,
                             &mut recipients,
                             &mut throwaway_from,
-                            &mut list_post,
                             &mut chat_disposition_notification_to,
                             &decrypted_mail.headers,
                         );
@@ -287,41 +255,26 @@ impl MimeMessage {
             parts: Vec::new(),
             header: headers,
             recipients,
-            list_post,
             from,
             chat_disposition_notification_to,
             decrypting_failed: false,
 
             // only non-empty if it was a valid autocrypt message
             signatures,
-            gossiped_addr,
+            gossipped_addr,
             is_forwarded: false,
             mdn_reports: Vec::new(),
             is_system_message: SystemMessage::Unknown,
             location_kml: None,
             message_kml: None,
-            sync_items: None,
-            webxdc_status_update: None,
             user_avatar: None,
             group_avatar: None,
             failure_report: None,
             footer: None,
             is_mime_modified: false,
             decoded_data: Vec::new(),
-            hop_info,
         };
-
-        match partial {
-            Some(org_bytes) => {
-                parser
-                    .create_stub_from_partial_download(context, org_bytes)
-                    .await?;
-            }
-            None => {
-                parser.parse_mime_recursive(context, &mail, false).await?;
-            }
-        };
-
+        parser.parse_mime_recursive(context, &mail, false).await?;
         parser.maybe_remove_bad_parts();
         parser.maybe_remove_inline_mailinglist_footer();
         parser.heuristically_parse_ndn(context).await;
@@ -403,18 +356,16 @@ impl MimeMessage {
     #[allow(clippy::indexing_slicing)]
     fn squash_attachment_parts(&mut self) {
         if let [textpart, filepart] = &self.parts[..] {
-            let need_drop = textpart.typ == Viewtype::Text
-                && match filepart.typ {
-                    Viewtype::Image
-                    | Viewtype::Gif
-                    | Viewtype::Sticker
-                    | Viewtype::Audio
-                    | Viewtype::Voice
-                    | Viewtype::Video
-                    | Viewtype::File
-                    | Viewtype::Webxdc => true,
-                    Viewtype::Unknown | Viewtype::Text | Viewtype::VideochatInvitation => false,
-                };
+            let need_drop = {
+                textpart.typ == Viewtype::Text
+                    && (filepart.typ == Viewtype::Image
+                        || filepart.typ == Viewtype::Gif
+                        || filepart.typ == Viewtype::Sticker
+                        || filepart.typ == Viewtype::Audio
+                        || filepart.typ == Viewtype::Voice
+                        || filepart.typ == Viewtype::Video
+                        || filepart.typ == Viewtype::File)
+            };
 
             if need_drop {
                 let mut filepart = self.parts.swap_remove(1);
@@ -546,7 +497,7 @@ impl MimeMessage {
             };
 
             if let Some(ref subject) = self.get_subject() {
-                if !self.has_chat_version() && self.webxdc_status_update.is_none() {
+                if !self.has_chat_version() {
                     part.msg = subject.to_string();
                 }
             }
@@ -839,18 +790,6 @@ impl MimeMessage {
                                 }
                             }
                         }
-                        Some("multi-device-sync") => {
-                            if let Some(second) = mail.subparts.get(1) {
-                                self.add_single_part_if_known(context, second, is_related)
-                                    .await?;
-                            }
-                        }
-                        Some("status-update") => {
-                            if let Some(second) = mail.subparts.get(1) {
-                                self.add_single_part_if_known(context, second, is_related)
-                                    .await?;
-                            }
-                        }
                         Some(_) => {
                             if let Some(first) = mail.subparts.get(0) {
                                 any_part_added = self
@@ -1020,13 +959,8 @@ impl MimeMessage {
         if decoded_data.is_empty() {
             return;
         }
-        let msg_type = if context
-            .is_webxdc_file(filename, decoded_data)
-            .await
-            .unwrap_or(false)
-        {
-            Viewtype::Webxdc
-        } else if filename.ends_with(".kml") {
+        // treat location/message kml file attachments specially
+        if filename.ends_with(".kml") {
             // XXX what if somebody sends eg an "location-highlights.kml"
             // attachment unrelated to location streaming?
             if filename.starts_with("location") || filename.starts_with("message") {
@@ -1042,29 +976,7 @@ impl MimeMessage {
                 }
                 return;
             }
-            msg_type
-        } else if filename == "multi-device-sync.json" {
-            let serialized = String::from_utf8_lossy(decoded_data)
-                .parse()
-                .unwrap_or_default();
-            self.sync_items = context
-                .parse_sync_items(serialized)
-                .await
-                .map_err(|err| {
-                    warn!(context, "failed to parse sync data: {}", err);
-                })
-                .ok();
-            return;
-        } else if filename == "status-update.json" {
-            let serialized = String::from_utf8_lossy(decoded_data)
-                .parse()
-                .unwrap_or_default();
-            self.webxdc_status_update = Some(serialized);
-            return;
-        } else {
-            msg_type
-        };
-
+        }
         /* we have a regular file attachment,
         write decoded data to new blob object */
 
@@ -1129,11 +1041,11 @@ impl MimeMessage {
         }
     }
 
-    pub fn repl_msg_by_error(&mut self, error_msg: &str) {
+    pub fn repl_msg_by_error(&mut self, error_msg: impl AsRef<str>) {
         self.is_system_message = SystemMessage::Unknown;
         if let Some(part) = self.parts.first_mut() {
             part.typ = Viewtype::Text;
-            part.msg = format!("[{}]", error_msg);
+            part.msg = format!("[{}]", error_msg.as_ref());
             self.parts.truncate(1);
         }
     }
@@ -1149,7 +1061,6 @@ impl MimeMessage {
         headers: &mut HashMap<String, String>,
         recipients: &mut Vec<SingleInfo>,
         from: &mut Vec<SingleInfo>,
-        list_post: &mut Option<String>,
         chat_disposition_notification_to: &mut Option<SingleInfo>,
         fields: &[mailparse::MailHeader<'_>],
     ) {
@@ -1180,10 +1091,6 @@ impl MimeMessage {
         if !from_new.is_empty() {
             *from = from_new;
         }
-        let list_post_new = get_list_post(fields);
-        if list_post_new.is_some() {
-            *list_post = list_post_new;
-        }
     }
 
     fn process_report(
@@ -1201,21 +1108,23 @@ impl MimeMessage {
 
         // must be present
         if let Some(_disposition) = report_fields.get_header_value(HeaderDef::Disposition) {
-            let original_message_id = report_fields
+            if let Some(original_message_id) = report_fields
                 .get_header_value(HeaderDef::OriginalMessageId)
-                .and_then(|v| parse_message_id(&v).ok());
-            let additional_message_ids = report_fields
-                .get_header_value(HeaderDef::AdditionalMessageIds)
-                .map_or_else(Vec::new, |v| {
-                    v.split(' ')
-                        .filter_map(|s| parse_message_id(s).ok())
-                        .collect()
-                });
+                .and_then(|v| parse_message_id(&v).ok())
+            {
+                let additional_message_ids = report_fields
+                    .get_header_value(HeaderDef::AdditionalMessageIds)
+                    .map_or_else(Vec::new, |v| {
+                        v.split(' ')
+                            .filter_map(|s| parse_message_id(s).ok())
+                            .collect()
+                    });
 
-            return Ok(Some(Report {
-                original_message_id,
-                additional_message_ids,
-            }));
+                return Ok(Some(Report {
+                    original_message_id,
+                    additional_message_ids,
+                }));
+            }
         }
         warn!(
             context,
@@ -1371,10 +1280,8 @@ impl MimeMessage {
         parts: &[Part],
     ) {
         for report in &self.mdn_reports {
-            for original_message_id in report
-                .original_message_id
-                .iter()
-                .chain(&report.additional_message_ids)
+            for original_message_id in
+                std::iter::once(&report.original_message_id).chain(&report.additional_message_ids)
             {
                 match message::handle_mdn(context, from_id, original_message_id, sent_timestamp)
                     .await
@@ -1424,9 +1331,6 @@ impl MimeMessage {
     }
 }
 
-/// Parses `Autocrypt-Gossip` headers from the email and applies them to peerstates.
-///
-/// Returns the set of mail recipient addresses for which valid gossip headers were found.
 async fn update_gossip_peerstates(
     context: &Context,
     message_time: i64,
@@ -1434,55 +1338,48 @@ async fn update_gossip_peerstates(
     gossip_headers: Vec<String>,
 ) -> Result<HashSet<String>> {
     // XXX split the parsing from the modification part
-    let mut gossiped_addr: HashSet<String> = Default::default();
+    let mut gossipped_addr: HashSet<String> = Default::default();
 
     for value in &gossip_headers {
-        let header = match value.parse::<Aheader>() {
-            Ok(header) => header,
-            Err(err) => {
-                warn!(context, "Failed parsing Autocrypt-Gossip header: {}", err);
-                continue;
+        let gossip_header = value.parse::<Aheader>();
+
+        if let Ok(ref header) = gossip_header {
+            if get_recipients(&mail.headers)
+                .iter()
+                .any(|info| info.addr == header.addr.to_lowercase())
+            {
+                let mut peerstate = Peerstate::from_addr(context, &header.addr).await?;
+                if let Some(ref mut peerstate) = peerstate {
+                    peerstate.apply_gossip(header, message_time);
+                    peerstate.save_to_db(&context.sql, false).await?;
+                } else {
+                    let p = Peerstate::from_gossip(header, message_time);
+                    p.save_to_db(&context.sql, true).await?;
+                    peerstate = Some(p);
+                }
+                if let Some(peerstate) = peerstate {
+                    peerstate
+                        .handle_fingerprint_change(context, message_time)
+                        .await?;
+                }
+
+                gossipped_addr.insert(header.addr.clone());
+            } else {
+                warn!(
+                    context,
+                    "Ignoring gossipped \"{}\" as the address is not in To/Cc list.", &header.addr,
+                );
             }
-        };
-
-        if !get_recipients(&mail.headers)
-            .iter()
-            .any(|info| info.addr == header.addr.to_lowercase())
-        {
-            warn!(
-                context,
-                "Ignoring gossiped \"{}\" as the address is not in To/Cc list.", &header.addr,
-            );
-            continue;
         }
-
-        let peerstate;
-        if let Some(mut p) = Peerstate::from_addr(context, &header.addr).await? {
-            p.apply_gossip(&header, message_time);
-            p.save_to_db(&context.sql, false).await?;
-            peerstate = p;
-        } else {
-            let p = Peerstate::from_gossip(&header, message_time);
-            p.save_to_db(&context.sql, true).await?;
-            peerstate = p;
-        };
-        peerstate
-            .handle_fingerprint_change(context, message_time)
-            .await?;
-
-        gossiped_addr.insert(header.addr.clone());
     }
 
-    Ok(gossiped_addr)
+    Ok(gossipped_addr)
 }
 
 #[derive(Debug)]
 pub(crate) struct Report {
     /// Original-Message-ID header
-    ///
-    /// It MUST be present if the original message has a Message-ID according to RFC 8098, but MS
-    /// Exchange does not add it nevertheless, in which case it is `None`.
-    original_message_id: Option<String>,
+    original_message_id: String,
     /// Additional-Message-IDs
     additional_message_ids: Vec<String>,
 }
@@ -1546,9 +1443,9 @@ pub struct Part {
     pub msg_raw: Option<String>,
     pub bytes: usize,
     pub param: Params,
-    pub(crate) org_filename: Option<String>,
+    org_filename: Option<String>,
     pub error: Option<String>,
-    pub(crate) dehtml_failed: bool,
+    dehtml_failed: bool,
 
     /// the part is a child or a descendant of multipart/related.
     /// typically, these are images that are referenced from text/html part
@@ -1556,7 +1453,7 @@ pub struct Part {
     ///
     /// note that multipart/related may contain further multipart nestings
     /// and all of them needs to be marked with `is_related`.
-    pub(crate) is_related: bool,
+    is_related: bool,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1672,14 +1569,6 @@ pub(crate) fn get_from(headers: &[MailHeader]) -> Vec<SingleInfo> {
     get_all_addresses_from_header(headers, |header_key| header_key == "from")
 }
 
-/// Returned addresses are normalized and lowercased.
-pub(crate) fn get_list_post(headers: &[MailHeader]) -> Option<String> {
-    get_all_addresses_from_header(headers, |header_key| header_key == "list-post")
-        .into_iter()
-        .next()
-        .map(|s| s.addr)
-}
-
 fn get_all_addresses_from_header<F>(headers: &[MailHeader], pred: F) -> Vec<SingleInfo>
 where
     F: Fn(String) -> bool,
@@ -2399,7 +2288,7 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
         assert_eq!(message.mdn_reports.len(), 1);
         assert_eq!(
             message.mdn_reports[0].original_message_id,
-            Some("foo@example.org".to_string())
+            "foo@example.org"
         );
         assert_eq!(
             &message.mdn_reports[0].additional_message_ids,
@@ -2901,6 +2790,7 @@ On 2020-10-25, Bob wrote:
             &t.ctx,
             include_bytes!("../test-data/message/subj_with_multimedia_msg.eml"),
             "INBOX",
+            1,
             false,
         )
         .await
@@ -3049,7 +2939,7 @@ Subject: ...
 
 Some quote.
 "###;
-        dc_receive_imf(&t, raw, "INBOX", false).await?;
+        dc_receive_imf(&t, raw, "INBOX", 1, false).await?;
 
         // Delta Chat generates In-Reply-To with a starting tab when Message-ID is too long.
         let raw = br###"In-Reply-To:
@@ -3066,7 +2956,7 @@ Subject: ...
 Some reply
 "###;
 
-        dc_receive_imf(&t, raw, "INBOX", false).await?;
+        dc_receive_imf(&t, raw, "INBOX", 2, false).await?;
 
         let msg = t.get_last_msg().await;
         assert_eq!(msg.get_text().unwrap(), "Some reply");
@@ -3086,21 +2976,21 @@ Some reply
 Chat-Version: 1.0\n\
 Message-ID: <foobarbaz@example.org>
 To: Bob <bob@example.org>
-From: Alice <alice@example.org>
+From: Alice <alice@example.com>
 Subject: subject
-Chat-Disposition-Notification-To: alice@example.org
+Chat-Disposition-Notification-To: alice@example.com
 
 Message.
 "###;
 
         // Bob receives message.
-        dc_receive_imf(&bob, raw, "INBOX", false).await?;
+        dc_receive_imf(&bob, raw, "INBOX", 1, false).await?;
         let msg = bob.get_last_msg().await;
         // Message is incoming.
         assert!(msg.param.get_bool(Param::WantsMdn).unwrap());
 
         // Alice receives copy-to-self.
-        dc_receive_imf(&alice, raw, "INBOX", false).await?;
+        dc_receive_imf(&alice, raw, "INBOX", 1, false).await?;
         let msg = alice.get_last_msg().await;
         // Message is outgoing, don't send read receipt to self.
         assert!(msg.param.get_bool(Param::WantsMdn).is_none());
@@ -3116,17 +3006,18 @@ Message.
         dc_receive_imf(
             &alice,
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-                 From: alice@example.org\n\
+                 From: alice@example.com\n\
                  To: bob@example.net\n\
                  Subject: foo\n\
                  Message-ID: first@example.com\n\
                  Chat-Version: 1.0\n\
-                 Chat-Disposition-Notification-To: alice@example.org\n\
+                 Chat-Disposition-Notification-To: alice@example.com\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
                  \n\
                  hello\n"
                 .as_bytes(),
             "INBOX",
+            1,
             false,
         )
         .await?;
@@ -3138,8 +3029,8 @@ Message.
         dc_receive_imf(
             &alice,
                 "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-                 From: alice@example.org\n\
-                 To: alice@example.org\n\
+                 From: alice@example.com\n\
+                 To: alice@example.com\n\
                  Subject: message opened\n\
                  Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
                  Chat-Version: 1.0\n\
@@ -3165,6 +3056,7 @@ Message.
                  --SNIPP--"
             .as_bytes(),
             "INBOX",
+            2,
             false,
         )
         .await?;
@@ -3175,18 +3067,4 @@ Message.
 
         Ok(())
     }
-
-    /// Test parsing of MDN sent by MS Exchange.
-    ///
-    /// It does not have required Original-Message-ID field, so it is useless, but we want to
-    /// recognize it as MDN nevertheless to avoid displaying it in the chat as normal message.
-    #[async_std::test]
-    async fn test_ms_exchange_mdn() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let raw =
-            include_bytes!("../test-data/message/ms_exchange_report_disposition_notification.eml");
-        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await?;
-        assert!(!mimeparser.mdn_reports.is_empty());
-        Ok(())
-    }
 }

src/oauth2.rs

@@ -41,7 +41,6 @@ struct Oauth2 {
 
 /// OAuth 2 Access Token Response
 #[derive(Debug, Deserialize)]
-#[allow(dead_code)]
 struct Response {
     // Should always be there according to: <https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/>
     // but previous code handled its abscense.
@@ -59,7 +58,7 @@ pub async fn dc_get_oauth2_url(
     redirect_uri: &str,
 ) -> Result<Option<String>> {
     let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
-    if let Some(oauth2) = Oauth2::from_address(context, addr, socks5_enabled).await {
+    if let Some(oauth2) = Oauth2::from_address(addr, socks5_enabled).await {
         context
             .sql
             .set_raw_config("oauth2_pending_redirect_uri", Some(redirect_uri))
@@ -80,7 +79,7 @@ pub async fn dc_get_oauth2_access_token(
     regenerate: bool,
 ) -> Result<Option<String>> {
     let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
-    if let Some(oauth2) = Oauth2::from_address(context, addr, socks5_enabled).await {
+    if let Some(oauth2) = Oauth2::from_address(addr, socks5_enabled).await {
         let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
@@ -226,7 +225,7 @@ pub async fn dc_get_oauth2_addr(
     code: &str,
 ) -> Result<Option<String>> {
     let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
-    let oauth2 = match Oauth2::from_address(context, addr, socks5_enabled).await {
+    let oauth2 = match Oauth2::from_address(addr, socks5_enabled).await {
         Some(o) => o,
         None => return Ok(None),
     };
@@ -254,13 +253,13 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    async fn from_address(context: &Context, addr: &str, skip_mx: bool) -> Option<Self> {
+    async fn from_address(addr: &str, skip_mx: bool) -> Option<Self> {
         let addr_normalized = normalize_addr(addr);
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            if let Some(oauth2_authorizer) = provider::get_provider_info(context, domain, skip_mx)
+            if let Some(oauth2_authorizer) = provider::get_provider_info(domain, skip_mx)
                 .await
                 .and_then(|provider| provider.oauth2_authorizer.as_ref())
             {
@@ -357,39 +356,32 @@ mod tests {
 
     #[async_std::test]
     async fn test_oauth_from_address() {
-        let t = TestContext::new().await;
         assert_eq!(
-            Oauth2::from_address(&t, "hello@gmail.com", false).await,
+            Oauth2::from_address("hello@gmail.com", false).await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address(&t, "hello@googlemail.com", false).await,
+            Oauth2::from_address("hello@googlemail.com", false).await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address(&t, "hello@yandex.com", false).await,
+            Oauth2::from_address("hello@yandex.com", false).await,
             Some(OAUTH2_YANDEX)
         );
         assert_eq!(
-            Oauth2::from_address(&t, "hello@yandex.ru", false).await,
+            Oauth2::from_address("hello@yandex.ru", false).await,
             Some(OAUTH2_YANDEX)
         );
-        assert_eq!(Oauth2::from_address(&t, "hello@web.de", false).await, None);
+
+        assert_eq!(Oauth2::from_address("hello@web.de", false).await, None);
     }
 
     #[async_std::test]
     async fn test_oauth_from_mx() {
-        // youtube staff seems to use "google workspace with oauth2", figures this out by MX lookup
-        let t = TestContext::new().await;
         assert_eq!(
-            Oauth2::from_address(&t, "hello@youtube.com", false).await,
+            Oauth2::from_address("hello@google.com", false).await,
             Some(OAUTH2_GMAIL)
         );
-        // without MX lookup, we would not know as youtube.com is not in our provider-db
-        assert_eq!(
-            Oauth2::from_address(&t, "hello@youtube.com", true).await,
-            None
-        );
     }
 
     #[async_std::test]

src/param.rs

@@ -4,6 +4,7 @@ use std::str;
 
 use anyhow::{bail, Error};
 use async_std::path::PathBuf;
+use itertools::Itertools;
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 
@@ -110,8 +111,8 @@ pub enum Param {
     /// For Jobs
     AlsoMove = b'M',
 
-    /// For MDN-sending job
-    MsgId = b'I',
+    /// For Jobs: space-separated list of message recipients
+    Recipients = b'R',
 
     /// For Groups
     ///
@@ -136,17 +137,8 @@ pub enum Param {
     /// For Chats
     Devicetalk = b'D',
 
-    /// For Chats: If this is a mailing list chat, contains the List-Post address.
-    /// None if there simply is no `List-Post` header in the mailing list.
-    /// Some("") if the mailing list is using multiple different List-Post headers.
-    ///
-    /// The List-Post address is the email address where the user can write to in order to
-    /// post something to the mailing list.
-    ListPost = b'p',
-
-    /// For Contacts: If this is the List-Post address of a mailing list, contains
-    /// the List-Id of the mailing list (which is also used as the group id of the chat).
-    ListId = b's',
+    /// For MDN-sending job
+    MsgId = b'I',
 
     /// For Contacts: timestamp of status (aka signature or footer) update.
     StatusTimestamp = b'j',
@@ -168,12 +160,6 @@ pub enum Param {
 
     /// For Chats: timestamp of protection settings update.
     ProtectionSettingsTimestamp = b'L',
-
-    /// For Webxdc Message Instances: Current summary
-    WebxdcSummary = b'N',
-
-    /// For Webxdc Message Instances: timestamp of summary update.
-    WebxdcSummaryTimestamp = b'Q',
 }
 
 /// An object for handling key=value parameter lists.
@@ -196,7 +182,7 @@ impl fmt::Display for Params {
                 f,
                 "{}={}",
                 *key as u8 as char,
-                value.split('\n').collect::<Vec<&str>>().join("\n\n")
+                value.split('\n').join("\n\n")
             )?;
         }
         Ok(())
@@ -206,11 +192,6 @@ impl fmt::Display for Params {
 impl str::FromStr for Params {
     type Err = Error;
 
-    /// Parse a raw string to Param.
-    ///
-    /// Silently ignore unknown keys:
-    /// they may come from a downgrade (when a shortly new version adds a key)
-    /// or from an upgrade (when a key is dropped but was used in the past)
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let mut inner = BTreeMap::new();
         let mut lines = s.lines().peekable();
@@ -230,6 +211,8 @@ impl str::FromStr for Params {
 
                 if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
                     inner.insert(key, value);
+                } else {
+                    bail!("Unknown key: {}", key);
                 }
             } else {
                 bail!("Not a key-value pair: {:?}", line);
@@ -432,12 +415,10 @@ impl<'a> ParamsFile<'a> {
 mod tests {
     use super::*;
 
-    use anyhow::Result;
     use async_std::fs;
     use async_std::path::Path;
 
     use crate::test_utils::TestContext;
-    use std::str::FromStr;
 
     #[test]
     fn test_dc_param() {
@@ -540,14 +521,4 @@ mod tests {
         assert!(p.get_path(Param::File, &t).unwrap().is_none());
         assert!(p.get_blob(Param::File, &t, false).await.unwrap().is_none());
     }
-
-    #[async_std::test]
-    async fn test_params_unknown_key() -> Result<()> {
-        // 'Z' is used as a key that is known to be unused; these keys should be ignored silently by definition.
-        let p = Params::from_str("w=12\nZ=13\nh=14")?;
-        assert_eq!(p.len(), 2);
-        assert_eq!(p.get(Param::Width), Some("12"));
-        assert_eq!(p.get(Param::Height), Some("14"));
-        Ok(())
-    }
 }

src/peerstate.rs

@@ -277,7 +277,7 @@ impl Peerstate {
 
                 let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
 
-                chat::add_info_msg(context, chat_id, &msg, timestamp).await?;
+                chat::add_info_msg(context, chat_id, msg, timestamp).await;
                 context.emit_event(EventType::ChatModified(chat_id));
             } else {
                 bail!("contact with peerstate.addr {:?} not found", &self.addr);
@@ -496,34 +496,11 @@ impl Peerstate {
     }
 }
 
-/// Removes duplicate peerstates from `acpeerstates` database table.
-///
-/// Normally there should be no more than one peerstate per address.
-/// However, the database does not enforce this condition.
-///
-/// Previously there were bugs that caused creation of additional
-/// peerstates when existing peerstate could not be read due to a
-/// temporary database error or a failure to parse stored data.  This
-/// procedure fixes the problem by removing duplicate records.
-pub(crate) async fn deduplicate_peerstates(sql: &Sql) -> Result<()> {
-    sql.execute(
-        "DELETE FROM acpeerstates
-         WHERE id NOT IN (
-         SELECT MIN(id)
-         FROM acpeerstates
-         GROUP BY addr
-         )",
-        paramsv![],
-    )
-    .await?;
-
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test_utils::alice_keypair;
+    use pretty_assertions::assert_eq;
 
     #[async_std::test]
     async fn test_peerstate_save_to_db() {

src/pgp.rs

@@ -4,11 +4,11 @@ use std::collections::{BTreeMap, HashSet};
 use std::io;
 use std::io::Cursor;
 
-use anyhow::{bail, ensure, format_err, Context as _, Result};
+use anyhow::{bail, ensure, format_err, Result};
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
-    SignedPublicSubKey, SignedSecretKey, StandaloneSignature, SubkeyParamsBuilder,
+    SignedPublicSubKey, SignedSecretKey, SubkeyParamsBuilder,
 };
 use pgp::crypto::{HashAlgorithm, SymmetricKeyAlgorithm};
 use pgp::types::{
@@ -248,7 +248,7 @@ pub async fn pk_encrypt(
         let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
             .keys()
             .iter()
-            .filter_map(select_pk_for_encryption)
+            .filter_map(|key| select_pk_for_encryption(key))
             .collect();
         let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
@@ -277,17 +277,16 @@ pub async fn pk_encrypt(
 /// Receiver private keys are provided in
 /// `private_keys_for_decryption`.
 ///
-/// Returns decrypted message and fingerprints
+/// If `ret_signature_fingerprints` is not `None`, stores fingerprints
 /// of all keys from the `public_keys_for_validation` keyring that
 /// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
 pub async fn pk_decrypt(
     ctext: Vec<u8>,
     private_keys_for_decryption: Keyring<SignedSecretKey>,
-    public_keys_for_validation: &Keyring<SignedPublicKey>,
-) -> Result<(Vec<u8>, HashSet<Fingerprint>)> {
-    let mut ret_signature_fingerprints: HashSet<Fingerprint> = Default::default();
-
+    public_keys_for_validation: Keyring<SignedPublicKey>,
+    ret_signature_fingerprints: Option<&mut HashSet<Fingerprint>>,
+) -> Result<Vec<u8>> {
     let msgs = async_std::task::spawn_blocking(move || {
         let cursor = Cursor::new(ctext);
         let (msg, _) = Message::from_armor_single(cursor)?;
@@ -309,54 +308,33 @@ pub async fn pk_decrypt(
             None => bail!("The decrypted message is empty"),
         };
 
-        if !public_keys_for_validation.is_empty() {
-            let pkeys = public_keys_for_validation.keys();
+        if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
+            if !public_keys_for_validation.is_empty() {
+                let fingerprints = async_std::task::spawn_blocking(move || {
+                    let pkeys = public_keys_for_validation.keys();
 
-            let mut fingerprints: Vec<Fingerprint> = Vec::new();
-            if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
-                for pkey in pkeys {
-                    if signed_msg.verify(&pkey.primary_key).is_ok() {
-                        let fp = DcKey::fingerprint(pkey);
-                        fingerprints.push(fp);
+                    let mut fingerprints: Vec<Fingerprint> = Vec::new();
+                    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+                        for pkey in pkeys {
+                            if signed_msg.verify(&pkey.primary_key).is_ok() {
+                                let fp = DcKey::fingerprint(pkey);
+                                fingerprints.push(fp);
+                            }
+                        }
                     }
-                }
-            }
+                    fingerprints
+                })
+                .await;
 
-            ret_signature_fingerprints.extend(fingerprints);
+                ret_signature_fingerprints.extend(fingerprints);
+            }
         }
-        Ok((content, ret_signature_fingerprints))
+        Ok(content)
     } else {
         bail!("No valid messages found");
     }
 }
 
-/// Validates detached signature.
-pub async fn pk_validate(
-    content: &[u8],
-    signature: &[u8],
-    public_keys_for_validation: &Keyring<SignedPublicKey>,
-) -> Result<HashSet<Fingerprint>> {
-    let mut ret: HashSet<Fingerprint> = Default::default();
-
-    let standalone_signature = StandaloneSignature::from_armor_single(Cursor::new(signature))?.0;
-    let pkeys = public_keys_for_validation.keys();
-
-    // Remove trailing CRLF before the delimiter.
-    // According to RFC 3156 it is considered to be part of the MIME delimiter for the purpose of
-    // OpenPGP signature calculation.
-    let content = content
-        .get(..content.len().saturating_sub(2))
-        .context("index is out of range")?;
-
-    for pkey in pkeys {
-        if standalone_signature.verify(pkey, content).is_ok() {
-            let fp = DcKey::fingerprint(pkey);
-            ret.insert(fp);
-        }
-    }
-    Ok(ret)
-}
-
 /// Symmetric encryption.
 pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
@@ -514,12 +492,15 @@ mod tests {
         decrypt_keyring.add(KEYS.alice_secret.clone());
         let mut sig_check_keyring: Keyring<SignedPublicKey> = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let (plain, valid_signatures) = pk_decrypt(
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
+        let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            &sig_check_keyring,
+            sig_check_keyring,
+            Some(&mut valid_signatures),
         )
         .await
+        .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
@@ -529,12 +510,15 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let (plain, valid_signatures) = pk_decrypt(
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
+        let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            &sig_check_keyring,
+            sig_check_keyring,
+            Some(&mut valid_signatures),
         )
         .await
+        .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
@@ -545,10 +529,15 @@ mod tests {
         let mut keyring = Keyring::new();
         keyring.add(KEYS.alice_secret.clone());
         let empty_keyring = Keyring::new();
-        let (plain, valid_signatures) =
-            pk_decrypt(CTEXT_SIGNED.as_bytes().to_vec(), keyring, &empty_keyring)
-                .await
-                .unwrap();
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
+        let plain = pk_decrypt(
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            keyring,
+            empty_keyring,
+            Some(&mut valid_signatures),
+        )
+        .await
+        .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
@@ -560,10 +549,12 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.bob_public.clone());
-        let (plain, valid_signatures) = pk_decrypt(
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
+        let plain = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            &sig_check_keyring,
+            sig_check_keyring,
+            Some(&mut valid_signatures),
         )
         .await
         .unwrap();
@@ -576,14 +567,34 @@ mod tests {
         let mut decrypt_keyring = Keyring::new();
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let sig_check_keyring = Keyring::new();
-        let (plain, valid_signatures) = pk_decrypt(
+        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
+        let plain = pk_decrypt(
             CTEXT_UNSIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            &sig_check_keyring,
+            sig_check_keyring,
+            Some(&mut valid_signatures),
         )
         .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
+
+    #[async_std::test]
+    async fn test_decrypt_signed_no_sigret() {
+        // Check decrypting signed cyphertext without providing the HashSet for signatures.
+        let mut decrypt_keyring = Keyring::new();
+        decrypt_keyring.add(KEYS.bob_secret.clone());
+        let mut sig_check_keyring = Keyring::new();
+        sig_check_keyring.add(KEYS.alice_public.clone());
+        let plain = pk_decrypt(
+            CTEXT_SIGNED.as_bytes().to_vec(),
+            decrypt_keyring,
+            sig_check_keyring,
+            None,
+        )
+        .await
+        .unwrap();
+        assert_eq!(plain, CLEARTEXT);
+    }
 }

src/provider.rs

@@ -3,10 +3,8 @@
 mod data;
 
 use crate::config::Config;
-use crate::context::Context;
 use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED};
-use anyhow::Result;
-use async_std_resolver::{config, resolver, resolver_from_system_conf, AsyncStdResolver};
+use async_std_resolver::{config, resolver};
 use chrono::{NaiveDateTime, NaiveTime};
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
@@ -83,23 +81,6 @@ pub struct Provider {
     pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
-/// Get resolver to query MX records.
-///
-/// We first try resolver_from_system_conf() which reads the system's resolver from `/etc/resolv.conf`.
-/// This does not work at least on some Androids, therefore we use use ResolverConfig::default()
-/// which default eg. to google's 8.8.8.8 or 8.8.4.4 as a fallback.
-async fn get_resolver() -> Result<AsyncStdResolver> {
-    if let Ok(resolver) = resolver_from_system_conf().await {
-        return Ok(resolver);
-    }
-    let resolver = resolver(
-        config::ResolverConfig::default(),
-        config::ResolverOpts::default(),
-    )
-    .await?;
-    Ok(resolver)
-}
-
 /// Returns provider for the given domain.
 ///
 /// This function looks up domain in offline database first. If not
@@ -108,11 +89,7 @@ async fn get_resolver() -> Result<AsyncStdResolver> {
 ///
 /// For compatibility, email address can be passed to this function
 /// instead of the domain.
-pub async fn get_provider_info(
-    context: &Context,
-    domain: &str,
-    skip_mx: bool,
-) -> Option<&'static Provider> {
+pub async fn get_provider_info(domain: &str, skip_mx: bool) -> Option<&'static Provider> {
     let domain = domain.rsplitn(2, '@').next()?;
 
     if let Some(provider) = get_provider_by_domain(domain) {
@@ -120,7 +97,7 @@ pub async fn get_provider_info(
     }
 
     if !skip_mx {
-        if let Some(provider) = get_provider_by_mx(context, domain).await {
+        if let Some(provider) = get_provider_by_mx(domain).await {
             return Some(provider);
         }
     }
@@ -140,8 +117,13 @@ pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
 /// Finds a provider based on MX record for the given domain.
 ///
 /// For security reasons, only Gmail can be configured this way.
-pub async fn get_provider_by_mx(context: &Context, domain: &str) -> Option<&'static Provider> {
-    if let Ok(resolver) = get_resolver().await {
+pub async fn get_provider_by_mx(domain: &str) -> Option<&'static Provider> {
+    if let Ok(resolver) = resolver(
+        config::ResolverConfig::default(),
+        config::ResolverOpts::default(),
+    )
+    .await
+    {
         let mut fqdn: String = domain.to_string();
         if !fqdn.ends_with('.') {
             fqdn.push('.');
@@ -166,8 +148,6 @@ pub async fn get_provider_by_mx(context: &Context, domain: &str) -> Option<&'sta
                 }
             }
         }
-    } else {
-        warn!(context, "cannot get a resolver to check MX records.");
     }
 
     None
@@ -194,7 +174,6 @@ mod tests {
 
     use super::*;
     use crate::dc_tools::time;
-    use crate::test_utils::TestContext;
     use chrono::NaiveDate;
 
     #[test]
@@ -244,13 +223,12 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_provider_info() {
-        let t = TestContext::new().await;
-        assert!(get_provider_info(&t, "", false).await.is_none());
-        assert!(get_provider_info(&t, "google.com", false).await.unwrap().id == "gmail");
+        assert!(get_provider_info("", false).await.is_none());
+        assert!(get_provider_info("google.com", false).await.unwrap().id == "gmail");
 
         // get_provider_info() accepts email addresses for backwards compatibility
         assert!(
-            get_provider_info(&t, "example@google.com", false)
+            get_provider_info("example@google.com", false)
                 .await
                 .unwrap()
                 .id
@@ -269,10 +247,4 @@ mod tests {
         assert!(get_provider_update_timestamp() <= time());
         assert!(get_provider_update_timestamp() > timestamp_past);
     }
-
-    #[async_std::test]
-    async fn test_get_resolver() -> Result<()> {
-        assert!(get_resolver().await.is_ok());
-        Ok(())
-    }
 }

src/provider/data.rs

@@ -131,35 +131,6 @@ static P_AUTISTICI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// blindzeln.org.md: delta.blinzeln.de, delta.blindzeln.org
-static P_BLINDZELN_ORG: Lazy<Provider> = Lazy::new(|| Provider {
-    id: "blindzeln.org",
-    status: Status::Ok,
-    before_login_hint: "",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/blindzeln-org",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "webbox222.server-home.org",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "webbox222.server-home.org",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-});
-
 // bluewin.ch.md: bluewin.ch
 static P_BLUEWIN_CH: Lazy<Provider> = Lazy::new(|| Provider {
     id: "bluewin.ch",
@@ -282,23 +253,61 @@ static P_DISROOT: Lazy<Provider> = Lazy::new(|| Provider {
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/disroot",
+    server: vec![],
+    config_defaults: None,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
+    oauth2_authorizer: None,
+});
+
+// dubby.org.md: dubby.org
+static P_DUBBY_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "dubby.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/dubby-org",
     server: vec![
         Server {
             protocol: Imap,
             socket: Ssl,
-            hostname: "disroot.org",
+            hostname: "dubby.org",
             port: 993,
             username_pattern: Email,
         },
         Server {
             protocol: Smtp,
             socket: Starttls,
-            hostname: "disroot.org",
+            hostname: "dubby.org",
             port: 587,
             username_pattern: Email,
         },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "dubby.org",
+            port: 465,
+            username_pattern: Email,
+        },
     ],
-    config_defaults: None,
+    config_defaults: Some(vec![
+        ConfigDefault {
+            key: Config::BccSelf,
+            value: "1",
+        },
+        ConfigDefault {
+            key: Config::SentboxWatch,
+            value: "0",
+        },
+        ConfigDefault {
+            key: Config::MvboxWatch,
+            value: "0",
+        },
+        ConfigDefault {
+            key: Config::MvboxMove,
+            value: "0",
+        },
+    ]),
     strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
@@ -437,6 +446,10 @@ static P_FIVE_CHAT: Lazy<Provider> = Lazy::new(|| Provider {
             key: Config::SentboxWatch,
             value: "0",
         },
+        ConfigDefault {
+            key: Config::MvboxWatch,
+            value: "0",
+        },
         ConfigDefault {
             key: Config::MvboxMove,
             value: "0",
@@ -626,35 +639,6 @@ static P_ICLOUD: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// infomaniak.com.md: ik.me
-static P_INFOMANIAK_COM: Lazy<Provider> = Lazy::new(|| Provider {
-    id: "infomaniak.com",
-    status: Status::Ok,
-    before_login_hint: "",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/infomaniak-com",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "mail.infomaniak.com",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "mail.infomaniak.com",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: Some(10),
-    oauth2_authorizer: None,
-});
-
 // kolst.com.md: kolst.com
 static P_KOLST_COM: Lazy<Provider> = Lazy::new(|| Provider {
     id: "kolst.com",
@@ -683,35 +667,6 @@ static P_KONTENT_COM: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// mail.de.md: mail.de
-static P_MAIL_DE: Lazy<Provider> = Lazy::new(|| Provider {
-    id: "mail.de",
-    status: Status::Ok,
-    before_login_hint: "",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/mail-de",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "imap.mail.de",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "smtp.mail.de",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-});
-
 // mail.ru.md: mail.ru, inbox.ru, internet.ru, bk.ru, list.ru
 static P_MAIL_RU: Lazy<Provider> = Lazy::new(|| {
     Provider {
@@ -767,22 +722,7 @@ static P_MAILBOX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/mailbox-org",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "imap.mailbox.org",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "smtp.mailbox.org",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
+    server: vec![],
     config_defaults: None,
     strict_tls: true,
     max_smtp_rcpt_to: None,
@@ -854,6 +794,10 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| Provider {
             key: Config::SentboxWatch,
             value: "0",
         },
+        ConfigDefault {
+            key: Config::MvboxWatch,
+            value: "0",
+        },
         ConfigDefault {
             key: Config::MvboxMove,
             value: "0",
@@ -934,7 +878,7 @@ static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ca, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
+// posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
 static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
     id: "posteo",
     status: Status::Ok,
@@ -963,7 +907,7 @@ static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// protonmail.md: protonmail.com, protonmail.ch, pm.me
+// protonmail.md: protonmail.com, protonmail.ch
 static P_PROTONMAIL: Lazy<Provider> = Lazy::new(|| {
     Provider {
     id: "protonmail",
@@ -1006,22 +950,7 @@ static P_RISEUP_NET: Lazy<Provider> = Lazy::new(|| Provider {
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/riseup-net",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "mail.riseup.net",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "mail.riseup.net",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
+    server: vec![],
     config_defaults: None,
     strict_tls: true,
     max_smtp_rcpt_to: None,
@@ -1078,22 +1007,7 @@ static P_SYSTEMLI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/systemli-org",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "mail.systemli.org",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "mail.systemli.org",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
+    server: vec![],
     config_defaults: None,
     strict_tls: true,
     max_smtp_rcpt_to: None,
@@ -1158,6 +1072,10 @@ static P_TESTRUN: Lazy<Provider> = Lazy::new(|| Provider {
             key: Config::SentboxWatch,
             value: "0",
         },
+        ConfigDefault {
+            key: Config::MvboxWatch,
+            value: "0",
+        },
         ConfigDefault {
             key: Config::MvboxMove,
             value: "0",
@@ -1445,7 +1363,7 @@ static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// zoho.md: zohomail.eu, zohomail.com, zoho.com
+// zoho.md: zohomail.eu, zoho.com
 static P_ZOHO: Lazy<Provider> = Lazy::new(|| Provider {
     id: "zoho",
     status: Status::Preparation,
@@ -1481,8 +1399,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("aol.com", &*P_AOL),
         ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
-        ("delta.blinzeln.de", &*P_BLINDZELN_ORG),
-        ("delta.blindzeln.org", &*P_BLINDZELN_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
         ("buzon.uy", &*P_BUZON_UY),
         ("chello.at", &*P_CHELLO_AT),
@@ -1490,6 +1406,7 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("comcast.net", &*P_COMCAST),
         ("dismail.de", &*P_DISMAIL_DE),
         ("disroot.org", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
         ("e.email", &*P_E_EMAIL),
         ("espiv.net", &*P_ESPIV_NET),
         ("example.com", &*P_EXAMPLE_COM),
@@ -1519,10 +1436,8 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
-        ("ik.me", &*P_INFOMANIAK_COM),
         ("kolst.com", &*P_KOLST_COM),
         ("kontent.com", &*P_KONTENT_COM),
-        ("mail.de", &*P_MAIL_DE),
         ("mail.ru", &*P_MAIL_RU),
         ("inbox.ru", &*P_MAIL_RU),
         ("internet.ru", &*P_MAIL_RU),
@@ -1543,7 +1458,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("posteo.af", &*P_POSTEO),
         ("posteo.at", &*P_POSTEO),
         ("posteo.be", &*P_POSTEO),
-        ("posteo.ca", &*P_POSTEO),
         ("posteo.ch", &*P_POSTEO),
         ("posteo.cl", &*P_POSTEO),
         ("posteo.co", &*P_POSTEO),
@@ -1592,7 +1506,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
-        ("pm.me", &*P_PROTONMAIL),
         ("qq.com", &*P_QQ),
         ("foxmail.com", &*P_QQ),
         ("riseup.net", &*P_RISEUP_NET),
@@ -1667,7 +1580,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("yggmail", &*P_YGGMAIL),
         ("ziggo.nl", &*P_ZIGGO_NL),
         ("zohomail.eu", &*P_ZOHO),
-        ("zohomail.com", &*P_ZOHO),
         ("zoho.com", &*P_ZOHO),
     ]
     .iter()
@@ -1682,13 +1594,13 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("aol", &*P_AOL),
         ("arcor.de", &*P_ARCOR_DE),
         ("autistici.org", &*P_AUTISTICI_ORG),
-        ("blindzeln.org", &*P_BLINDZELN_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
         ("buzon.uy", &*P_BUZON_UY),
         ("chello.at", &*P_CHELLO_AT),
         ("comcast", &*P_COMCAST),
         ("dismail.de", &*P_DISMAIL_DE),
         ("disroot", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
         ("e.email", &*P_E_EMAIL),
         ("espiv.net", &*P_ESPIV_NET),
         ("example.com", &*P_EXAMPLE_COM),
@@ -1703,10 +1615,8 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("i.ua", &*P_I_UA),
         ("i3.net", &*P_I3_NET),
         ("icloud", &*P_ICLOUD),
-        ("infomaniak.com", &*P_INFOMANIAK_COM),
         ("kolst.com", &*P_KOLST_COM),
         ("kontent.com", &*P_KONTENT_COM),
-        ("mail.de", &*P_MAIL_DE),
         ("mail.ru", &*P_MAIL_RU),
         ("mail2tor", &*P_MAIL2TOR),
         ("mailbox.org", &*P_MAILBOX_ORG),
@@ -1743,4 +1653,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd(2022, 1, 11));
+    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 8, 17));

src/qr_code_generator.rs

@@ -1,265 +0,0 @@
-use anyhow::Result;
-use qrcodegen::{QrCode, QrCodeEcc};
-
-use crate::{
-    blob::BlobObject,
-    chat::{Chat, ChatId},
-    color::color_int_to_hex_string,
-    config::Config,
-    constants::DC_CONTACT_ID_SELF,
-    contact::Contact,
-    context::Context,
-    securejoin, stock_str,
-};
-
-pub async fn get_securejoin_qr_svg(context: &Context, chat_id: Option<ChatId>) -> Result<String> {
-    if let Some(chat_id) = chat_id {
-        generate_join_group_qr_code(context, chat_id).await
-    } else {
-        generate_verification_qr(context).await
-    }
-}
-
-async fn generate_join_group_qr_code(context: &Context, chat_id: ChatId) -> Result<String> {
-    let chat = Chat::load_from_db(context, chat_id).await?;
-
-    let avatar = match chat.get_profile_image(context).await? {
-        Some(path) => {
-            let avatar_blob = BlobObject::from_path(context, &path)?;
-            Some(std::fs::read(avatar_blob.to_abs_path())?)
-        }
-        None => None,
-    };
-
-    inner_generate_secure_join_qr_code(
-        &stock_str::secure_join_group_qr_description(context, &chat).await,
-        &securejoin::dc_get_securejoin_qr(context, Some(chat_id)).await?,
-        &color_int_to_hex_string(chat.get_color(context).await?),
-        avatar,
-        chat.get_name().chars().next().unwrap_or('#'),
-    )
-}
-
-async fn generate_verification_qr(context: &Context) -> Result<String> {
-    let contact = Contact::get_by_id(context, DC_CONTACT_ID_SELF).await?;
-
-    let avatar = match contact.get_profile_image(context).await? {
-        Some(path) => {
-            let avatar_blob = BlobObject::from_path(context, &path)?;
-            Some(std::fs::read(avatar_blob.to_abs_path())?)
-        }
-        None => None,
-    };
-
-    let displayname = match context.get_config(Config::Displayname).await? {
-        Some(name) => name,
-        None => contact.get_addr().to_owned(),
-    };
-
-    inner_generate_secure_join_qr_code(
-        &stock_str::setup_contact_qr_description(context, &displayname, contact.get_addr()).await,
-        &securejoin::dc_get_securejoin_qr(context, None).await?,
-        &color_int_to_hex_string(contact.get_color()),
-        avatar,
-        displayname.chars().next().unwrap_or('#'),
-    )
-}
-
-fn inner_generate_secure_join_qr_code(
-    raw_qrcode_description: &str,
-    qrcode_content: &str,
-    color: &str,
-    avatar: Option<Vec<u8>>,
-    avatar_letter: char,
-) -> Result<String> {
-    let qrcode_description = &escaper::encode_minimal(raw_qrcode_description);
-    // config
-    let width = 515.0;
-    let height = 630.0;
-    let logo_offset = 28.0;
-    let qr_code_size = 400.0;
-    let qr_translate_up = 40.0;
-    let text_y_pos = ((height - qr_code_size) / 2.0) + qr_code_size;
-    let avatar_border_size = 9.0;
-    let card_border_size = 2.0;
-    let card_roundness = 40.0;
-
-    let qr = QrCode::encode_text(qrcode_content, QrCodeEcc::Medium)?;
-    let mut svg = String::with_capacity(28000);
-    let mut w = tagger::new(&mut svg);
-
-    w.elem("svg", |d| {
-        d.attr("xmlns", "http://www.w3.org/2000/svg")?;
-        d.attr("viewBox", format_args!("0 0 {} {}", width, height))?;
-        Ok(())
-    })?
-    .build(|w| {
-        // White Background apears like a card
-        w.single("rect", |d| {
-            d.attr("x", card_border_size)?;
-            d.attr("y", card_border_size)?;
-            d.attr("rx", card_roundness)?;
-            d.attr("stroke", "#c6c6c6")?;
-            d.attr("stroke-width", card_border_size)?;
-            d.attr("width", width - (card_border_size * 2.0))?;
-            d.attr("height", height - (card_border_size * 2.0))?;
-            d.attr("style", "fill:#f2f2f2")?;
-            Ok(())
-        })?;
-        // Qrcode
-        w.elem("g", |d| {
-            d.attr(
-                "transform",
-                format!(
-                    "translate({},{})",
-                    (width - qr_code_size) / 2.0,
-                    ((height - qr_code_size) / 2.0) - qr_translate_up
-                ),
-            )
-            // If the qr code should be in the wrong place,
-            // we could also translate and scale the points in the path already,
-            // but that would make the resulting svg way bigger in size and might bring up rounding issues,
-            // so better avoid doing it manually if possible
-        })?
-        .build(|w| {
-            w.single("path", |d| {
-                let mut path_data = String::with_capacity(0);
-                let scale = qr_code_size / qr.size() as f32;
-
-                for y in 0..qr.size() {
-                    for x in 0..qr.size() {
-                        if qr.get_module(x, y) {
-                            path_data += &format!("M{},{}h1v1h-1z", x, y);
-                        }
-                    }
-                }
-
-                d.attr("style", "fill:#000000")?;
-                d.attr("d", path_data)?;
-                d.attr("transform", format!("scale({})", scale))
-            })
-        })?;
-
-        // Text
-        const BIG_TEXT_CHARS_PER_LINE: usize = 32;
-        const SMALL_TEXT_CHARS_PER_LINE: usize = 38;
-        let chars_per_line = if qrcode_description.len() > SMALL_TEXT_CHARS_PER_LINE * 2 {
-            SMALL_TEXT_CHARS_PER_LINE
-        } else {
-            BIG_TEXT_CHARS_PER_LINE
-        };
-        let lines = textwrap::fill(qrcode_description, chars_per_line);
-        let (text_font_size, text_y_shift) = if lines.split('\n').count() <= 2 {
-            (27.0, 0.0)
-        } else {
-            (19.0, -10.0)
-        };
-        for (count, line) in lines.split('\n').enumerate() {
-            w.elem("text", |d| {
-                d.attr(
-                    "y",
-                    (count as f32 * (text_font_size * 1.2)) + text_y_pos + text_y_shift,
-                )?;
-                d.attr("x", width / 2.0)?;
-                d.attr("text-anchor", "middle")?;
-                d.attr(
-                    "style",
-                    format!(
-                        "font-family:sans-serif;\
-                        font-weight:bold;\
-                        font-size:{}px;\
-                        fill:#000000;\
-                        stroke:none",
-                        text_font_size
-                    ),
-                )
-            })?
-            .build(|w| w.put_raw(line))?;
-        }
-        // contact avatar in middle of qrcode
-        const LOGO_SIZE: f32 = 94.4;
-        const HALF_LOGO_SIZE: f32 = LOGO_SIZE / 2.0;
-        let logo_position_in_qr = (qr_code_size / 2.0) - HALF_LOGO_SIZE;
-        let logo_position_x = ((width - qr_code_size) / 2.0) + logo_position_in_qr;
-        let logo_position_y =
-            ((height - qr_code_size) / 2.0) - qr_translate_up + logo_position_in_qr;
-
-        w.single("circle", |d| {
-            d.attr("cx", logo_position_x + HALF_LOGO_SIZE)?;
-            d.attr("cy", logo_position_y + HALF_LOGO_SIZE)?;
-            d.attr("r", HALF_LOGO_SIZE + avatar_border_size)?;
-            d.attr("style", "fill:#f2f2f2")
-        })?;
-
-        if let Some(img) = avatar {
-            w.elem("defs", tagger::no_attr())?.build(|w| {
-                w.elem("clipPath", |d| d.attr("id", "avatar-cut"))?
-                    .build(|w| {
-                        w.single("circle", |d| {
-                            d.attr("cx", logo_position_x + HALF_LOGO_SIZE)?;
-                            d.attr("cy", logo_position_y + HALF_LOGO_SIZE)?;
-                            d.attr("r", HALF_LOGO_SIZE)
-                        })
-                    })
-            })?;
-
-            w.single("image", |d| {
-                d.attr("x", logo_position_x)?;
-                d.attr("y", logo_position_y)?;
-                d.attr("width", HALF_LOGO_SIZE * 2.0)?;
-                d.attr("height", HALF_LOGO_SIZE * 2.0)?;
-                d.attr("preserveAspectRatio", "none")?;
-                d.attr("clip-path", "url(#avatar-cut)")?;
-                d.attr(
-                    "href", /*might need xlink:href instead if it doesn't work on older devices?*/
-                    format!("data:image/jpeg;base64,{}", base64::encode(img)),
-                )
-            })?;
-        } else {
-            w.single("circle", |d| {
-                d.attr("cx", logo_position_x + HALF_LOGO_SIZE)?;
-                d.attr("cy", logo_position_y + HALF_LOGO_SIZE)?;
-                d.attr("r", HALF_LOGO_SIZE)?;
-                d.attr("style", format!("fill:{}", &color))
-            })?;
-
-            let avatar_font_size = LOGO_SIZE * 0.65;
-            let font_offset = avatar_font_size * 0.1;
-            w.elem("text", |d| {
-                d.attr("y", logo_position_y + HALF_LOGO_SIZE + font_offset)?;
-                d.attr("x", logo_position_x + HALF_LOGO_SIZE)?;
-                d.attr("text-anchor", "middle")?;
-                d.attr("dominant-baseline", "central")?;
-                d.attr("alignment-baseline", "middle")?;
-                d.attr(
-                    "style",
-                    format!(
-                        "font-family:sans-serif;\
-                            font-weight:400;\
-                            font-size:{}px;\
-                            fill:#ffffff;",
-                        avatar_font_size
-                    ),
-                )
-            })?
-            .build(|w| w.put_raw(avatar_letter.to_uppercase()))?;
-        }
-
-        // Footer logo
-        const FOOTER_HEIGHT: f32 = 35.0;
-        const FOOTER_WIDTH: f32 = 198.0;
-        w.elem("g", |d| {
-            d.attr(
-                "transform",
-                format!(
-                    "translate({},{})",
-                    (width - FOOTER_WIDTH) / 2.0,
-                    height - logo_offset - FOOTER_HEIGHT - text_y_shift
-                ),
-            )
-        })?
-        .build(|w| w.put_raw(include_str!("../assets/qrcode_logo_footer.svg")))
-    })?;
-
-    Ok(svg)
-}

src/quota.rs

@@ -1,8 +1,8 @@
 //! # Support for IMAP QUOTA extension.
 
-use anyhow::{anyhow, Context as _, Result};
+use anyhow::{anyhow, Result};
 use async_imap::types::{Quota, QuotaResource};
-use std::collections::BTreeMap;
+use indexmap::IndexMap;
 
 use crate::chat::add_device_msg_with_importance;
 use crate::config::Config;
@@ -20,17 +20,14 @@ use crate::{job, stock_str, EventType};
 /// quota icon is "yellow".
 pub const QUOTA_WARN_THRESHOLD_PERCENTAGE: u64 = 80;
 
-// warning again after this usage percentage is reached,
-// quota icon is "red".
-pub const QUOTA_ERROR_THRESHOLD_PERCENTAGE: u64 = 95;
+// warning is already issued at QUOTA_WARN_THRESHOLD_PERCENTAGE,
+// this threshold only makes the quota icon "red".
+pub const QUOTA_ERROR_THRESHOLD_PERCENTAGE: u64 = 99;
 
 /// if quota is below this value (again),
 /// QuotaExceeding is cleared.
 /// This value should be a bit below QUOTA_WARN_THRESHOLD_PERCENTAGE to
 /// avoid jittering and lots of warnings when quota is exactly at the warning threshold.
-///
-/// We do not repeat warnings on a daily base or so as some provider
-/// providers report bad values and we would then spam the user.
 pub const QUOTA_ALLCLEAR_PERCENTAGE: u64 = 75;
 
 // if recent quota is older,
@@ -43,7 +40,7 @@ pub struct QuotaInfo {
     /// set to `Err()` if the provider does not support quota or on other errors,
     /// set to `Ok()` for valid quota information.
     /// Updated by `Action::UpdateRecentQuota`
-    pub(crate) recent: Result<BTreeMap<String, Vec<QuotaResource>>>,
+    pub(crate) recent: Result<IndexMap<String, Vec<QuotaResource>>>,
 
     /// Timestamp when structure was modified.
     pub(crate) modified: i64,
@@ -52,8 +49,8 @@ pub struct QuotaInfo {
 async fn get_unique_quota_roots_and_usage(
     folders: Vec<String>,
     imap: &mut Imap,
-) -> Result<BTreeMap<String, Vec<QuotaResource>>> {
-    let mut unique_quota_roots: BTreeMap<String, Vec<QuotaResource>> = BTreeMap::new();
+) -> Result<IndexMap<String, Vec<QuotaResource>>> {
+    let mut unique_quota_roots: IndexMap<String, Vec<QuotaResource>> = IndexMap::new();
     for folder in folders {
         let (quota_roots, quotas) = &imap.get_quota_roots(&folder).await?;
         // if there are new quota roots found in this imap folder, add them to the list
@@ -64,12 +61,12 @@ async fn get_unique_quota_roots_and_usage(
                     .iter()
                     .find(|q| &q.root_name == quota_root_name)
                     .cloned()
-                    .context("quota_root should have a quota")?;
+                    .ok_or_else(|| anyhow!("quota_root should have a quota"))?;
                 // replace old quotas, because between fetching quotaroots for folders,
                 // messages could be recieved and so the usage could have been changed
                 *unique_quota_roots
                     .entry(quota_root_name.clone())
-                    .or_insert_with(Vec::new) = quota.resources;
+                    .or_insert(vec![]) = quota.resources;
             }
         }
     }
@@ -77,7 +74,7 @@ async fn get_unique_quota_roots_and_usage(
 }
 
 fn get_highest_usage<'t>(
-    unique_quota_roots: &'t BTreeMap<String, Vec<QuotaResource>>,
+    unique_quota_roots: &'t IndexMap<String, Vec<QuotaResource>>,
 ) -> Result<(u64, &'t String, &QuotaResource)> {
     let mut highest: Option<(u64, &'t String, &QuotaResource)> = None;
     for (name, resources) in unique_quota_roots {
@@ -96,27 +93,18 @@ fn get_highest_usage<'t>(
         }
     }
 
-    highest.context("no quota_resource found, this is unexpected")
-}
-
-/// Checks if a quota warning is needed.
-pub fn needs_quota_warning(curr_percentage: u64, warned_at_percentage: u64) -> bool {
-    (curr_percentage >= QUOTA_WARN_THRESHOLD_PERCENTAGE
-        && warned_at_percentage < QUOTA_WARN_THRESHOLD_PERCENTAGE)
-        || (curr_percentage >= QUOTA_ERROR_THRESHOLD_PERCENTAGE
-            && warned_at_percentage < QUOTA_ERROR_THRESHOLD_PERCENTAGE)
+    highest.ok_or_else(|| anyhow!("no quota_resource found, this is unexpected"))
 }
 
 impl Context {
     // Adds a job to update `quota.recent`
     pub(crate) async fn schedule_quota_update(&self) -> Result<()> {
-        if !job::action_exists(self, Action::UpdateRecentQuota).await? {
-            job::add(
-                self,
-                job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0),
-            )
-            .await?;
-        }
+        job::kill_action(self, Action::UpdateRecentQuota).await?;
+        job::add(
+            self,
+            job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0),
+        )
+        .await;
         Ok(())
     }
 
@@ -137,24 +125,25 @@ impl Context {
         }
 
         let quota = if imap.can_check_quota() {
-            let folders = get_watched_folders(self).await?;
+            let folders = get_watched_folders(self).await;
             get_unique_quota_roots_and_usage(folders, imap).await
         } else {
-            Err(anyhow!(stock_str::not_supported_by_provider(self).await))
+            Err(anyhow!("Quota not supported by your provider."))
         };
 
         if let Ok(quota) = &quota {
             match get_highest_usage(quota) {
                 Ok((highest, _, _)) => {
-                    if needs_quota_warning(
-                        highest,
-                        self.get_config_int(Config::QuotaExceeding).await? as u64,
-                    ) {
-                        self.set_config(Config::QuotaExceeding, Some(&highest.to_string()))
-                            .await?;
-                        let mut msg = Message::new(Viewtype::Text);
-                        msg.text = Some(stock_str::quota_exceeding(self, highest).await);
-                        add_device_msg_with_importance(self, None, Some(&mut msg), true).await?;
+                    if highest >= QUOTA_WARN_THRESHOLD_PERCENTAGE {
+                        if self.get_config_int(Config::QuotaExceeding).await? == 0 {
+                            self.set_config(Config::QuotaExceeding, Some(&highest.to_string()))
+                                .await?;
+
+                            let mut msg = Message::new(Viewtype::Text);
+                            msg.text = Some(stock_str::quota_exceeding(self, highest).await);
+                            add_device_msg_with_importance(self, None, Some(&mut msg), true)
+                                .await?;
+                        }
                     } else if highest <= QUOTA_ALLCLEAR_PERCENTAGE {
                         self.set_config(Config::QuotaExceeding, None).await?;
                     }
@@ -175,30 +164,11 @@ impl Context {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use crate::quota::{
         QUOTA_ALLCLEAR_PERCENTAGE, QUOTA_ERROR_THRESHOLD_PERCENTAGE,
         QUOTA_WARN_THRESHOLD_PERCENTAGE,
     };
 
-    #[async_std::test]
-    async fn test_needs_quota_warning() -> Result<()> {
-        assert!(!needs_quota_warning(0, 0));
-        assert!(!needs_quota_warning(10, 0));
-        assert!(!needs_quota_warning(70, 0));
-        assert!(!needs_quota_warning(75, 0));
-        assert!(!needs_quota_warning(79, 0));
-        assert!(needs_quota_warning(80, 0));
-        assert!(needs_quota_warning(81, 0));
-        assert!(!needs_quota_warning(85, 80));
-        assert!(!needs_quota_warning(85, 81));
-        assert!(needs_quota_warning(95, 82));
-        assert!(!needs_quota_warning(97, 95));
-        assert!(!needs_quota_warning(97, 96));
-        assert!(!needs_quota_warning(1000, 96));
-        Ok(())
-    }
-
     #[allow(clippy::assertions_on_constants)]
     #[async_std::test]
     async fn test_quota_thresholds() -> anyhow::Result<()> {