prepare 1.79 (#3281)

* update changelog for 1.79.0

* bump version to 1.79.0

.github/dependabot.yml

@@ -3,7 +3,7 @@ updates:
   - package-ecosystem: "cargo"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "monthly"
     commit-message:
       prefix: "cargo"
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 50

.github/mergeable.yml

@@ -0,0 +1,26 @@
+version: 2
+mergeable:
+  - when: pull_request.*
+    name: "Changelog check"
+    validate:
+      - do: or
+        validate:
+        - do: description
+          must_include:
+            regex: '#skip-changelog'
+        - do: and
+          validate:
+            - do: dependent
+              changed:
+                file: 'src/**'
+                required: ['CHANGELOG.md']
+            - do: dependent
+              changed:
+                file: 'deltachat-ffi/**'
+                required: ['CHANGELOG.md']
+    fail:
+      - do: checks
+        status: 'action_required' 
+        payload:
+          title: Changlog might need an update
+          summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -8,6 +8,9 @@ on:
       - staging
       - trying
 
+env:
+  RUSTFLAGS: -Dwarnings
+
 jobs:
 
   fmt:
@@ -21,6 +24,8 @@ jobs:
           toolchain: stable
           override: true
       - run: rustup component add rustfmt
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v1
       - uses: actions-rs/cargo@v1
         with:
           command: fmt
@@ -35,10 +40,12 @@ jobs:
             toolchain: stable
             components: clippy
             override: true
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v1
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples
+          args: --workspace --tests --examples --benches
 
   docs:
     name: Rust doc comments
@@ -70,19 +77,19 @@ jobs:
         include:
           # Currently used Rust version, same as in `rust-toolchain` file.
           - os: ubuntu-latest
-            rust: 1.54.0
+            rust: 1.60.0
             python: 3.9
           - os: windows-latest
-            rust: 1.54.0
+            rust: 1.60.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.51.0
+          # Minimum Supported Rust Version = 1.56.0
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.51.0
+            rust: 1.56.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:
@@ -94,31 +101,14 @@ jobs:
         toolchain: ${{ matrix.rust }}
         override: true
 
-    - name: Cache cargo registry
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/registry
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo index
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/git
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo build
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+    - name: Cache rust cargo artifacts
+      uses: swatinem/rust-cache@v1
 
     - name: check
       uses: actions-rs/cargo@v1
-      env:
-        RUSTFLAGS: -D warnings
       with:
         command: check
-        args: --all --bins --examples --tests --features repl
+        args: --all --bins --examples --tests --features repl --benches
 
     - name: tests
       uses: actions-rs/cargo@v1
@@ -150,4 +140,4 @@ jobs:
         DCC_RS_TARGET: debug
         DCC_RS_DEV: ${{ github.workspace }}
       working-directory: python
-      run: tox -e lint,doc,py3
+      run: tox -e lint,mypy,doc,py3

.github/workflows/dependabot.yml

@@ -0,0 +1,21 @@
+name: Dependabot auto-approve
+on: pull_request
+
+permissions:
+  pull-requests: write
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v1.1.1
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Approve a PR
+        run: gh pr review --approve "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/upload-docs.yml

@@ -0,0 +1,28 @@
+name: Build & Deploy Documentation on rs.delta.chat
+
+on:
+  push:
+    branches:
+      - master
+      - docs-gh-action
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions-rs/toolchain@v1
+    - name: Build the documentation with cargo
+      run: |
+        cargo doc --package deltachat --no-deps
+    - name: Upload to rs.delta.chat
+      uses: up9cloud/action-rsync@v1.3
+      env:
+        USER: ${{ secrets.USERNAME }}
+        KEY: ${{ secrets.KEY }}
+        HOST: "delta.chat"
+        SOURCE: "target/doc"
+        TARGET: "/var/www/html/rs/"
+

.github/workflows/upload-ffi-docs.yml

@@ -0,0 +1,28 @@
+name: Build & Deploy Documentation on cffi.delta.chat
+
+on:
+  push:
+    branches:
+      - master
+      - docs-gh-action
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions-rs/toolchain@v1
+    - name: Build the documentation with cargo
+      run: |
+        cargo doc --package deltachat_ffi --no-deps
+    - name: Upload to cffi.delta.chat
+      uses: up9cloud/action-rsync@v1.3
+      env:
+        USER: ${{ secrets.USERNAME }}
+        KEY: ${{ secrets.KEY }}
+        HOST: "delta.chat"
+        SOURCE: "target/doc"
+        TARGET: "/var/www/html/cffi/"
+

.gitignore

@@ -29,3 +29,7 @@ deltachat-ffi/xml
 
 coverage/
 .DS_Store
+.vscode/launch.json
+python/accounts.txt
+python/all-testaccounts.txt
+tmp/

CHANGELOG.md

@@ -1,5 +1,268 @@
 # Changelog
 
+## 1.79.0
+
+### Changes
+- Send locations in the background regardless of SMTP loop activity #3247
+- refactorings #3268
+- improve tests and ci #3266 #3271
+
+### Fixes
+- simplify `dc_stop_io()` and remove potential panics and race conditions #3273
+- fix correct message escaping consisting of a dot in SMTP protocol #3265
+
+
+## 1.78.0
+
+### API-Changes
+- replaced stock string `DC_STR_ONE_MOMENT` by `DC_STR_NOT_CONNECTED` #3222
+- add `dc_resend_msgs()` #3238
+- `dc_provider_new_from_email()` does no longer do an DNS lookup for checking custom domains,
+  this is done by `dc_provider_new_from_email_with_dns()` now #3256
+
+### Changes
+- introduce multiple self addresses with the "configured" address always being the primary one #2896
+- Further improve finding the correct server after logging in #3208
+- `get_connectivity_html()` returns HTML as non-scalable #3213
+- add update-serial to `DC_EVENT_WEBXDC_STATUS_UPDATE` #3215
+- Speed up message receiving via IMAP a bit #3225
+- mark messages as seen on IMAP in batches #3223
+- remove Received: based draft detection heuristic #3230
+- Use pkgconfig for building Python package #2590
+- don't start io on unconfigured context #2664
+- do not assign group IDs to ad-hoc groups #2798
+- dynamic libraries use dylib extension on Darwin #3226
+- refactorings #3217 #3219 #3224 #3235 #3239 #3244 #3254
+- improve documentation #3214 #3220 #3237
+- improve tests and ci #3212 #3233 #3241 #3242 #3252 #3250 #3255 #3260
+
+### Fixes
+- Take `delete_device_after` into account when calculating ephemeral loop timeout #3211 #3221
+- Fix a bug where a blocked contact could send a contact request #3218
+- Make sure, videochat-room-names are always URL-safe #3231
+- Try removing account folder multiple times in case of failure #3229
+- Ignore messages from all spam folders if there are many #3246
+- Hide location-only messages instead of displaying empty bubbles #3248
+
+
+## 1.77.0
+
+### API changes
+- change semantics of `dc_get_webxdc_status_updates()` second parameter
+  and remove update-id from `DC_EVENT_WEBXDC_STATUS_UPDATE` #3081
+
+### Changes
+- add more SMTP logging #3093
+- place common headers like `From:` before the large `Autocrypt:` header #3079
+- keep track of securejoin joiner status in database to survive restarts #2920
+- remove never used `SentboxMove` option #3111
+- improve speed by caching config values #3131 #3145
+- optimize `markseen_msgs` #3141
+- automatically accept chats with outgoing messages #3143
+- `dc_receive_imf` refactorings #3154 #3156 #3159
+- add index to speedup deletion of expired ephemeral messages #3155
+- muted chats stay archived on new messages #3184
+- support `min_api` from Webxdc manifests #3206
+- do not read whole webxdc file into memory #3109
+- improve tests, refactorings #3073 #3096 #3102 #3108 #3139 #3128 #3133 #3142 #3153 #3151 #3174 #3170 #3148 #3179 #3185
+- improve documentation #2983 #3112 #3103 #3118 #3120
+
+### Fixes
+- speed up loading of chat messages by a factor of 20 #3171 #3194 #3173
+- fix an issue where the app crashes when trying to export a backup #3195
+- hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
+- do not delete messages without Message-IDs as duplicates #3095
+- assign replies from a different email address to the correct chat #3119
+- assing outgoing private replies to the correct chat #3177
+- start ephemeral timer when seen status is synchronized via IMAP #3122
+- do not create empty contact requests with "setup changed" messages;
+  instead, send a "setup changed" message into all chats we share with the peer #3187
+- do not delete duplicate messages on IMAP immediately to accidentally deleting
+  the last copy #3138
+- clear more columns when message expires due to `delete_device_after` setting #3181
+- do not try to use stale SMTP connections #3180
+- slightly improve finding the correct server after logging in #3207
+- retry message sending automatically if loop is not interrupted #3183
+- fix a bug where sometimes the file extension of a long filename containing a dot was cropped #3098
+
+
+## 1.76.0
+
+### Changes
+- move messages in batches #3058
+- delete messages in batches #3060
+- python: remove arbitrary timeouts from tests #3059
+- refactorings #3026
+
+### Fixes
+- avoid archived, fresh chats #3053
+- Also resync UIDs in folders that are not configured #2289
+- treat "NO" IMAP response to MOVE and COPY commands as an error #3058
+- Fix a bug where messages in the Spam folder created contact requests #3015
+- Fix a bug where drafts disappeared after some days #3067
+- Parse MS Exchange read receipts and mark the original message as read #3075
+- do not retry message sending infinitely in case of permanent SMTP failure #3070
+- set message state to failed when retry limit is exceeded #3072
+
+
+## 1.75.0
+
+### Changes
+- optimize `delete_expired_imap_messages()` #3047
+
+
+## 1.74.0
+
+### Fixes
+- avoid reconnection loop when message without Message-ID is marked as seen #3044
+
+
+## 1.73.0
+
+### API changes
+- added `only_fetch_mvbox` config #3028
+
+### Changes
+- don't watch Sent folder by default #3025
+- use webxdc app name in chatlist/quotes/replies etc. #3027
+- make it possible to cancel message sending by removing the message #3034,
+  this was previosuly removed in 1.71.0 #2939
+- synchronize Seen flags only on watched folders to speed up
+  folder scanning #3041
+- remove direct dependency on `byteorder` crate #3031
+- refactorings #3023 #3013
+- update provider database #3043
+- improve documentation #3017 #3018 #3021
+
+### Fixes
+- fix splitting off text from webxdc messages #3032
+- call slow `delete_expired_imap_messages()` less often #3037
+- make synchronization of Seen status more robust in case unsolicited FETCH
+  result without UID is returned #3022
+- fetch Inbox before scanning folders to ensure iOS does
+  not kill the app before it gets to fetch the Inbox in background #3040
+
+
+## 1.72.0
+
+### Fixes
+- run migrations on backup import #3006
+
+
+## 1.71.0
+
+### API Changes
+- added APIs to handle database passwords: `dc_context_new_closed()`, `dc_context_open()`,
+  `dc_context_is_open()` and `dc_accounts_add_closed_account()` #2956 #2972
+- use second parameter of `dc_imex` to provide backup passphrase #2980
+- added `DC_MSG_WEBXDC`, `dc_send_webxdc_status_update()`,
+  `dc_get_webxdc_status_updates()`, `dc_msg_get_webxdc_blob()`, `dc_msg_get_webxdc_info()`
+  and `DC_EVENT_WEBXDC_STATUS_UPDATE` #2826 #2971 #2975 #2977 #2979 #2993 #2994 #2998 #3001 #3003
+- added `dc_msg_get_parent()` #2984
+- added `dc_msg_force_plaintext()` API for bots #2847
+- allow removing quotes on drafts `dc_msg_set_quote(msg, NULL)` #2950
+- removed `mvbox_watch` option; watching is enabled when `mvbox_move` is enabled #2906
+- removed `inbox_watch` option #2922
+- deprecated `os_name` in `dc_context_new()`, pass `NULL` or an empty string #2956
+
+### Changes
+- start making it possible to write to mailing lists #2736
+- add `hop_info` to `dc_get_info()` #2751 #2914 #2923
+- add information about whether the database is encrypted or not to `dc_get_info()` #3000
+- selfstatus now defaults to empty #2951 #2960
+- validate detached cryptographic signatures as used eg. by Thunderbird #2865
+- do not change the draft's `msg_id` on updates and sending #2887
+- add `imap` table to keep track of message UIDs #2909 #2938
+- replace `SendMsgToSmtp` jobs which stored outgoing messages in blobdir with `smtp` SQL table #2939 #2996
+- sql: enable `auto_vacuum=INCREMENTAL` #2931
+- sql: build rusqlite with sqlcipher #2934
+- synchronize Seen status across devices #2942
+- `dc_preconfigure_keypair` now takes ascii armored keys instead of base64 #2862
+- put removed member in Bcc instead of To in the message about removal #2864
+- improve group updates #2889
+- re-write the blob filename creation loop #2981
+- update provider database (11 Jan 2022) #2959
+- python: allow timeout for internal configure tracker API #2967
+- python: remove API deprecated in Python 3.10 #2907
+- refactorings #2932 #2957 #2947
+- improve tests #2863 #2866 #2881 #2908 #2918 #2901 #2973
+- improve documentation #2880 #2886 #2895
+- improve ci #2919 #2926 #2969 #2999
+
+### Fixes
+- fix leaving groups #2929
+- fix unread count #2861
+- make `add_parts()` not early-exit #2879
+- recognize MS Exchange read receipts as read receipts #2890
+- create parent directory if creating a new file fails #2978
+- save "configured" flag later #2974
+- improve log #2928
+- `dc_receive_imf`: don't fail on invalid address in the To field #2940
+
+
+## 1.70.0
+
+### Fixes
+- fix: do not abort Param parsing on unknown keys #2856
+- fix: execute `Chat-Group-Member-Removed:` even when arriving disordered #2857
+
+
+## 1.69.0
+
+### Fixes
+- fix group-related system messages in multi-device setups #2848
+- fix "Google Workspace" (former "G Suite") issues related to bad resolvers #2852
+
+
+## 1.68.0
+
+### Fixes
+- fix chat assignment when forwarding #2843
+- fix layout issues with the generated QR code svg #2842
+
+
+## 1.67.0
+
+### API changes
+- `dc_get_securejoin_qr_svg(chat_id)` added #2815
+- added stock-strings `DC_STR_SETUP_CONTACT_QR_DESC` and `DC_STR_SECURE_JOIN_GROUP_QR_DESC`
+
+
+## 1.66.0
+
+### API changes
+- `dc_contact_get_last_seen()` added #2823
+- python: `Contact.last_seen` added #2823
+- removed `DC_STR_NEWGROUPDRAFT`, we don't set draft after creating group anymore #2805
+
+### Changes
+- python: add cutil.from_optional_dc_charpointer() #2824
+- refactorings #2807 #2822 #2825
+
+
+## 1.65.0
+
+### Changes
+- python: add mypy support and some type hints #2809
+
+### Fixes
+- do not disable ephemeral timer when downloading a message partially #2811
+- apply existing ephemeral timer also to partially downloaded messages;
+  after full download, the ephemeral timer starts over #2811
+- replace user-visible error on verification failure with warning;
+  the error is logged to the corresponding chat anyway #2808
+
+
+## 1.64.0
+
+### Fixes
+- add 'waiting for being added to the group' only for group-joins,
+  not for setup-contact #2797
+- prioritize In-Reply-To: and References: headers over group IDs when assigning
+  messages to chats to fix incorrect assignment of Delta Chat replies to
+  classic email threads #2795
+
+
 ## 1.63.0
 
 ### API changes
@@ -40,10 +303,10 @@
 ## 1.61.0
 
 ### API Changes
-- download-on-demand added: `dc_msg_get_download_status()`, `dc_download_full_msg()`
+- download-on-demand added: `dc_msg_get_download_state()`, `dc_download_full_msg()`
   and `download_limit` config option #2631 #2696
 - `dc_create_broadcast_list()` and chat type `DC_CHAT_TYPE_BROADCAST` added #2707 #2722
-- allow ui-specific configs: `dc_set_ui_config()` and `dc_get_ui_config()` #2672
+- allow ui-specific configs using `ui.`-prefix in key (`dc_set_config(context, "ui.*", value)`) #2672
 - new strings from `DC_STR_PARTIAL_DOWNLOAD_MSG_BODY`
   to `DC_STR_PART_OF_TOTAL_USED` #2631 #2694 #2707 #2723
 - emit warnings and errors from account manager with account-id 0 #2712

CMakeLists.txt

@@ -4,10 +4,18 @@ include(GNUInstallDirs)
 
 find_program(CARGO cargo)
 
+if(APPLE)
+       set(DYNAMIC_EXT "dylib")
+elseif(UNIX)
+       set(DYNAMIC_EXT "so")
+else()
+       set(DYNAMIC_EXT "dll")
+endif()
+
 add_custom_command(
 	OUTPUT
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
+	"target/release/libdeltachat.${DYNAMIC_EXT}"
 	"target/release/pkgconfig/deltachat.pc"
         COMMAND
         PREFIX=${CMAKE_INSTALL_PREFIX}
@@ -32,11 +40,11 @@ add_custom_target(
 	ALL
 	DEPENDS
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
+	"target/release/libdeltachat.${DYNAMIC_EXT}"
 	"target/release/pkgconfig/deltachat.pc"
 )
 
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "deltachat"
-version = "1.63.0"
+version = "1.79.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
-edition = "2018"
+edition = "2021"
 license = "MPL-2.0"
-resolver = "2"
+rust-version = "1.56"
 
 [profile.dev]
 debug = 0
@@ -19,15 +19,14 @@ ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
 async-imap = { git = "https://github.com/async-email/async-imap" }
 async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
-async-std-resolver = "0.20"
-async-std = { version = "1", features = ["unstable"] }
-async-tar = "0.4"
+async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", default-features=false, features = ["smtp-transport", "socks5"] }
+async-std-resolver = "0.21"
+async-std = { version = "1" }
+async-tar = { version = "0.4", default-features=false }
 async-trait = "0.1"
 backtrace = "0.3"
 base64 = "0.13"
 bitflags = "1.3"
-byteorder = "1.3"
 chrono = "0.4"
 dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
@@ -35,39 +34,36 @@ encoded-words = { git = "https://github.com/async-email/encoded-words", branch="
 escaper = "0.1"
 futures = "0.3"
 hex = "0.4.0"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-indexmap = "1.7"
-itertools = "0.10"
+image = { version = "0.24.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
-log = {version = "0.4.8", optional = true }
+log = {version = "0.4.16", optional = true }
 mailparse = "0.13"
 native-tls = "0.2"
 num_cpus = "1.13"
 num-derive = "0.3"
 num-traits = "0.2"
-once_cell = "1.8.0"
+once_cell = "1.10.0"
 percent-encoding = "2.0"
 pgp = { version = "0.7", default-features = false }
 pretty_env_logger = { version = "0.4", optional = true }
 quick-xml = "0.22"
 r2d2 = "0.8"
-r2d2_sqlite = "0.18"
+r2d2_sqlite = "0.20"
 rand = "0.7"
 regex = "1.5"
-rusqlite = "0.25"
+rusqlite = { version = "0.27", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-rustyline = { version = "9.0", optional = true }
+rustyline = { version = "9", optional = true }
 sanitize-filename = "0.3"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9"
-sha2 = "0.9"
+sha-1 = "0.10"
+sha2 = "0.10"
 smallvec = "1"
-stop-token = "0.6"
-strum = "0.22"
-strum_macros = "0.22"
+strum = "0.24"
+strum_macros = "0.24"
 surf = { version = "2.3", default-features = false, features = ["h1-client"] }
 thiserror = "1"
 toml = "0.5"
@@ -75,16 +71,19 @@ url = "2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 fast-socks5 = "0.4"
 humansize = "1"
+qrcodegen = "1.7.0"
+tagger = "4.3.3"
+textwrap = "0.15.0"
+zip = { version = "0.6.2", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
 async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = "0.3"
+criterion = { version = "0.3.4", features = ["async_std"] }
 futures-lite = "1.12"
 log = "0.4"
-pretty_assertions = "1.0"
 pretty_env_logger = "0.4"
-proptest = "1"
+proptest = { version = "1", default-features = false, features = ["std"] }
 tempfile = "3"
 
 [workspace]
@@ -116,9 +115,21 @@ harness = false
 name = "search_msgs"
 harness = false
 
+[[bench]]
+name = "receive_emails"
+harness = false
+
+[[bench]]
+name = "get_chat_msgs"
+harness = false
+
+[[bench]]
+name = "get_chatlist"
+harness = false
+
 [features]
 default = ["vendored"]
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl"]
 nightly = ["pgp/nightly"]

README.md

@@ -12,6 +12,8 @@ To download and install the official compiler for the Rust programming language,
 $ curl https://sh.rustup.rs -sSf | sh
 ```
 
+> On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
+
 ## Using the CLI client
 
 Compile and run Delta Chat Core command line utility, using `cargo`:
@@ -125,11 +127,11 @@ $ cargo test -- --ignored
 
 Language bindings are available for:
 
-- [C](https://c.delta.chat)
-- [Node.js](https://www.npmjs.com/package/deltachat-node)
-- [Python](https://py.delta.chat)
-- [Go](https://github.com/deltachat/go-deltachat/)
-- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
+- **C** \[[📂 source](./deltachat-ffi) | [📚 docs](https://c.delta.chat)\]
+- **Node.js** \[[📂 source](https://github.com/deltachat/deltachat-node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
+- **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
+- **Go** \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
+- **Free Pascal** \[[📂 source](https://github.com/deltachat/deltachat-fp/)\]
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library

assets/icon-webxdc.svg

@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="80mm"
+   height="297mm"
+   viewBox="0 0 80 297"
+   version="1.1"
+   id="svg71"
+   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
+   sodipodi:docname="icon-webxdc.svg"
+   inkscape:export-filename="C:\Users\user\OneDrive - BFW-Leipzig\Documents\LogoDC\finalohnerand.png"
+   inkscape:export-xdpi="96"
+   inkscape:export-ydpi="96">
+  <metadata
+     id="metadata856">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <sodipodi:namedview
+     id="namedview73"
+     pagecolor="#767676"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0"
+     inkscape:pagecheckerboard="true"
+     inkscape:document-units="mm"
+     showgrid="false"
+     showborder="false"
+     inkscape:snap-bbox="true"
+     inkscape:bbox-paths="true"
+     inkscape:bbox-nodes="true"
+     inkscape:snap-bbox-edge-midpoints="true"
+     inkscape:snap-bbox-midpoints="true"
+     inkscape:object-paths="true"
+     inkscape:snap-intersection-paths="true"
+     inkscape:zoom="1.4142136"
+     inkscape:cx="-90.271136"
+     inkscape:cy="-1233.1209"
+     inkscape:window-width="1864"
+     inkscape:window-height="1027"
+     inkscape:window-x="56"
+     inkscape:window-y="25"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1"
+     inkscape:snap-global="false"
+     showguides="false"
+     inkscape:guide-bbox="true"
+     inkscape:document-rotation="0"
+     units="px">
+    <sodipodi:guide
+       position="-154.76097,641.11689"
+       orientation="0,-1"
+       id="guide21118" />
+    <sodipodi:guide
+       position="-60.286487,633.36619"
+       orientation="0,-1"
+       id="guide21120" />
+  </sodipodi:namedview>
+  <defs
+     id="defs68">
+    <linearGradient
+       id="linearGradient4375">
+      <stop
+         style="stop-color:#364e59;stop-opacity:1;"
+         offset="0"
+         id="stop4377" />
+      <stop
+         style="stop-color:#364e59;stop-opacity:0;"
+         offset="1"
+         id="stop4379" />
+    </linearGradient>
+  </defs>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <rect
+       style="fill:#1a1a1a;stroke:#000000;stroke-width:0.167903"
+       id="rect880"
+       width="79.8321"
+       height="79.8321"
+       x="-64.03286"
+       y="-375.9097"
+       ry="0" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path3799-2"
+       d="m -24.089585,-372.59579 c -19.986026,0.24336 -36.196903,16.666 -36.196903,36.67011 0,20.00409 16.210877,36.03233 36.196903,35.78912 19.0024236,-0.076 14.5340713,-10.6146 35.538854,-0.85693 -11.50627538,-17.97454 0.390097,-20.36737 0.658079,-35.81316 0,-20.00411 -16.2108788,-36.03235 -36.196911,-35.78914 z"
+       style="fill:#364e59;fill-opacity:1;stroke:none;stroke-width:1.93355"
+       sodipodi:nodetypes="sscccs" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="M -54.193871,-325.26419 Z"
+       id="path3846" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="M -49.397951,-326.67773 Z"
+       id="path3848" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m -49.397951,-326.67773 v 0 0"
+       id="path3850" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.01;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+       d="m -51.35133,-325.0334 -7.964067,5.98895 z"
+       id="path3965" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path11037"
+       d="m -24.089585,-372.19891 c -19.986026,0.24156 -36.196903,16.54352 -36.196903,36.40062 0,7.86524 2.543315,15.1113 6.857155,20.97971 6.577146,8.94734 11.123515,9.77363 11.123515,9.77363 1.343237,1.78324 10.270932,4.3223 10.270932,4.3223 l 16.791727,-70.86654 -0.468369,-0.33457 c 0.458597,0.26445 0.428277,-0.27515 -8.378035,-0.27515 z"
+       style="fill:#7cc5cc;fill-opacity:1;stroke:none;stroke-width:1.92643"
+       sodipodi:nodetypes="sssccccss" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="M -49.944239,-310.69957 Z"
+       id="path13674" />
+    <g
+       id="g15178"
+       transform="matrix(0.79975737,0,0,0.79975737,53.088959,-63.716396)">
+      <rect
+         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
+         id="rect15072"
+         width="29.897917"
+         height="6.8791666"
+         x="-334.4964"
+         y="-154.51025"
+         transform="rotate(45)" />
+      <rect
+         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
+         id="rect15072-5"
+         width="29.897917"
+         height="6.8791666"
+         x="147.63107"
+         y="-334.4964"
+         transform="rotate(-45)"
+         inkscape:transform-center-x="-0.74835017"
+         inkscape:transform-center-y="0.37417525" />
+    </g>
+    <g
+       id="g22468"
+       transform="translate(3.3033974)">
+      <g
+         id="g15178-0"
+         transform="matrix(-0.79975737,0,0,0.79975737,-103.11028,-63.716404)"
+         style="fill:#7cc5cc;fill-opacity:1">
+        <rect
+           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
+           id="rect15072-2"
+           width="29.897917"
+           height="6.8791666"
+           x="-334.4964"
+           y="-154.51025"
+           transform="rotate(45)" />
+        <rect
+           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
+           id="rect15072-5-5"
+           width="29.897917"
+           height="6.8791666"
+           x="147.63107"
+           y="-334.4964"
+           transform="rotate(-45)"
+           inkscape:transform-center-x="-0.74835017"
+           inkscape:transform-center-y="0.37417525" />
+      </g>
+    </g>
+  </g>
+</svg>

assets/qrcode_logo_footer.svg

@@ -0,0 +1,10 @@
+<text
+     xml:space="preserve"
+     style="font-weight:bold;font-size:24.4118px;line-height:1.25;font-family:sans-serif;fill:#aaaaaa;fill-opacity:1;stroke:none;stroke-width:0.915439"
+     x="42.325161"
+     y="23.32255"
+     id="text72398">get.delta.chat</text>
+  <path
+     id="path84310"
+     style="opacity:0.25;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.915439"
+     d="M 17.13769,0.00129321 C 7.6753075,0.11650893 0,7.8915283 0,17.362467 c 0,9.47094 7.6753075,17.059745 17.13769,16.944599 8.99669,-0.03598 6.880074,-5.025654 16.824785,-0.405885 -5.447648,-8.510047 0.184241,-9.642482 0.311117,-16.955289 0,-9.4709395 -7.673512,-17.0597453 -17.135895,-16.94459879 z M 17.0769,4.9986797 c 1.84214,0 3.447355,0.253959 4.815003,0.7616693 1.381603,0.5076411 2.072253,1.207862 2.072253,2.0990711 0,0.4286855 -0.167495,0.7836052 -0.50242,1.0656242 -0.334921,0.2819844 -0.724544,0.4237724 -1.171121,0.4237724 -0.641952,0 -1.396532,-0.3909376 -2.261778,-1.169353 C 19.14963,7.3898036 18.402555,6.83791 17.788507,6.5220182 17.188416,6.1950547 16.484552,6.0321266 15.675129,6.0321266 c -1.032717,0 -1.883352,0.1854523 -2.553215,0.5578447 -0.655913,0.372254 -0.98517,0.8460916 -0.98517,1.4214436 0,0.5414792 0.272815,1.0495355 0.817093,1.5233385 0.544275,0.4738026 1.946291,1.3367446 4.207097,2.5889976 2.414319,1.342419 4.117377,2.390985 5.108232,3.146807 1.004795,0.755857 1.821505,1.675853 2.449514,2.758846 0.628002,1.082993 0.942253,2.227607 0.942253,3.434674 0,2.120834 -0.929555,3.993314 -2.785656,5.617786 -1.84214,1.613228 -3.99694,2.41915 -6.467082,2.41915 -2.246845,0 -4.145607,-0.647976 -5.694677,-1.945312 -1.5490699,-1.297336 -2.3225722,-3.028063 -2.3225722,-5.194049 0,-2.087031 0.8506345,-3.83094 2.5532182,-5.229825 1.716541,-1.398884 3.824203,-2.245599 6.322256,-2.538897 -0.697774,-0.631749 -1.668763,-1.387225 -2.910816,-2.267155 -1.367648,-0.970199 -2.287914,-1.73045 -2.762402,-2.283243 -0.474491,-0.5640381 -0.711618,-1.1795944 -0.711618,-1.8451814 0,-0.9927581 0.572093,-1.7710351 1.716451,-2.3351077 1.144362,-0.5753173 2.636724,-0.8635642 4.478865,-0.8635642 z m 1.110327,10.3738083 c -4.005262,0.5302 -6.007576,2.75279 -6.007576,6.667322 0,2.01932 0.49495,3.587291 1.485805,4.704157 1.004806,1.116832 2.169696,1.675299 3.495479,1.675299 1.381602,0 2.520072,-0.535632 3.413229,-1.60738 0.893168,-1.082959 1.339187,-2.545264 1.339187,-4.384079 0,-2.662348 -1.242022,-5.013441 -3.726124,-7.055319 z" />

benches/contacts.rs

@@ -8,9 +8,7 @@ async fn address_book_benchmark(n: u32, read_count: u32) {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
+    let context = Context::new(dbfile.into(), id).await.unwrap();
 
     let book = (0..n)
         .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))

benches/create_account.rs

@@ -8,7 +8,7 @@ async fn create_accounts(n: u32) {
     let dir = tempdir().unwrap();
     let p: PathBuf = dir.path().join("accounts").into();
 
-    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+    let mut accounts = Accounts::new(p.clone()).await.unwrap();
 
     for expected_id in 2..n {
         let id = accounts.add_account().await.unwrap();

benches/get_chat_msgs.rs

@@ -0,0 +1,40 @@
+use async_std::path::Path;
+
+use criterion::async_executor::AsyncStdExecutor;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+use deltachat::chat::{self, ChatId};
+use deltachat::chatlist::Chatlist;
+use deltachat::context::Context;
+
+async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
+    let id = 100;
+    let context = Context::new(dbfile.into(), id).await.unwrap();
+
+    for c in chats.iter().take(10) {
+        black_box(chat::get_chat_msgs(&context, *c, 0, None).await.ok());
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let chats: Vec<_> = async_std::task::block_on(async {
+            let context = Context::new((&path).into(), 100).await.unwrap();
+            let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
+            let len = chatlist.len();
+            (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
+        });
+
+        c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
+            b.to_async(AsyncStdExecutor)
+                .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
+        });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/get_chatlist.rs

@@ -0,0 +1,27 @@
+use criterion::async_executor::AsyncStdExecutor;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+use deltachat::chatlist::Chatlist;
+use deltachat::context::Context;
+
+async fn get_chat_list_benchmark(context: &Context) {
+    Chatlist::try_load(context, 0, None, None).await.unwrap();
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let context =
+            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
+        c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
+            b.to_async(AsyncStdExecutor)
+                .iter(|| get_chat_list_benchmark(black_box(&context)))
+        });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/receive_emails.rs

@@ -0,0 +1,84 @@
+use async_std::{path::PathBuf, task::block_on};
+use criterion::{
+    async_executor::AsyncStdExecutor, black_box, criterion_group, criterion_main, BatchSize,
+    Criterion,
+};
+use deltachat::{
+    config::Config,
+    context::Context,
+    dc_receive_imf::dc_receive_imf,
+    imex::{imex, ImexMode},
+};
+use tempfile::tempdir;
+
+async fn recv_all_emails(context: Context) -> Context {
+    for i in 0..100 {
+        let imf_raw = format!(
+            "Subject: Benchmark
+Message-ID: Mr.OssSYnOFkhR.{i}@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+To: alice@example.com
+From: sender@testrun.org
+Chat-Version: 1.0
+Chat-Disposition-Notification-To: sender@testrun.org
+Chat-User-Avatar: 0
+In-Reply-To: Mr.OssSYnOFkhR.{i_dec}@testrun.org
+MIME-Version: 1.0
+
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+Hello {i}",
+            i = i,
+            i_dec = i - 1,
+        );
+        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), false)
+            .await
+            .unwrap();
+    }
+    context
+}
+
+async fn create_context() -> Context {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let id = 100;
+    let context = Context::new(dbfile.into(), id).await.unwrap();
+
+    let backup: PathBuf = std::env::current_dir()
+        .unwrap()
+        .join("delta-chat-backup.tar")
+        .into();
+    if backup.exists().await {
+        println!("Importing backup");
+        imex(&context, ImexMode::ImportBackup, &backup, None)
+            .await
+            .unwrap();
+    }
+
+    let addr = "alice@example.com";
+    context.set_config(Config::Addr, Some(addr)).await.unwrap();
+    context
+        .set_config(Config::ConfiguredAddr, Some(addr))
+        .await
+        .unwrap();
+    context
+        .set_config(Config::Configured, Some("1"))
+        .await
+        .unwrap();
+    context
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Receive messages");
+    group.bench_function("Receive 100 simple text msgs", |b| {
+        b.to_async(AsyncStdExecutor).iter_batched(
+            || block_on(create_context()),
+            |context| recv_all_emails(black_box(context)),
+            BatchSize::LargeInput,
+        );
+    });
+    group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/search_msgs.rs

@@ -6,9 +6,7 @@ use std::path::Path;
 async fn search_benchmark(path: impl AsRef<Path>) {
     let dbfile = path.as_ref();
     let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
+    let context = Context::new(dbfile.into(), id).await.unwrap();
 
     for _ in 0..10u32 {
         context.search_msgs(None, "hello").await.unwrap();
@@ -22,6 +20,8 @@ fn criterion_benchmark(c: &mut Criterion) {
         c.bench_function("search hello", |b| {
             b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
         });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
     }
 }
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.63.0"
+version = "1.79.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/README.md

@@ -1,5 +1,9 @@
 # Delta Chat C Interface
 
+## Installation
+
+see `Installing libdeltachat system wide` in [../README.md](../README.md)
+
 ## Documentation
 
 To generate the C Interface documentation,

deltachat-ffi/src/lib.rs

@@ -25,16 +25,19 @@ use std::time::{Duration, SystemTime};
 use anyhow::Context as _;
 use async_std::sync::RwLock;
 use async_std::task::{block_on, spawn};
+use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use num_traits::{FromPrimitive, ToPrimitive};
+use rand::Rng;
 
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
-use deltachat::contact::{Contact, Origin};
+use deltachat::contact::{Contact, ContactId, Origin};
 use deltachat::context::Context;
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
 use deltachat::stock_str::StockMessage;
+use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
 
@@ -62,7 +65,7 @@ pub type dc_context_t = Context;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_context_new(
-    os_name: *const libc::c_char,
+    _os_name: *const libc::c_char,
     dbfile: *const libc::c_char,
     blobdir: *const libc::c_char,
 ) -> *mut dc_context_t {
@@ -73,21 +76,10 @@ pub unsafe extern "C" fn dc_context_new(
         return ptr::null_mut();
     }
 
-    let os_name = if os_name.is_null() {
-        String::from("DcFFI")
-    } else {
-        to_string_lossy(os_name)
-    };
-
     let ctx = if blobdir.is_null() || *blobdir == 0 {
-        use rand::Rng;
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
-        block_on(Context::new(
-            os_name,
-            as_path(dbfile).to_path_buf().into(),
-            id,
-        ))
+        block_on(Context::new(as_path(dbfile).to_path_buf().into(), id))
     } else {
         eprintln!("blobdir can not be defined explicitly anymore");
         return ptr::null_mut();
@@ -95,12 +87,63 @@ pub unsafe extern "C" fn dc_context_new(
     match ctx {
         Ok(ctx) => Box::into_raw(Box::new(ctx)),
         Err(err) => {
-            eprintln!("failed to create context: {}", err);
+            eprintln!("failed to create context: {:#}", err);
             ptr::null_mut()
         }
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *mut dc_context_t {
+    setup_panic!();
+
+    if dbfile.is_null() {
+        eprintln!("ignoring careless call to dc_context_new_closed()");
+        return ptr::null_mut();
+    }
+
+    let id = rand::thread_rng().gen();
+    match block_on(Context::new_closed(
+        as_path(dbfile).to_path_buf().into(),
+        id,
+    )) {
+        Ok(context) => Box::into_raw(Box::new(context)),
+        Err(err) => {
+            eprintln!("failed to create context: {:#}", err);
+            ptr::null_mut()
+        }
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_context_open(
+    context: *mut dc_context_t,
+    passphrase: *const libc::c_char,
+) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_context_open()");
+        return 0;
+    }
+
+    let ctx = &*context;
+    let passphrase = to_string_lossy(passphrase);
+    block_on(ctx.open(passphrase))
+        .log_err(ctx, "dc_context_open() failed")
+        .map(|b| b as libc::c_int)
+        .unwrap_or(0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_context_is_open(context: *mut dc_context_t) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_context_is_open()");
+        return 0;
+    }
+
+    let ctx = &*context;
+    block_on(ctx.is_open()) as libc::c_int
+}
+
 /// Release the context structure.
 ///
 /// This function releases the memory of the `dc_context_t` structure.
@@ -192,7 +235,7 @@ pub unsafe extern "C" fn dc_get_config(
                     .unwrap_or_default()
                     .strdup(),
                 Err(_) => {
-                    warn!(ctx, "dc_get_config(): invalid key");
+                    warn!(ctx, "dc_get_config(): invalid key '{}'", &key);
                     "".strdup()
                 }
             }
@@ -450,14 +493,17 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ChatEphemeralTimerModified { chat_id, .. } => chat_id.to_u32() as libc::c_int,
         EventType::ContactsChanged(id) | EventType::LocationChanged(id) => {
             let id = id.unwrap_or_default();
-            id as libc::c_int
+            id.to_u32() as libc::c_int
         }
         EventType::ConfigureProgress { progress, .. } | EventType::ImexProgress(progress) => {
             *progress as libc::c_int
         }
         EventType::ImexFileWritten(_) => 0,
         EventType::SecurejoinInviterProgress { contact_id, .. }
-        | EventType::SecurejoinJoinerProgress { contact_id, .. } => *contact_id as libc::c_int,
+        | EventType::SecurejoinJoinerProgress { contact_id, .. } => {
+            contact_id.to_u32() as libc::c_int
+        }
+        EventType::WebxdcStatusUpdate { msg_id, .. } => msg_id.to_u32() as libc::c_int,
     }
 }
 
@@ -489,8 +535,8 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ImexFileWritten(_)
         | EventType::MsgsNoticed(_)
         | EventType::ConnectivityChanged
-        | EventType::SelfavatarChanged
-        | EventType::ChatModified(_) => 0,
+        | EventType::SelfavatarChanged => 0,
+        EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
         | EventType::MsgDelivered { msg_id, .. }
@@ -499,6 +545,10 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         EventType::SecurejoinInviterProgress { progress, .. }
         | EventType::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
         EventType::ChatEphemeralTimerModified { timer, .. } => timer.to_u32() as libc::c_int,
+        EventType::WebxdcStatusUpdate {
+            status_update_serial,
+            ..
+        } => status_update_serial.to_u32() as libc::c_int,
     }
 }
 
@@ -540,6 +590,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SecurejoinJoinerProgress { .. }
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
+        | EventType::WebxdcStatusUpdate { .. }
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -641,8 +692,8 @@ pub unsafe extern "C" fn dc_preconfigure_keypair(
     let ctx = &*context;
     block_on(async move {
         let addr = dc_tools::EmailAddress::new(&to_string_lossy(addr))?;
-        let public = key::SignedPublicKey::from_base64(&to_string_lossy(public_data))?;
-        let secret = key::SignedSecretKey::from_base64(&to_string_lossy(secret_data))?;
+        let public = key::SignedPublicKey::from_asc(&to_string_lossy(public_data))?.0;
+        let secret = key::SignedSecretKey::from_asc(&to_string_lossy(secret_data))?.0;
         let keypair = key::KeyPair {
             addr,
             public,
@@ -669,7 +720,11 @@ pub unsafe extern "C" fn dc_get_chatlist(
     let ctx = &*context;
     let qs = to_opt_string_lossy(query_str);
 
-    let qi = if query_id == 0 { None } else { Some(query_id) };
+    let qi = if query_id == 0 {
+        None
+    } else {
+        Some(ContactId::new(query_id))
+    };
 
     block_on(async move {
         match chatlist::Chatlist::try_load(ctx, flags as usize, qs.as_deref(), qi)
@@ -697,7 +752,7 @@ pub unsafe extern "C" fn dc_create_chat_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        ChatId::create_for_contact(ctx, contact_id)
+        ChatId::create_for_contact(ctx, ContactId::new(contact_id))
             .await
             .log_err(ctx, "Failed to create chat from contact_id")
             .map(|id| id.to_u32())
@@ -717,7 +772,7 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        ChatId::lookup_by_contact(ctx, contact_id)
+        ChatId::lookup_by_contact(ctx, ContactId::new(contact_id))
             .await
             .log_err(ctx, "Failed to get chat for contact_id")
             .unwrap_or_default() // unwraps the Result
@@ -829,6 +884,48 @@ pub unsafe extern "C" fn dc_send_videochat_invitation(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_send_webxdc_status_update(
+    context: *mut dc_context_t,
+    msg_id: u32,
+    json: *const libc::c_char,
+    descr: *const libc::c_char,
+) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_send_webxdc_status_update()");
+        return 0;
+    }
+    let ctx = &*context;
+
+    block_on(ctx.send_webxdc_status_update(
+        MsgId::new(msg_id),
+        &to_string_lossy(json),
+        &to_string_lossy(descr),
+    ))
+    .log_err(ctx, "Failed to send webxdc update")
+    .is_ok() as libc::c_int
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_webxdc_status_updates(
+    context: *mut dc_context_t,
+    msg_id: u32,
+    last_known_serial: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_webxdc_status_updates()");
+        return "".strdup();
+    }
+    let ctx = &*context;
+
+    block_on(ctx.get_webxdc_status_updates(
+        MsgId::new(msg_id),
+        StatusUpdateSerial::new(last_known_serial),
+    ))
+    .unwrap_or_else(|_| "".to_string())
+    .strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_draft(
     context: *mut dc_context_t,
@@ -1252,7 +1349,10 @@ pub unsafe extern "C" fn dc_get_chat_contacts(
         let arr = dc_array_t::from(
             chat::get_chat_contacts(ctx, ChatId::new(chat_id))
                 .await
-                .unwrap_or_log_default(ctx, "Failed get_chat_contacts"),
+                .unwrap_or_log_default(ctx, "Failed get_chat_contacts")
+                .iter()
+                .map(|id| id.to_u32())
+                .collect::<Vec<u32>>(),
         );
         Box::into_raw(Box::new(arr))
     })
@@ -1362,7 +1462,7 @@ pub unsafe extern "C" fn dc_is_contact_in_chat(
     block_on(chat::is_contact_in_chat(
         ctx,
         ChatId::new(chat_id),
-        contact_id,
+        ContactId::new(contact_id),
     ))
     .log_err(ctx, "is_contact_in_chat failed")
     .unwrap_or_default() as libc::c_int
@@ -1383,7 +1483,7 @@ pub unsafe extern "C" fn dc_add_contact_to_chat(
     block_on(chat::add_contact_to_chat(
         ctx,
         ChatId::new(chat_id),
-        contact_id,
+        ContactId::new(contact_id),
     ))
     .log_err(ctx, "Failed to add contact")
     .is_ok() as libc::c_int
@@ -1404,7 +1504,7 @@ pub unsafe extern "C" fn dc_remove_contact_from_chat(
     block_on(chat::remove_contact_from_chat(
         ctx,
         ChatId::new(chat_id),
-        contact_id,
+        ContactId::new(contact_id),
     ))
     .log_err(ctx, "Failed to remove contact")
     .is_ok() as libc::c_int
@@ -1651,6 +1751,27 @@ pub unsafe extern "C" fn dc_forward_msgs(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_resend_msgs(
+    context: *mut dc_context_t,
+    msg_ids: *const u32,
+    msg_cnt: libc::c_int,
+) -> libc::c_int {
+    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
+        eprintln!("ignoring careless call to dc_resend_msgs()");
+        return 0;
+    }
+    let ctx = &*context;
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
+
+    if let Err(err) = block_on(chat::resend_msgs(ctx, &msg_ids)) {
+        error!(ctx, "Resending failed: {}", err);
+        0
+    } else {
+        1
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_markseen_msgs(
     context: *mut dc_context_t,
@@ -1736,10 +1857,11 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     let ctx = &*context;
 
     block_on(async move {
-        Contact::lookup_id_by_addr(ctx, to_string_lossy(addr), Origin::IncomingReplyTo)
+        Contact::lookup_id_by_addr(ctx, &to_string_lossy(addr), Origin::IncomingReplyTo)
             .await
             .unwrap_or_log_default(ctx, "failed to lookup id")
-            .unwrap_or(0)
+            .map(|id| id.to_u32())
+            .unwrap_or_default()
     })
 }
 
@@ -1759,6 +1881,7 @@ pub unsafe extern "C" fn dc_create_contact(
     block_on(async move {
         Contact::create(ctx, &name, &to_string_lossy(addr))
             .await
+            .map(|id| id.to_u32())
             .unwrap_or(0)
     })
 }
@@ -1797,7 +1920,9 @@ pub unsafe extern "C" fn dc_get_contacts(
 
     block_on(async move {
         match Contact::get_all(ctx, flags, query).await {
-            Ok(contacts) => Box::into_raw(Box::new(dc_array_t::from(contacts))),
+            Ok(contacts) => Box::into_raw(Box::new(dc_array_t::from(
+                contacts.iter().map(|id| id.to_u32()).collect::<Vec<u32>>(),
+            ))),
             Err(_) => ptr::null_mut(),
         }
     })
@@ -1834,7 +1959,10 @@ pub unsafe extern "C" fn dc_get_blocked_contacts(
             Contact::get_all_blocked(ctx)
                 .await
                 .log_err(ctx, "Can't get blocked contacts")
-                .unwrap_or_default(),
+                .unwrap_or_default()
+                .iter()
+                .map(|id| id.to_u32())
+                .collect::<Vec<u32>>(),
         )))
     })
 }
@@ -1845,7 +1973,8 @@ pub unsafe extern "C" fn dc_block_contact(
     contact_id: u32,
     block: libc::c_int,
 ) {
-    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL as u32 {
+    let contact_id = ContactId::new(contact_id);
+    if context.is_null() || contact_id.is_special() {
         eprintln!("ignoring careless call to dc_block_contact()");
         return;
     }
@@ -1875,7 +2004,7 @@ pub unsafe extern "C" fn dc_get_contact_encrinfo(
     let ctx = &*context;
 
     block_on(async move {
-        Contact::get_encrinfo(ctx, contact_id)
+        Contact::get_encrinfo(ctx, ContactId::new(contact_id))
             .await
             .map(|s| s.strdup())
             .unwrap_or_else(|e| {
@@ -1890,7 +2019,8 @@ pub unsafe extern "C" fn dc_delete_contact(
     context: *mut dc_context_t,
     contact_id: u32,
 ) -> libc::c_int {
-    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL as u32 {
+    let contact_id = ContactId::new(contact_id);
+    if context.is_null() || contact_id.is_special() {
         eprintln!("ignoring careless call to dc_delete_contact()");
         return 0;
     }
@@ -1916,7 +2046,7 @@ pub unsafe extern "C" fn dc_get_contact(
     let ctx = &*context;
 
     block_on(async move {
-        Contact::get_by_id(ctx, contact_id)
+        Contact::get_by_id(ctx, ContactId::new(contact_id))
             .await
             .map(|contact| Box::into_raw(Box::new(ContactWrapper { context, contact })))
             .unwrap_or_else(|_| ptr::null_mut())
@@ -1928,7 +2058,7 @@ pub unsafe extern "C" fn dc_imex(
     context: *mut dc_context_t,
     what_raw: libc::c_int,
     param1: *const libc::c_char,
-    _param2: *const libc::c_char,
+    param2: *const libc::c_char,
 ) {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_imex()");
@@ -1941,12 +2071,13 @@ pub unsafe extern "C" fn dc_imex(
             return;
         }
     };
+    let passphrase = to_opt_string_lossy(param2);
 
     let ctx = &*context;
 
     if let Some(param1) = to_opt_string_lossy(param1) {
         spawn(async move {
-            imex::imex(ctx, what, param1.as_ref())
+            imex::imex(ctx, what, param1.as_ref(), passphrase)
                 .await
                 .log_err(ctx, "IMEX failed")
         });
@@ -2075,6 +2206,27 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
         .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_securejoin_qr_svg(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to generate_verification_qr()");
+        return "".strdup();
+    }
+    let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
+
+    block_on(get_securejoin_qr_svg(ctx, chat_id))
+        .unwrap_or_else(|_| "".to_string())
+        .strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_join_securejoin(
     context: *mut dc_context_t,
@@ -2317,7 +2469,7 @@ pub unsafe extern "C" fn dc_array_get_contact_id(
         return 0;
     }
 
-    (*array).get_location(index).contact_id
+    (*array).get_location(index).contact_id.to_u32()
 }
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_msg_id(
@@ -2431,7 +2583,14 @@ pub unsafe extern "C" fn dc_chatlist_get_chat_id(
         return 0;
     }
     let ffi_list = &*chatlist;
-    ffi_list.list.get_chat_id(index as usize).to_u32()
+    let ctx = &*ffi_list.context;
+    match ffi_list.list.get_chat_id(index as usize) {
+        Ok(chat_id) => chat_id.to_u32(),
+        Err(err) => {
+            warn!(ctx, "get_chat_id failed: {}", err);
+            0
+        }
+    }
 }
 
 #[no_mangle]
@@ -2826,7 +2985,7 @@ pub unsafe extern "C" fn dc_msg_get_from_id(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_from_id()
+    ffi_msg.message.get_from_id().to_u32()
 }
 
 #[no_mangle]
@@ -2948,6 +3107,61 @@ pub unsafe extern "C" fn dc_msg_get_filename(msg: *mut dc_msg_t) -> *mut libc::c
     ffi_msg.message.get_filename().unwrap_or_default().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_webxdc_blob(
+    msg: *mut dc_msg_t,
+    filename: *const libc::c_char,
+    ret_bytes: *mut libc::size_t,
+) -> *mut libc::c_char {
+    if msg.is_null() || filename.is_null() || ret_bytes.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_webxdc_blob()");
+        return ptr::null_mut();
+    }
+    let ffi_msg = &*msg;
+    let ctx = &*ffi_msg.context;
+    let blob = block_on(async move {
+        ffi_msg
+            .message
+            .get_webxdc_blob(ctx, &to_string_lossy(filename))
+            .await
+    });
+    match blob {
+        Ok(blob) => {
+            *ret_bytes = blob.len();
+            let ptr = libc::malloc(*ret_bytes);
+            libc::memcpy(ptr, blob.as_ptr() as *mut libc::c_void, *ret_bytes);
+            ptr as *mut libc::c_char
+        }
+        Err(err) => {
+            eprintln!("failed read blob from archive: {}", err);
+            ptr::null_mut()
+        }
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_webxdc_info(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_webxdc_info()");
+        return "".strdup();
+    }
+    let ffi_msg = &*msg;
+    let ctx = &*ffi_msg.context;
+
+    block_on(async move {
+        let info = match ffi_msg.message.get_webxdc_info(ctx).await {
+            Ok(info) => info,
+            Err(err) => {
+                error!(ctx, "dc_msg_get_webxdc_info() failed to get info: {}", err);
+                return "".strdup();
+            }
+        };
+        serde_json::to_string(&info)
+            .unwrap_or_log_default(ctx, "dc_msg_get_webxdc_info() failed to serialise to json")
+            .strdup()
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_filemime(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -3360,17 +3574,21 @@ pub unsafe extern "C" fn dc_msg_set_quote(msg: *mut dc_msg_t, quote: *const dc_m
         return;
     }
     let ffi_msg = &mut *msg;
-    let ffi_quote = &*quote;
-
-    if ffi_msg.context != ffi_quote.context {
-        eprintln!("ignoring attempt to quote message from a different context");
-        return;
-    }
+    let quote_msg = if quote.is_null() {
+        None
+    } else {
+        let ffi_quote = &*quote;
+        if ffi_msg.context != ffi_quote.context {
+            eprintln!("ignoring attempt to quote message from a different context");
+            return;
+        }
+        Some(&ffi_quote.message)
+    };
 
     block_on(async move {
         ffi_msg
             .message
-            .set_quote(&*ffi_msg.context, &ffi_quote.message)
+            .set_quote(&*ffi_msg.context, quote_msg)
             .await
             .log_err(&*ffi_msg.context, "failed to set quote")
             .ok();
@@ -3413,6 +3631,39 @@ pub unsafe extern "C" fn dc_msg_get_quoted_msg(msg: *const dc_msg_t) -> *mut dc_
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_parent(msg: *const dc_msg_t) -> *mut dc_msg_t {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_parent()");
+        return ptr::null_mut();
+    }
+    let ffi_msg: &MessageWrapper = &*msg;
+    let context = &*ffi_msg.context;
+    let res = block_on(async move {
+        ffi_msg
+            .message
+            .parent(context)
+            .await
+            .log_err(context, "failed to get parent message")
+            .unwrap_or(None)
+    });
+
+    match res {
+        Some(message) => Box::into_raw(Box::new(MessageWrapper { context, message })),
+        None => ptr::null_mut(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_force_plaintext(msg: *mut dc_msg_t) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_force_plaintext()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg.message.force_plaintext();
+}
+
 // dc_contact_t
 
 /// FFI struct for [dc_contact_t]
@@ -3445,7 +3696,7 @@ pub unsafe extern "C" fn dc_contact_get_id(contact: *mut dc_contact_t) -> u32 {
         return 0;
     }
     let ffi_contact = &*contact;
-    ffi_contact.contact.get_id()
+    ffi_contact.contact.get_id().to_u32()
 }
 
 #[no_mangle]
@@ -3544,6 +3795,16 @@ pub unsafe extern "C" fn dc_contact_get_status(contact: *mut dc_contact_t) -> *m
     ffi_contact.contact.get_status().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_last_seen(contact: *mut dc_contact_t) -> i64 {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_last_seen()");
+        return 0;
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.last_seen()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_is_blocked(contact: *mut dc_contact_t) -> libc::c_int {
     if contact.is_null() {
@@ -3712,6 +3973,25 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
     }
     let addr = to_string_lossy(addr);
 
+    let ctx = &*context;
+
+    match block_on(provider::get_provider_info(ctx, addr.as_str(), true)) {
+        Some(provider) => provider,
+        None => ptr::null_mut(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_new_from_email_with_dns(
+    context: *const dc_context_t,
+    addr: *const libc::c_char,
+) -> *const dc_provider_t {
+    if context.is_null() || addr.is_null() {
+        eprintln!("ignoring careless call to dc_provider_new_from_email_with_dns()");
+        return ptr::null();
+    }
+    let addr = to_string_lossy(addr);
+
     let ctx = &*context;
     let socks5_enabled = block_on(async move {
         ctx.get_config_bool(config::Config::Socks5Enabled)
@@ -3721,7 +4001,11 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
 
     match socks5_enabled {
         Ok(socks5_enabled) => {
-            match block_on(provider::get_provider_info(addr.as_str(), socks5_enabled)) {
+            match block_on(provider::get_provider_info(
+                ctx,
+                addr.as_str(),
+                socks5_enabled,
+            )) {
                 Some(provider) => provider,
                 None => ptr::null_mut(),
             }
@@ -3803,7 +4087,7 @@ pub type dc_accounts_t = AccountsWrapper;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_new(
-    os_name: *const libc::c_char,
+    _os_name: *const libc::c_char,
     dbfile: *const libc::c_char,
 ) -> *mut dc_accounts_t {
     setup_panic!();
@@ -3813,13 +4097,7 @@ pub unsafe extern "C" fn dc_accounts_new(
         return ptr::null_mut();
     }
 
-    let os_name = if os_name.is_null() {
-        String::from("DcFFI")
-    } else {
-        to_string_lossy(os_name)
-    };
-
-    let accs = block_on(Accounts::new(os_name, as_path(dbfile).to_path_buf().into()));
+    let accs = block_on(Accounts::new(as_path(dbfile).to_path_buf().into()));
 
     match accs {
         Ok(accs) => Box::into_raw(Box::new(AccountsWrapper::new(accs))),
@@ -3924,6 +4202,30 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_add_closed_account(accounts: *mut dc_accounts_t) -> u32 {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_add_closed_account()");
+        return 0;
+    }
+
+    let accounts = &mut *accounts;
+
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        match accounts.add_closed_account().await {
+            Ok(id) => id,
+            Err(err) => {
+                accounts.emit_event(EventType::Error(format!(
+                    "Failed to add account: {:#}",
+                    err
+                )));
+                0
+            }
+        }
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_remove_account(
     accounts: *mut dc_accounts_t,

deltachat-ffi/src/lot.rs

@@ -111,19 +111,19 @@ impl Lot {
         match self {
             Self::Summary(_) => Default::default(),
             Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { contact_id, .. } => *contact_id,
+                Qr::AskVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::AskVerifyGroup { .. } => Default::default(),
-                Qr::FprOk { contact_id } => *contact_id,
-                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default(),
+                Qr::FprOk { contact_id } => contact_id.to_u32(),
+                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => *contact_id,
+                Qr::Addr { contact_id } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
-                Qr::WithdrawVerifyContact { contact_id, .. } => *contact_id,
+                Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::WithdrawVerifyGroup { .. } => Default::default(),
-                Qr::ReviveVerifyContact { contact_id, .. } => *contact_id,
+                Qr::ReviveVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::ReviveVerifyGroup { .. } => Default::default(),
             },
             Self::Error(_) => Default::default(),

draft/aeap-mvp.rst

@@ -0,0 +1,33 @@
+AEAP MVP
+========
+
+Changes to the UIs
+------------------
+
+- The secondary self addresses (see below) are shown in the UI, but not editable.
+
+- When the user changed the email address in the configure screen, show a dialog to the user, either directly explaining things or with a link to the FAQ (see "Other" below)
+
+Changes in the core
+-------------------
+
+- We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
+
+- If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
+
+  - don't forget to deduplicate secondary self addresses in case the user switches back and forth between addresses).
+
+  - The key stays the same.
+
+- No changes for 1:1 chats, there simply is a new one
+
+- When we send a message to a group, and the primary address is not a member of a group, but a secondary address is:
+  
+  Add Chat-Group-Member-Removed=<old address> and Chat-Group-Member-Added=<new address> headers to this message
+
+  - On the receiving side, make sure that we accept this (even in verified groups) if the message is signed and the key stayed the same
+
+Other
+-----
+
+- The user is responsible that messages to the old address arrive at the new address, for example by configuring the old provider to forward all emails to the new one.

draft/webxdc-dev-reference.md

@@ -0,0 +1,185 @@
+# Webxdc Developer Reference
+
+## Webxdc File Format
+
+- a **Webxdc app** is a **ZIP-file** with the extension `.xdc`
+- the ZIP-file must use the default compression methods as of RFC 1950,
+  this is "Deflate" or "Store"
+- the ZIP-file must contain at least the file `index.html`
+- if the Webxdc app is started, `index.html` is opened in a restricted webview
+  that allow accessing resources only from the ZIP-file
+
+
+## Webxdc API
+
+There are some additional APIs available once `webxdc.js` is included
+(the file will be provided by the concrete implementations,
+no need to add `webxdc.js` to your ZIP-file):
+
+```html
+<script src="webxdc.js"></script>
+```
+
+### sendUpdate()
+
+```js
+window.webxdc.sendUpdate(update, descr);
+```
+
+Webxdc apps are usually shared in a chat and run independently on each peer.
+To get a shared state, the peers use `sendUpdate()` to send updates to each other.
+
+- `update`: an object with the following properties:  
+    - `update.payload`: any javascript primitive, array or object.
+    - `update.info`: optional, short, informational message that will be added to the chat,
+       eg. "Alice voted" or "Bob scored 123 in MyGame";
+       usually only one line of text is shown,
+       use this option sparingly to not spam the chat.
+    - `update.summary`: optional, short text, shown beside app icon;
+       it is recommended to use some aggregated value,  eg. "8 votes", "Highscore: 123"
+
+- `descr`: short, human-readable description what this update is about.
+  this is shown eg. as a fallback text in an email program.
+
+All peers, including the sending one,
+will receive the update by the callback given to `setUpdateListener()`.
+
+There are situations where the user cannot send messages to a chat,
+eg. if the webxdc instance comes as a contact request or if the user has left a group.
+In these cases, you can still call `sendUpdate()`,
+however, the update won't be sent to other peers
+and you won't get the update by `setUpdateListener()`.
+
+
+### setUpdateListener()
+
+```js
+let promise = window.webxdc.setUpdateListener((update) => {}, serial);
+```
+
+With `setUpdateListener()` you define a callback that receives the updates
+sent by `sendUpdate()`. The callback is called for updates sent by you or other peers.
+The `serial` specifies the last serial that you know about (defaults to 0). 
+The returned promise resolves when the listener has processed all the update messages known at the time when  `setUpdateListener` was called. 
+
+Each `update` which is passed to the callback comes with the following properties: 
+
+- `update.payload`: equals the payload given to `sendUpdate()`
+
+- `update.serial`: the serial number of this update.
+  Serials are larger `0` and newer serials have higher numbers.
+  There may be gaps in the serials
+  and it is not guaranteed that the next serial is exactly incremented by one.
+
+- `update.max_serial`: the maximum serial currently known.
+  If `max_serial` equals `serial` this update is the last update (until new network messages arrive).
+
+- `update.info`: optional, short, informational message (see `send_update`) 
+
+- `update.summary`: optional, short text, shown beside app icon (see `send_update`)
+
+
+### selfAddr
+
+```js
+window.webxdc.selfAddr
+```
+
+Property with the peer's own address.
+This is esp. useful if you want to differ between different peers -
+just send the address along with the payload,
+and, if needed, compare the payload addresses against selfAddr() later on.
+
+
+### selfName
+
+```js
+window.webxdc.selfName
+```
+
+Property with the peer's own name.
+This is name chosen by the user in their settings,
+if there is nothing set, that defaults to the peer's address.
+
+
+## manifest.toml
+
+If the ZIP-file contains a `manifest.toml` in its root directory,
+some basic information are read and used from there.
+
+the `manifest.toml` has the following format
+
+```toml
+name = "My App Name"
+```
+
+- **name** - The name of the app.
+  If no name is set or if there is no manifest, the filename is used as the app name.
+
+
+## App Icon
+
+If the ZIP-root contains an `icon.png` or `icon.jpg`,
+these files are used as the icon for the app.
+The icon should be a square at reasonable width/height;
+round corners etc. will be added by the implementations as needed.
+If no icon is set, a default icon will be used.
+
+
+## Webxdc Examples
+
+The following example shows an input field and  every input is show on all peers.
+
+```html
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8"/>
+    <script src="webxdc.js"></script>
+  </head>
+  <body>
+    <input id="input" type="text"/>
+    <a href="" onclick="sendMsg(); return false;">Send</a>
+    <p id="output"></p>
+    <script>
+    
+      function sendMsg() {
+        msg = document.getElementById("input").value;
+        window.webxdc.sendUpdate({payload: msg}, 'Someone typed "'+msg+'".');
+      }
+    
+      function receiveUpdate(update) {
+        document.getElementById('output').innerHTML += update.payload + "<br>";
+      }
+    
+      window.webxdc.setUpdateListener(receiveUpdate, 0);
+    </script>
+  </body>
+</html>
+```
+
+[Webxdc Development Tool](https://github.com/deltachat/webxdc-dev)
+offers an **Webxdc Simulator** that can be used in many browsers without any installation needed.
+You can also use that repository as a template for your own app -
+just clone and start adapting things to your need.
+
+
+### Advanced Examples
+
+- [2048](https://github.com/adbenitez/2048.xdc)
+- [Draw](https://github.com/adbenitez/draw.xdc)
+- [Poll](https://github.com/r10s/webxdc-poll/)
+- [Tic Tac Toe](https://github.com/Simon-Laux/tictactoe.xdc)
+- Even more with [Topic #webxdc on Github](https://github.com/topics/webxdc)
+
+
+## Closing Remarks
+
+- older devices might not have the newest js features in their webview,
+  you may want to transpile your code down to an older js version eg. with https://babeljs.io
+- viewport and scaling features are implementation specific,
+  if you want to have an explicit behavior, you can add eg.
+  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
+- there are tons of ideas for enhancements of the API and the file format,
+  eg. in the future, we will may define icon- and manifest-files,
+  allow to aggregate the state or add metadata.

examples/repl/cmdline.rs

@@ -17,11 +17,10 @@ use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
-use deltachat::message::{self, Message, MessageState, MsgId};
+use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
-use deltachat::EventType;
 use deltachat::{config, provider};
 use std::fs;
 use std::time::{Duration, SystemTime};
@@ -84,6 +83,7 @@ async fn reset_tables(context: &Context, bits: i32) {
             )
             .await
             .unwrap();
+        context.sql().config_cache().write().await.clear();
         context
             .sql()
             .execute("DELETE FROM leftgrps;", paramsv![])
@@ -92,16 +92,13 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_event(EventType::MsgsChanged {
-        chat_id: ChatId::new(0),
-        msg_id: MsgId::new(0),
-    });
+    context.emit_msgs_changed_without_ids();
 }
 
 async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
     let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false).await {
+    if let Err(err) = dc_receive_imf(context, &data, false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -163,10 +160,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
+        context.emit_msgs_changed_without_ids();
     }
     true
 }
@@ -209,7 +203,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         contact_id,
         msgtext.unwrap_or_default(),
         if msg.has_html() { "[HAS-HTML]️" } else { "" },
-        if msg.get_from_id() == 1 {
+        if msg.get_from_id() == ContactId::SELF {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -267,9 +261,8 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
     Ok(())
 }
 
-async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
+async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()> {
     for contact_id in contacts {
-        let line;
         let mut line2 = "".to_string();
         let contact = Contact::get_by_id(context, *contact_id).await?;
         let name = contact.get_display_name();
@@ -284,24 +277,20 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
         } else {
             ""
         };
-        line = format!(
+        let line = format!(
             "{}{} <{}>",
             if !name.is_empty() {
-                &name
+                name
             } else {
                 "<name unset>"
             },
             verified_str,
-            if !addr.is_empty() {
-                &addr
-            } else {
-                "addr unset"
-            }
+            if !addr.is_empty() { addr } else { "addr unset" }
         );
-        let peerstate = Peerstate::from_addr(context, &addr)
+        let peerstate = Peerstate::from_addr(context, addr)
             .await
             .expect("peerstate error");
-        if peerstate.is_some() && *contact_id != 1 {
+        if peerstate.is_some() && *contact_id != ContactId::SELF {
             line2 = format!(
                 ", prefer-encrypt={}",
                 peerstate.as_ref().unwrap().prefer_encrypt
@@ -387,6 +376,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  sendfile <file> [<text>]\n\
                  sendhtml <file for html-part> [<text for plain-part>]\n\
                  sendsyncmsg\n\
+                 sendupdate <msg-id> <json status update>\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -409,6 +399,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
+                 resend <msg-id>\n\
                  markseen <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
@@ -423,12 +414,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listblocked\n\
                  ======================================Misc.==\n\
                  getqr [<chat-id>]\n\
+                 getqrsvg [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
                  joinqr <qr-content>\n\
                  setqr <qr-content>\n\
                  providerinfo <addr>\n\
-                 event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
                  clear -- clear screen\n\
@@ -470,20 +461,32 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportBackup, dir.as_ref()).await?;
+            imex(
+                &context,
+                ImexMode::ExportBackup,
+                dir.as_ref(),
+                Some(arg2.to_string()),
+            )
+            .await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(&context, ImexMode::ImportBackup, arg1.as_ref()).await?;
+            imex(
+                &context,
+                ImexMode::ImportBackup,
+                arg1.as_ref(),
+                Some(arg2.to_string()),
+            )
+            .await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref()).await?;
+            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref(), None).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref()).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref(), None).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -562,7 +565,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 );
 
                 for i in (0..cnt).rev() {
-                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)?).await?;
                     println!(
                         "{}#{}: {} [{} fresh] {}{}{}{}",
                         chat_prefix(&chat),
@@ -705,7 +708,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id: u32 = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
 
             println!("Single#{} created successfully.", chat_id,);
@@ -733,7 +736,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id_0: u32 = arg1.parse()?;
+            let contact_id_0 = ContactId::new(arg1.parse()?);
             chat::add_contact_to_chat(&context, sel_chat.as_ref().unwrap().get_id(), contact_id_0)
                 .await?;
             println!("Contact added to chat.");
@@ -741,7 +744,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id_1: u32 = arg1.parse()?;
+            let contact_id_1 = ContactId::new(arg1.parse()?);
             chat::remove_contact_from_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
@@ -754,7 +757,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "groupname" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            chat::set_chat_name(&context, sel_chat.as_ref().unwrap().get_id(), arg1).await?;
+            chat::set_chat_name(
+                &context,
+                sel_chat.as_ref().unwrap().get_id(),
+                format!("{} {}", arg1, arg2).trim(),
+            )
+            .await?;
 
             println!("Chat name set");
         }
@@ -901,6 +909,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             Some(msg_id) => println!("sync message sent as {}.", msg_id),
             None => println!("sync message not needed."),
         },
+        "sendupdate" => {
+            ensure!(
+                !arg1.is_empty() && !arg2.is_empty(),
+                "Arguments <msg-id> <json status update> expected"
+            );
+            let msg_id = MsgId::new(arg1.parse()?);
+            context
+                .send_webxdc_status_update(msg_id, arg2, "this is a webxdc status update")
+                .await?;
+        }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -908,13 +926,24 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
+            let query = format!("{} {}", arg1, arg2).trim().to_string();
             let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
             let time_start = std::time::SystemTime::now();
-            let msglist = context.search_msgs(chat, arg1).await?;
+            let msglist = context.search_msgs(chat, &query).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
-            println!("{} messages.", msglist.len());
+            println!(
+                "{}{} messages for {}search of \"{}\"",
+                msglist.len(),
+                if msglist.len() == 1000 { "+" } else { "" },
+                if chat.is_none() {
+                    "global "
+                } else {
+                    "in-chat-"
+                },
+                query,
+            );
             println!("{:?} to create this list", time_needed);
         }
         "draft" => {
@@ -1071,6 +1100,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(&context, &msg_ids, chat_id).await?;
         }
+        "resend" => {
+            ensure!(!arg1.is_empty(), "Arguments <msg-id> expected");
+
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
+            chat::resend_msgs(&context, &msg_ids).await?;
+        }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
@@ -1110,7 +1146,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id: u32 = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
@@ -1136,7 +1172,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     if 0 != i {
                         res += ", ";
                     }
-                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)?).await?;
                     res += &format!("{}#{}", chat_prefix(&chat), chat.get_id());
                 }
             }
@@ -1145,16 +1181,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "delcontact" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            Contact::delete(&context, arg1.parse()?).await?;
+            Contact::delete(&context, ContactId::new(arg1.parse()?)).await?;
         }
         "block" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             Contact::block(&context, contact_id).await?;
         }
         "unblock" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             Contact::unblock(&context, contact_id).await?;
         }
         "listblocked" => {
@@ -1179,7 +1215,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let socks5_enabled = context
                 .get_config_bool(config::Config::Socks5Enabled)
                 .await?;
-            match provider::get_provider_info(arg1, socks5_enabled).await {
+            match provider::get_provider_info(&context, arg1, socks5_enabled).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);
@@ -1195,17 +1231,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 }
             }
         }
-        // TODO: implement this again, unclear how to match this through though, without writing a parser.
-        // "event" => {
-        //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
-        //     let event = arg1.parse()?;
-        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
-        //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
-        //     println!(
-        //         "Sending event {:?}({}), received value {}.",
-        //         event, event as usize, r,
-        //     );
-        // }
         "fileinfo" => {
             ensure!(!arg1.is_empty(), "Argument <file> missing.");
 

examples/repl/main.rs

@@ -33,6 +33,8 @@ use rustyline::{
 
 mod cmdline;
 use self::cmdline::*;
+use deltachat::qr_code_generator::get_securejoin_qr_svg;
+use std::fs;
 
 /// Event Handler
 fn receive_event(event: EventType) {
@@ -167,7 +169,7 @@ const DB_COMMANDS: [&str; 10] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 35] = [
+const CHAT_COMMANDS: [&str; 36] = [
     "listchats",
     "listarchived",
     "chat",
@@ -189,6 +191,7 @@ const CHAT_COMMANDS: [&str; 35] = [
     "sendfile",
     "sendhtml",
     "sendsyncmsg",
+    "sendupdate",
     "videochat",
     "draft",
     "listmedia",
@@ -204,11 +207,12 @@ const CHAT_COMMANDS: [&str; 35] = [
     "accept",
     "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 7] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
+    "resend",
     "markseen",
     "delmsg",
     "download",
@@ -226,10 +230,10 @@ const CONTACT_COMMANDS: [&str; 9] = [
 ];
 const MISC_COMMANDS: [&str; 11] = [
     "getqr",
+    "getqrsvg",
     "getbadqr",
     "checkqr",
     "joinqr",
-    "event",
     "fileinfo",
     "clear",
     "exit",
@@ -294,7 +298,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
@@ -412,7 +416,7 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            let group = arg1.parse::<u32>().ok().map(ChatId::new);
             let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
             if !qr.is_empty() {
                 if arg0 == "getbadqr" && qr.len() > 40 {
@@ -427,6 +431,20 @@ async fn handle_cmd(
                 io::stderr().write_all(&output.stderr).unwrap();
             }
         }
+        "getqrsvg" => {
+            ctx.start_io().await;
+            let group = arg1.parse::<u32>().ok().map(ChatId::new);
+            let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
+            match get_securejoin_qr_svg(&ctx, group).await {
+                Ok(svg) => {
+                    fs::write(&file, svg)?;
+                    println!("QR code svg written to: {:#?}", file);
+                }
+                Err(err) => {
+                    bail!("Failed to get QR code svg: {}", err);
+                }
+            }
+        }
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {

examples/simple.rs

@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
+    let ctx = Context::new(dbfile.into(), 0)
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;

python/install_python_bindings.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
 
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
         subprocess.check_call([

python/mypy.ini

@@ -0,0 +1,23 @@
+[mypy]
+
+[mypy-deltachat.capi.*]
+ignore_missing_imports = True
+
+[mypy-pluggy.*]
+ignore_missing_imports = True
+
+[mypy-cffi.*]
+ignore_missing_imports = True
+
+[mypy-imapclient.*]
+ignore_missing_imports = True
+
+[mypy-pytest.*]
+ignore_missing_imports = True
+
+[mypy-_pytest.*]
+ignore_missing_imports = True
+
+[mypy-imap_tools.*]
+ignore_missing_imports = True
+

python/pyproject.toml

@@ -1,5 +1,6 @@
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
+requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
+build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]
 root = ".."

python/setup.py

@@ -11,7 +11,11 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
+        setup_requires=[
+            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
+            'pkgconfig',
+        ],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -19,9 +19,9 @@ except DistributionNotFound:
 
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
-        for name, val in vars(const).items():
+        for name in dir(const):
             if name.startswith("DC_EVENT_"):
-                _DC_EVENTNAME_MAP[val] = name
+                _DC_EVENTNAME_MAP[getattr(const, name)] = name
     return _DC_EVENTNAME_MAP[integer]
 
 
@@ -75,7 +75,6 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("addr", args.email)
         ac.set_config("mail_pw", args.password)
         ac.set_config("mvbox_move", "0")
-        ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
         ac.set_config("bot", "1")
         configtracker = ac.configure()

python/src/deltachat/_build.py

@@ -8,9 +8,9 @@ import shutil
 import subprocess
 import tempfile
 import textwrap
-import types
 
 import cffi
+import pkgconfig  # type: ignore
 
 
 def local_build_flags(projdir, target):
@@ -19,36 +19,31 @@ def local_build_flags(projdir, target):
     :param projdir: The root directory of the deltachat-core-rust project.
     :param target: The rust build target, `debug` or `release`.
     """
-    flags = types.SimpleNamespace()
+    flags = {}
     if platform.system() == 'Darwin':
-        flags.libs = ['resolv', 'dl']
-        flags.extra_link_args = [
+        flags['libraries'] = ['resolv', 'dl']
+        flags['extra_link_args'] = [
                 '-framework', 'CoreFoundation',
                 '-framework', 'CoreServices',
                 '-framework', 'Security',
         ]
     elif platform.system() == 'Linux':
-        flags.libs = ['rt', 'dl', 'm']
-        flags.extra_link_args = []
+        flags['libraries'] = ['rt', 'dl', 'm']
+        flags['extra_link_args'] = []
     else:
         raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
     target_dir = os.environ.get("CARGO_TARGET_DIR")
     if target_dir is None:
         target_dir = os.path.join(projdir, 'target')
-    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags.objs[0]), flags.objs
-    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    flags['extra_objects'] = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags['extra_objects'][0]), flags['extra_objects']
+    flags['include_dirs'] = [os.path.join(projdir, 'deltachat-ffi')]
     return flags
 
 
 def system_build_flags():
     """Construct build flags for building against an installed libdeltachat."""
-    flags = types.SimpleNamespace()
-    flags.libs = ['deltachat']
-    flags.objs = []
-    flags.incs = []
-    flags.extra_link_args = []
-    return flags
+    return pkgconfig.parse('deltachat')
 
 
 def extract_functions(flags):
@@ -69,7 +64,7 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags.incs,
+                      include_dirs=flags['include_dirs'],
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
             return dst_fp.read()
@@ -105,7 +100,7 @@ def find_header(flags):
         try:
             os.chdir(tmpdir)
             cc.compile(sources=["where.c"],
-                       include_dirs=flags.incs,
+                       include_dirs=flags['include_dirs'],
                        macros=[("PY_CFFI_INC", "1")])
         finally:
             os.chdir(cwd)
@@ -183,10 +178,7 @@ def ffibuilder():
                 return DC_EVENT_DATA2_IS_STRING(e);
             }
         """,
-        include_dirs=flags.incs,
-        libraries=flags.libs,
-        extra_objects=flags.objs,
-        extra_link_args=flags.extra_link_args,
+        **flags,
     )
     builder.cdef("""
         typedef int... time_t;

python/src/deltachat/account.py

@@ -8,13 +8,14 @@ import os
 from array import array
 from . import const
 from .capi import ffi, lib
-from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
+from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
 from .tracker import ImexTracker, ConfigureTracker
 from . import hookspec
 from .events import EventThread
+from typing import Union, Any, Dict, Optional, List, Generator
 
 
 class MissingCredentials(ValueError):
@@ -28,7 +29,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True):
+    def __init__(self, db_path, os_name=None, logging=True) -> None:
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -58,11 +59,11 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
-    def disable_logging(self):
+    def disable_logging(self) -> None:
         """ disable logging. """
         self._logging = False
 
-    def enable_logging(self):
+    def enable_logging(self) -> None:
         """ re-enable logging. """
         self._logging = True
 
@@ -73,12 +74,12 @@ class Account(object):
         if self._logging:
             self._pm.hook.ac_log_line(message=msg)
 
-    def _check_config_key(self, name):
+    def _check_config_key(self, name: str) -> None:
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
                            name, self._configkeys))
 
-    def get_info(self):
+    def get_info(self) -> Dict[str, str]:
         """ return dictionary of built config parameters. """
         lines = from_dc_charpointer(lib.dc_get_info(self._dc_context))
         d = {}
@@ -105,19 +106,19 @@ class Account(object):
             cursor += len(entry) + 1
         log("")
 
-    def set_stock_translation(self, id, string):
+    def set_stock_translation(self, id: int, string: str) -> None:
         """ set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
         :returns: None
         """
-        string = string.encode("utf8")
-        res = lib.dc_set_stock_translation(self._dc_context, id, string)
+        bytestring = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, bytestring)
         if res == 0:
             raise ValueError("could not set translation string")
 
-    def set_config(self, name, value):
+    def set_config(self, name: str, value: Optional[str]) -> None:
         """ set configuration values.
 
         :param name: config key name (unicode)
@@ -125,16 +126,16 @@ class Account(object):
         :returns: None
         """
         self._check_config_key(name)
-        name = name.encode("utf8")
-        if name == b"addr" and self.is_configured():
+        namebytes = name.encode("utf8")
+        if namebytes == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
         if value is not None:
-            value = value.encode("utf8")
+            valuebytes = value.encode("utf8")
         else:
-            value = ffi.NULL
-        lib.dc_set_config(self._dc_context, name, value)
+            valuebytes = ffi.NULL
+        lib.dc_set_config(self._dc_context, namebytes, valuebytes)
 
-    def get_config(self, name):
+    def get_config(self, name: str) -> str:
         """ return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
@@ -143,12 +144,12 @@ class Account(object):
         """
         if name != "sys.config_keys":
             self._check_config_key(name)
-        name = name.encode("utf8")
-        res = lib.dc_get_config(self._dc_context, name)
+        namebytes = name.encode("utf8")
+        res = lib.dc_get_config(self._dc_context, namebytes)
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr, public, secret):
+    def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None:
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
@@ -160,7 +161,7 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def update_config(self, kwargs):
+    def update_config(self, kwargs: Dict[str, Any]) -> None:
         """ update config values.
 
         :param kwargs: name=value config settings for this account.
@@ -170,7 +171,7 @@ class Account(object):
         for key, value in kwargs.items():
             self.set_config(key, str(value))
 
-    def is_configured(self):
+    def is_configured(self) -> bool:
         """ determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
@@ -178,7 +179,13 @@ class Account(object):
         """
         return True if lib.dc_is_configured(self._dc_context) else False
 
-    def set_avatar(self, img_path):
+    def is_open(self) -> bool:
+        """Determine if account is open
+
+        :returns True if account is open."""
+        return True if lib.dc_context_is_open(self._dc_context) else False
+
+    def set_avatar(self, img_path: Optional[str]) -> None:
         """Set self avatar.
 
         :raises ValueError: if profile image could not be set
@@ -190,20 +197,18 @@ class Account(object):
             assert os.path.exists(img_path), img_path
             self.set_config("selfavatar", img_path)
 
-    def check_is_configured(self):
+    def check_is_configured(self) -> None:
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_latest_backupfile(self, backupdir):
+    def get_latest_backupfile(self, backupdir) -> Optional[str]:
         """ return the latest backup file in a given directory.
         """
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
-        if res == ffi.NULL:
-            return None
-        return from_dc_charpointer(res)
+        return from_optional_dc_charpointer(res)
 
-    def get_blobdir(self):
+    def get_blobdir(self) -> str:
         """ return the directory for files.
 
         All sent files are copied to this directory if necessary.
@@ -211,15 +216,15 @@ class Account(object):
         """
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
-    def get_self_contact(self):
+    def get_self_contact(self) -> Contact:
         """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name=None):
-        """ create a (new) Contact or return an existing one.
+    def create_contact(self, obj, name: Optional[str] = None) -> Contact:
+        """create a (new) Contact or return an existing one.
 
         Calling this method will always result in the same
         underlying contact id.  If there already is a Contact
@@ -236,13 +241,13 @@ class Account(object):
         contact_id = lib.dc_create_contact(self._dc_context, name, addr)
         return Contact(self, contact_id)
 
-    def get_contact(self, obj):
+    def get_contact(self, obj) -> Optional[Contact]:
         if isinstance(obj, Contact):
             return obj
         (_, addr) = self.get_contact_addr_and_name(obj)
         return self.get_contact_by_addr(addr)
 
-    def get_contact_addr_and_name(self, obj, name=None):
+    def get_contact_addr_and_name(self, obj, name: Optional[str] = None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -260,7 +265,7 @@ class Account(object):
             name = displayname
         return (name, addr)
 
-    def delete_contact(self, contact):
+    def delete_contact(self, contact: Contact) -> bool:
         """ delete a Contact.
 
         :param contact: contact object obtained
@@ -271,22 +276,23 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email):
+    def get_contact_by_addr(self, email: str) -> Optional[Contact]:
         """ get a contact for the email address or None if it's blocked or doesn't exist. """
         _, addr = parseaddr(email)
         addr = as_dc_charpointer(addr)
         contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
         if contact_id:
             return self.get_contact_by_id(contact_id)
+        return None
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
+    def get_contact_by_id(self, contact_id: int) -> Contact:
+        """ return Contact instance or raise an exception.
         :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
+        :returns: :class:`deltachat.contact.Contact` instance.
         """
         return Contact(self, contact_id)
 
-    def get_blocked_contacts(self):
+    def get_blocked_contacts(self) -> List[Contact]:
         """ return a list of all blocked contacts.
 
         :returns: list of :class:`deltachat.contact.Contact` objects.
@@ -297,8 +303,13 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_contacts(self, query=None, with_self=False, only_verified=False):
-        """ get a (filtered) list of contacts.
+    def get_contacts(
+        self,
+        query: Optional[str] = None,
+        with_self: bool = False,
+        only_verified: bool = False,
+    ) -> List[Contact]:
+        """get a (filtered) list of contacts.
 
         :param query: if a string is specified, only return contacts
                       whose name or e-mail matches query.
@@ -318,7 +329,7 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_fresh_messages(self):
+    def get_fresh_messages(self) -> Generator[Message, None, None]:
         """ yield all fresh messages from all chats. """
         dc_array = ffi.gc(
             lib.dc_get_fresh_msgs(self._dc_context),
@@ -326,12 +337,17 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj):
+    def create_chat(self, obj) -> Chat:
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
-    def create_group_chat(self, name, contacts=None, verified=False):
-        """ create a new group chat object.
+    def create_group_chat(
+        self,
+        name: str,
+        contacts: Optional[List[Contact]] = None,
+        verified: bool = False,
+    ) -> Chat:
+        """create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
@@ -347,7 +363,7 @@ class Account(object):
                 chat.add_contact(contact)
         return chat
 
-    def get_chats(self):
+    def get_chats(self) -> List[Chat]:
         """ return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
@@ -364,17 +380,17 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self):
+    def get_device_chat(self) -> Chat:
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
-    def get_message_by_id(self, msg_id):
+    def get_message_by_id(self, msg_id: int) -> Message:
         """ return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
-    def get_chat_by_id(self, chat_id):
+    def get_chat_by_id(self, chat_id: int) -> Chat:
         """ return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
@@ -386,19 +402,21 @@ class Account(object):
         lib.dc_chat_unref(res)
         return Chat(self, chat_id)
 
-    def mark_seen_messages(self, messages):
+    def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
         """ mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
         arr = array("i")
         for msg in messages:
-            msg = getattr(msg, "id", msg)
-            arr.append(msg)
+            if isinstance(msg, Message):
+                arr.append(msg.id)
+            else:
+                arr.append(msg)
         msg_ids = ffi.cast("uint32_t*", ffi.from_buffer(arr))
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
-    def forward_messages(self, messages, chat):
+    def forward_messages(self, messages: List[Message], chat: Chat) -> None:
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -408,7 +426,7 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
-    def delete_messages(self, messages):
+    def delete_messages(self, messages: List[Message]) -> None:
         """ delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -466,7 +484,7 @@ class Account(object):
     def imex(self, path, imex_cmd):
         lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
 
-    def initiate_key_transfer(self):
+    def initiate_key_transfer(self) -> str:
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
@@ -477,7 +495,7 @@ class Account(object):
             raise RuntimeError("could not send out autocrypt setup message")
         return from_dc_charpointer(res)
 
-    def get_setup_contact_qr(self):
+    def get_setup_contact_qr(self) -> str:
         """ get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -527,7 +545,9 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
+    def set_location(
+        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
+    ) -> None:
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -571,7 +591,7 @@ class Account(object):
     def get_connectivity(self):
         return lib.dc_get_connectivity(self._dc_context)
 
-    def get_connectivity_html(self):
+    def get_connectivity_html(self) -> str:
         return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
 
     def all_work_done(self):
@@ -621,7 +641,7 @@ class Account(object):
         """
         lib.dc_maybe_network(self._dc_context)
 
-    def configure(self, reconfigure=False):
+    def configure(self, reconfigure: bool = False) -> ConfigureTracker:
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
@@ -634,11 +654,11 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def wait_shutdown(self):
+    def wait_shutdown(self) -> None:
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def stop_io(self):
+    def stop_io(self) -> None:
         """ stop core IO scheduler if it is running. """
         self.log("stop_ongoing")
         self.stop_ongoing()
@@ -646,7 +666,7 @@ class Account(object):
         self.log("dc_stop_io (stop core IO scheduler)")
         lib.dc_stop_io(self._dc_context)
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         """ shutdown and destroy account (stop callback thread, close and remove
         underlying dc_context)."""
         if self._dc_context is None:

python/src/deltachat/chat.py

@@ -5,10 +5,11 @@ import calendar
 import json
 from datetime import datetime, timezone
 import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
+from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
+from typing import Optional
 
 
 class Chat(object):
@@ -17,20 +18,20 @@ class Chat(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id):
+    def __init__(self, account, id) -> None:
         from .account import Account
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         return self.id == getattr(other, "id", None) and \
                self.account._dc_context == other.account._dc_context
 
-    def __ne__(self, other):
+    def __ne__(self, other) -> bool:
         return not (self == other)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
@@ -40,7 +41,7 @@ class Chat(object):
             lib.dc_chat_unref
         )
 
-    def delete(self):
+    def delete(self) -> None:
         """Delete this chat and all its messages.
 
         Note:
@@ -50,24 +51,24 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
-    def block(self):
+    def block(self) -> None:
         """Block this chat."""
         lib.dc_block_chat(self.account._dc_context, self.id)
 
-    def accept(self):
+    def accept(self) -> None:
         """Accept this contact request chat."""
         lib.dc_accept_chat(self.account._dc_context, self.id)
 
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self):
+    def is_group(self) -> bool:
         """ return true if this chat is a group chat.
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
-    def is_muted(self):
+    def is_muted(self) -> bool:
         """ return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
@@ -90,7 +91,7 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self):
+    def can_send(self) -> bool:
         """Check if messages can be sent to a give chat.
         This is not true eg. for the contact requests or for the device-talk
 
@@ -98,30 +99,30 @@ class Chat(object):
         """
         return lib.dc_chat_can_send(self._dc_chat)
 
-    def is_protected(self):
+    def is_protected(self) -> bool:
         """ return True if this chat is a protected chat.
 
         :returns: True if chat is protected, False otherwise.
         """
         return lib.dc_chat_is_protected(self._dc_chat)
 
-    def get_name(self):
+    def get_name(self) -> Optional[str]:
         """ return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
-    def set_name(self, name):
+    def set_name(self, name: str) -> bool:
         """ set name of this chat.
 
         :param name: as a unicode string.
-        :returns: None
+        :returns: True on success, False otherwise
         """
         name = as_dc_charpointer(name)
-        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
+        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
 
-    def mute(self, duration=None):
+    def mute(self, duration: Optional[int] = None) -> None:
         """ mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
@@ -135,7 +136,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
-    def unmute(self):
+    def unmute(self) -> None:
         """ unmutes the chat
 
         :returns: None
@@ -144,7 +145,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
-    def get_mute_duration(self):
+    def get_mute_duration(self) -> int:
         """ Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
@@ -152,37 +153,37 @@ class Chat(object):
         """
         return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
 
-    def get_ephemeral_timer(self):
+    def get_ephemeral_timer(self) -> int:
         """ get ephemeral timer.
 
         :returns: ephemeral timer value in seconds
         """
         return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
 
-    def set_ephemeral_timer(self, timer):
+    def set_ephemeral_timer(self, timer: int) -> bool:
         """ set ephemeral timer.
 
         :param: timer value in seconds
 
-        :returns: None
+        :returns: True on success, False otherwise
         """
-        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
+        return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
 
-    def get_type(self):
+    def get_type(self) -> int:
         """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self):
+    def get_encryption_info(self) -> Optional[str]:
         """Return encryption info for this chat.
 
         :returns: a string with encryption preferences of all chat members"""
         res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
         return from_dc_charpointer(res)
 
-    def get_join_qr(self):
+    def get_join_qr(self) -> Optional[str]:
         """ get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -194,7 +195,7 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
-    def send_msg(self, msg):
+    def send_msg(self, msg: Message) -> Message:
         """send a message by using a ready Message object.
 
         :param msg: a :class:`deltachat.message.Message` instance
@@ -516,7 +517,7 @@ class Chat(object):
                     lib.dc_array_get_timestamp(dc_array, i),
                     timezone.utc
                 ),
-                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+                marker=from_optional_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]

python/src/deltachat/const.py

@@ -1,7 +1,13 @@
+from typing import Any, List
+
 from .capi import lib
 
 
-for name in dir(lib):
+def __getattr__(name: str) -> Any:
     if name.startswith("DC_"):
-        globals()[name] = getattr(lib, name)
-del name
+        return getattr(lib, name)
+    return globals()[name]
+
+
+def __dir__() -> List[str]:
+    return sorted(name for name in dir(lib) if name.startswith("DC_"))

python/src/deltachat/contact.py

@@ -1,10 +1,12 @@
 """ Contact object. """
 
-from . import props
-from .cutil import from_dc_charpointer
-from .capi import lib, ffi
+from datetime import date, datetime, timezone
+from typing import Optional
+
+from . import const, props
+from .capi import ffi, lib
 from .chat import Chat
-from . import const
+from .cutil import from_dc_charpointer, from_optional_dc_charpointer
 
 
 class Contact(object):
@@ -35,18 +37,25 @@ class Contact(object):
         )
 
     @props.with_doc
-    def addr(self):
+    def addr(self) -> str:
         """ normalized e-mail address for this account. """
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self):
+    def name(self) -> str:
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
     # deprecated alias
     display_name = name
 
+    @props.with_doc
+    def last_seen(self) -> date:
+        """Last seen timestamp."""
+        return datetime.fromtimestamp(
+            lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc
+        )
+
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -67,15 +76,13 @@ class Contact(object):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
 
-    def get_profile_image(self):
+    def get_profile_image(self) -> Optional[str]:
         """Get contact profile image.
 
         :returns: path to profile image, None if no profile image exists.
         """
         dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
-        if dc_res == ffi.NULL:
-            return None
-        return from_dc_charpointer(dc_res)
+        return from_optional_dc_charpointer(dc_res)
 
     @property
     def status(self):

python/src/deltachat/cutil.py

@@ -1,6 +1,9 @@
 from .capi import lib
 from .capi import ffi
 from datetime import datetime, timezone
+from typing import Optional, TypeVar, Generator, Callable
+
+T = TypeVar('T')
 
 
 def as_dc_charpointer(obj):
@@ -11,21 +14,28 @@ def as_dc_charpointer(obj):
     return obj
 
 
-def iter_array(dc_array_t, constructor):
+def iter_array(dc_array_t, constructor: Callable[[int], T]) -> Generator[T, None, None]:
     for i in range(0, lib.dc_array_get_cnt(dc_array_t)):
         yield constructor(lib.dc_array_get_id(dc_array_t, i))
 
 
-def from_dc_charpointer(obj):
+def from_dc_charpointer(obj) -> str:
     if obj != ffi.NULL:
         return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    raise ValueError
+
+
+def from_optional_dc_charpointer(obj) -> Optional[str]:
+    if obj != ffi.NULL:
+        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    return None
 
 
 class DCLot:
-    def __init__(self, dc_lot):
+    def __init__(self, dc_lot) -> None:
         self._dc_lot = dc_lot
 
-    def id(self):
+    def id(self) -> int:
         return lib.dc_lot_get_id(self._dc_lot)
 
     def state(self):

python/src/deltachat/direct_imap.py

@@ -4,25 +4,22 @@ and for cleaning up inbox/mvbox for each test function run.
 """
 
 import io
-import email
 import ssl
 import pathlib
-from imapclient import IMAPClient
-from imapclient.exceptions import IMAPClientError
+from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
 import imaplib
 import deltachat
-from deltachat import const
+from deltachat import const, Account
+from typing import List
 
 
-SEEN = b'\\Seen'
-DELETED = b'\\Deleted'
 FLAGS = b'FLAGS'
 FETCH = b'FETCH'
 ALL = "1:*"
 
 
 @deltachat.global_hookimpl
-def dc_account_extra_configure(account):
+def dc_account_extra_configure(account: Account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
@@ -36,12 +33,10 @@ def dc_account_extra_configure(account):
                     assert imap.select_folder(folder)
                     imap.delete(ALL, expunge=True)
                 else:
-                    imap.conn.delete_folder(folder)
+                    imap.conn.folder.delete(folder)
                     # We just deleted the folder, so we have to make DC forget about it, too
                     if account.get_config("configured_sentbox_folder") == folder:
                         account.set_config("configured_sentbox_folder", None)
-                    if account.get_config("configured_spam_folder") == folder:
-                        account.set_config("configured_spam_folder", None)
 
             setattr(account, "direct_imap", imap)
 
@@ -62,7 +57,7 @@ def dc_account_after_shutdown(account):
 
 
 class DirectImap:
-    def __init__(self, account):
+    def __init__(self, account: Account) -> None:
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
         self._idling = False
@@ -88,37 +83,34 @@ class DirectImap:
             ssl_context.verify_mode = ssl.CERT_NONE
 
         if security == const.DC_SOCKET_STARTTLS:
-            self.conn = IMAPClient(host, port, ssl=False)
-            self.conn.starttls(ssl_context)
-        elif security == const.DC_SOCKET_PLAIN:
-            self.conn = IMAPClient(host, port, ssl=False)
-        elif security == const.DC_SOCKET_SSL:
-            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
+            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
+        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
+            self.conn = MailBox(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
 
     def shutdown(self):
         try:
-            self.conn.idle_done()
-        except (OSError, IMAPClientError):
+            self.idle_done()
+        except (OSError, imaplib.IMAP4.abort):
             pass
         try:
             self.conn.logout()
-        except (OSError, IMAPClientError):
+        except (OSError, imaplib.IMAP4.abort):
             print("Could not logout direct_imap conn")
 
     def create_folder(self, foldername):
         try:
-            self.conn.create_folder(foldername)
-        except imaplib.IMAP4.error as e:
+            self.conn.folder.create(foldername)
+        except errors.MailboxFolderCreateError as e:
             print("Can't create", foldername, "probably it already exists:", str(e))
 
-    def select_folder(self, foldername):
+    def select_folder(self, foldername: str) -> tuple:
         assert not self._idling
-        return self.conn.select_folder(foldername)
+        return self.conn.folder.set(foldername)
 
-    def select_config_folder(self, config_name):
+    def select_config_folder(self, config_name: str):
         """ Return info about selected folder if it is
         configured, otherwise None. """
         if "_" not in config_name:
@@ -127,50 +119,36 @@ class DirectImap:
         if foldername:
             return self.select_folder(foldername)
 
-    def list_folders(self):
+    def list_folders(self) -> List[str]:
         """ return list of all existing folder names"""
         assert not self._idling
-        folders = []
-        for meta, sep, foldername in self.conn.list_folders():
-            folders.append(foldername)
-        return folders
+        return [folder.name for folder in self.conn.folder.list()]
 
-    def delete(self, range, expunge=True):
+    def delete(self, uid_list: str, expunge=True):
         """ delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.set_flags(range, [DELETED])
+        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
         if expunge:
             self.conn.expunge()
 
-    def get_all_messages(self):
+    def get_all_messages(self) -> List[MailMessage]:
         assert not self._idling
+        return [mail for mail in self.conn.fetch()]
 
-        # Flush unsolicited responses. IMAPClient has problems
-        # dealing with them: https://github.com/mjs/imapclient/issues/334
-        # When this NOOP was introduced, next FETCH returned empty
-        # result instead of a single message, even though IMAP server
-        # can only return more untagged responses than required, not
-        # less.
-        self.conn.noop()
-
-        return self.conn.fetch(ALL, [FLAGS])
-
-    def get_unread_messages(self):
+    def get_unread_messages(self) -> List[str]:
         assert not self._idling
-        res = self.conn.fetch(ALL, [FLAGS])
-        return [uid for uid in res
-                if SEEN not in res[uid][FLAGS]]
+        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
         if messages:
-            res = self.conn.set_flags(messages, [SEEN])
+            res = self.conn.flag(messages, MailMessageFlags.SEEN, True)
             print("marked seen:", messages, res)
 
-    def get_unread_cnt(self):
+    def get_unread_cnt(self) -> int:
         return len(self.get_unread_messages())
 
     def dump_imap_structures(self, dir, logfile):
@@ -192,21 +170,20 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[]', FLAGS]
-            for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[]']
-                if not body_bytes:
-                    log("Message", uid, "has empty body")
+            for msg in self.conn.fetch(mark_seen=False):
+                body = getattr(msg.obj, "text", None)
+                if not body:
+                    body = getattr(msg.obj, "html", None)
+                if not body:
+                    log("Message", msg.uid, "has empty body")
                     continue
 
-                flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(uid))
-                fn.write_bytes(body_bytes)
-                log("Message", uid, fn)
-                email_message = email.message_from_bytes(body_bytes)
-                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+                fn = path.joinpath(str(msg.uid))
+                fn.write_bytes(body)
+                log("Message", msg.uid, fn)
+                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
@@ -216,51 +193,58 @@ class DirectImap:
     def idle_start(self):
         """ switch this connection to idle mode. non-blocking. """
         assert not self._idling
-        res = self.conn.idle()
+        res = self.conn.idle.start()
         self._idling = True
         return res
 
-    def idle_check(self, terminate=False):
+    def idle_check(self, terminate=False, timeout=None) -> List[bytes]:
         """ (blocking) wait for next idle message from server. """
         assert self._idling
         self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check(timeout=30)
-        if len(res) == 0:
-            raise TimeoutError
+        res = self.conn.idle.poll(timeout=timeout)
         if terminate:
             self.idle_done()
         self.account.log("imap-direct: idle_check returned {!r}".format(res))
         return res
 
-    def idle_wait_for_seen(self):
-        """ Return first message with SEEN flag
-        from a running idle-stream REtiurn.
+    def idle_wait_for_new_message(self, terminate=False, timeout=None) -> bytes:
+        while 1:
+            for item in self.idle_check(timeout=timeout):
+                if b'EXISTS' in item or b'RECENT' in item:
+                    if terminate:
+                        self.idle_done()
+                    return item
+
+    def idle_wait_for_seen(self, terminate=False, timeout=None) -> int:
+        """ Return first message with SEEN flag from a running idle-stream.
         """
         while 1:
-            for item in self.idle_check():
-                if item[1] == FETCH:
-                    if item[2][0] == FLAGS:
-                        if SEEN in item[2][1]:
-                            return item[0]
+            for item in self.idle_check(timeout=timeout):
+                if FETCH in item:
+                    self.account.log(str(item))
+                    if FLAGS in item and rb'\Seen' in item:
+                        if terminate:
+                            self.idle_done()
+                        return int(item.split(b' ')[1])
 
     def idle_done(self):
         """ send idle-done to server if we are currently in idle mode. """
         if self._idling:
-            res = self.conn.idle_done()
+            res = self.conn.idle.stop()
             self._idling = False
             return res
 
-    def append(self, folder, msg):
+    def append(self, folder: str, msg: str):
         """Upload a message to *folder*.
         Trailing whitespace or a linebreak at the beginning will be removed automatically.
         """
         if msg.startswith("\n"):
             msg = msg[1:]
         msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(folder, msg)
+        self.conn.append(bytes(msg, encoding='ascii'), folder)
 
-    def get_uid_by_message_id(self, message_id):
-        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
+    def get_uid_by_message_id(self, message_id) -> str:
+        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
         if len(msgs) == 0:
             raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
         return msgs[0]

python/src/deltachat/events.py

@@ -9,11 +9,11 @@ from .hookspec import account_hookimpl
 from contextlib import contextmanager
 from .capi import ffi, lib
 from .message import map_system_message
-from .cutil import from_dc_charpointer
+from .cutil import from_optional_dc_charpointer
 
 
 class FFIEvent:
-    def __init__(self, name, data1, data2):
+    def __init__(self, name: str, data1, data2):
         self.name = name
         self.data1 = data1
         self.data2 = data2
@@ -29,18 +29,18 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account):
+    def __init__(self, account) -> None:
         self.account = account
         self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
+    def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
         self.account.log(str(ffi_event))
 
     @account_hookimpl
     def ac_log_line(self, message):
-        t = threading.currentThread()
+        t = threading.current_thread()
         tname = getattr(t, "name", t)
         if tname == "MainThread":
             tname = "MAIN"
@@ -69,7 +69,7 @@ class FFIEventTracker:
         self._event_queue = Queue()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
+    def ac_process_ffi_event(self, ffi_event: FFIEvent):
         self._event_queue.put(ffi_event)
 
     def set_timeout(self, timeout):
@@ -96,7 +96,7 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_contains(self, regex):
+    def get_info_contains(self, regex: str) -> FFIEvent:
         rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
@@ -176,6 +176,7 @@ class FFIEventTracker:
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
+        return None
 
     def wait_msg_delivered(self, msg):
         ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
@@ -189,10 +190,10 @@ class EventThread(threading.Thread):
 
     With each Account init this callback thread is started.
     """
-    def __init__(self, account):
+    def __init__(self, account) -> None:
         self.account = account
         super(EventThread, self).__init__(name="events")
-        self.setDaemon(True)
+        self.daemon = True
         self._marked_for_shutdown = False
         self.start()
 
@@ -202,17 +203,17 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def mark_shutdown(self):
+    def mark_shutdown(self) -> None:
         self._marked_for_shutdown = True
 
-    def wait(self, timeout=None):
+    def wait(self, timeout=None) -> None:
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
         self.join(timeout=timeout)
 
-    def run(self):
+    def run(self) -> None:
         """ get and run events until shutdown. """
         with self.log_execution("EVENT THREAD"):
             self._inner_run()
@@ -234,7 +235,7 @@ class EventThread(threading.Thread):
             # function which provides us signature info of an event call
             evt_name = deltachat.get_dc_event_name(evt)
             if lib.dc_event_has_string_data(evt):
-                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
+                data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event))
             else:
                 data2 = lib.dc_event_get_data2_int(event)
 
@@ -250,7 +251,7 @@ class EventThread(threading.Thread):
                 if self.account._dc_context is not None:
                     raise
 
-    def _map_ffi_event(self, ffi_event):
+    def _map_ffi_event(self, ffi_event: FFIEvent):
         name = ffi_event.name
         account = self.account
         if name == "DC_EVENT_CONFIGURE_PROGRESS":

python/src/deltachat/message.py

@@ -3,10 +3,11 @@
 import os
 import re
 from . import props
-from .cutil import from_dc_charpointer, as_dc_charpointer
+from .cutil import from_dc_charpointer, from_optional_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
 from . import const
 from datetime import datetime, timezone
+from typing import Optional
 
 
 class Message(object):
@@ -75,7 +76,7 @@ class Message(object):
         return lib.dc_msg_get_id(self._dc_msg)
 
     @props.with_doc
-    def text(self):
+    def text(self) -> str:
         """unicode text of this messages (might be empty if not a text message). """
         return from_dc_charpointer(lib.dc_msg_get_text(self._dc_msg))
 
@@ -84,9 +85,9 @@ class Message(object):
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
     @props.with_doc
-    def html(self):
+    def html(self) -> str:
         """html text of this messages (might be empty if not an html message). """
-        return from_dc_charpointer(
+        return from_optional_dc_charpointer(
             lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
 
     def has_html(self):
@@ -113,12 +114,13 @@ class Message(object):
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
-    def basename(self):
+    def basename(self) -> str:
         """basename of the attachment if it exists, otherwise empty string. """
+        # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 
     @props.with_doc
-    def filemime(self):
+    def filemime(self) -> str:
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
@@ -130,7 +132,7 @@ class Message(object):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
-    def get_setupcodebegin(self):
+    def get_setupcodebegin(self) -> str:
         """ return the first characters of a setup code in a setup message. """
         return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
 
@@ -146,7 +148,7 @@ class Message(object):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
 
-    def get_message_info(self):
+    def get_message_info(self) -> str:
         """ Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
@@ -203,11 +205,11 @@ class Message(object):
             return datetime.fromtimestamp(ts, timezone.utc)
 
     @property
-    def quoted_text(self):
+    def quoted_text(self) -> Optional[str]:
         """Text inside the quote
 
         :returns: Quoted text"""
-        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
+        return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
 
     @property
     def quote(self):
@@ -223,6 +225,10 @@ class Message(object):
         """Quote setter"""
         lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
 
+    def force_plaintext(self) -> None:
+        """Force the message to be sent in plain text."""
+        lib.dc_msg_force_plaintext(self._dc_msg)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -240,9 +246,9 @@ class Message(object):
             return email.message_from_string(s)
 
     @property
-    def error(self):
+    def error(self) -> Optional[str]:
         """Error message"""
-        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
+        return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
 
     @property
     def chat(self):
@@ -255,12 +261,12 @@ class Message(object):
         return Chat(self.account, chat_id)
 
     @props.with_doc
-    def override_sender_name(self):
+    def override_sender_name(self) -> Optional[str]:
         """the name that should be shown over the message instead of the contact display name.
 
         Usually used to impersonate someone else.
         """
-        return from_dc_charpointer(
+        return from_optional_dc_charpointer(
             lib.dc_msg_get_override_sender_name(self._dc_msg))
 
     def set_override_sender_name(self, name):

python/src/deltachat/provider.py

@@ -14,7 +14,7 @@ class Provider(object):
     :param domain: The email to get the provider info for.
     """
 
-    def __init__(self, account, addr):
+    def __init__(self, account, addr) -> None:
         provider = ffi.gc(
             lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
             lib.dc_provider_unref,
@@ -24,19 +24,19 @@ class Provider(object):
         self._provider = provider
 
     @property
-    def overview_page(self):
+    def overview_page(self) -> str:
         """URL to the overview page of the provider on providers.delta.chat."""
         return from_dc_charpointer(
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self):
+    def get_before_login_hints(self) -> str:
         """Should be shown to the user on login."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hints(self._provider))
+            lib.dc_provider_get_before_login_hint(self._provider))
 
     @property
-    def status(self):
+    def status(self) -> int:
         """The status of the provider information.
 
         This is one of the

python/src/deltachat/testplugin.py

@@ -9,6 +9,7 @@ import fnmatch
 import time
 import weakref
 import tempfile
+from typing import List, Dict, Callable
 
 import pytest
 import requests
@@ -126,7 +127,7 @@ def pytest_report_header(config, startdir):
 
 
 class SessionLiveConfigFromFile:
-    def __init__(self, fn):
+    def __init__(self, fn) -> None:
         self.fn = fn
         self.configlist = []
         for line in open(fn):
@@ -137,19 +138,21 @@ class SessionLiveConfigFromFile:
                     d[name] = value
                 self.configlist.append(d)
 
-    def get(self, index):
+    def get(self, index: int):
         return self.configlist[index]
 
-    def exists(self):
+    def exists(self) -> bool:
         return bool(self.configlist)
 
 
 class SessionLiveConfigFromURL:
-    def __init__(self, url):
+    configlist: List[Dict[str, str]]
+
+    def __init__(self, url: str) -> None:
         self.configlist = []
         self.url = url
 
-    def get(self, index):
+    def get(self, index: int):
         try:
             return self.configlist[index]
         except IndexError:
@@ -162,7 +165,7 @@ class SessionLiveConfigFromURL:
             self.configlist.append(config)
             return config
 
-    def exists(self):
+    def exists(self) -> bool:
         return bool(self.configlist)
 
 
@@ -179,7 +182,7 @@ def session_liveconfig(request):
 @pytest.fixture
 def data(request):
     class Data:
-        def __init__(self):
+        def __init__(self) -> None:
             # trying to find test data heuristically
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
@@ -210,7 +213,10 @@ def data(request):
 def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
     class AccountMaker:
-        def __init__(self):
+        _finalizers: List[Callable[[], None]]
+        _accounts: List[Account]
+
+        def __init__(self) -> None:
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
@@ -235,7 +241,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._evtracker.set_timeout(30)
             ac.addr = ac.get_self_contact().addr
             ac.set_config("displayname", logid)
             if not quiet:
@@ -297,21 +302,20 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 self._preconfigure_key(ac, configdict['addr'])
             return ac, dict(configdict)
 
-        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
+        def get_online_configuring_account(self, sentbox=False, move=False,
                                            pre_generated_key=True, quiet=False, config={}):
             ac, configdict = self.get_online_config(
                 pre_generated_key=pre_generated_key, quiet=quiet)
             configdict.update(config)
-            configdict["mvbox_watch"] = str(int(mvbox))
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
             ac._configtracker = ac.configure()
             return ac
 
-        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
+        def get_one_online_account(self, pre_generated_key=True, move=False):
             ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
+                pre_generated_key=pre_generated_key, move=move)
             self.wait_configure_and_start_io([ac1])
             return ac1
 
@@ -330,7 +334,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
-            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
+            """ Clones addr, mail_pw, mvbox_move, sentbox_watch and the
             direct_imap object of an online account. This simulates the user setting
             up a new device without importing a backup.
 
@@ -345,7 +349,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
-                mvbox_watch=account.get_config("mvbox_watch"),
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
@@ -423,7 +426,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                         pass
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
-        def get_accepted_chat(self, ac1, ac2):
+        def get_accepted_chat(self, ac1: Account, ac2: Account):
             ac2.create_chat(ac1)
             return ac1.create_chat(ac2)
 
@@ -451,7 +454,9 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
 
 class BotProcess:
-    def __init__(self, popen, bot_cfg):
+    stdout_queue: queue.Queue
+
+    def __init__(self, popen, bot_cfg) -> None:
         self.popen = popen
         self.addr = bot_cfg["addr"]
 
@@ -459,10 +464,10 @@ class BotProcess:
         # the (unicode) lines available for readers through a queue.
         self.stdout_queue = queue.Queue()
         self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.setDaemon(1)
+        t.daemon = True
         t.start()
 
-    def _run_stdout_thread(self):
+    def _run_stdout_thread(self) -> None:
         try:
             while 1:
                 line = self.popen.stdout.readline()
@@ -474,10 +479,10 @@ class BotProcess:
         finally:
             self.stdout_queue.put(None)
 
-    def kill(self):
+    def kill(self) -> None:
         self.popen.kill()
 
-    def wait(self, timeout=30):
+    def wait(self, timeout=None) -> None:
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
@@ -486,7 +491,7 @@ class BotProcess:
             print("+++FNMATCH:", next_pattern)
             ignored = []
             while 1:
-                line = self.stdout_queue.get(timeout=15)
+                line = self.stdout_queue.get()
                 if line is None:
                     if ignored:
                         print("BOT stdout terminated after these lines")
@@ -509,14 +514,14 @@ def tmp_db_path(tmpdir):
 @pytest.fixture
 def lp():
     class Printer:
-        def sec(self, msg):
+        def sec(self, msg: str) -> None:
             print()
             print("=" * 10, msg, "=" * 10)
 
-        def step(self, msg):
+        def step(self, msg: str) -> None:
             print("-" * 5, "step " + msg, "-" * 5)
 
-        def indent(self, msg):
+        def indent(self, msg: str) -> None:
             print("  " + msg)
 
     return Printer()

python/src/deltachat/tracker.py

@@ -90,11 +90,11 @@ class ConfigureTracker:
             if data1 is None or evdata == data1:
                 break
 
-    def wait_finish(self):
+    def wait_finish(self, timeout=None):
         """ wait until configure is completed.
 
         Raise Exception if Configure failed
         """
-        if not self._configure_events.get():
+        if not self._configure_events.get(timeout=timeout):
             content = "\n".join(map(str, self._ffi_events))
             raise ConfigureFailed(content)

python/tests/bench_test_setup.py

@@ -0,0 +1,18 @@
+"""
+run with:
+
+     pytest -vv --durations=10 bench_empty.py
+
+to see timings of test setups.
+"""
+
+import pytest
+
+
+class TestEmpty:
+    def test_prepare_setup_measurings(self, acfactory):
+        acfactory.get_many_online_accounts(5)
+
+    @pytest.mark.parametrize("num", range(0, 5))
+    def test_setup_online_accounts(self, acfactory, num):
+        acfactory.get_many_online_accounts(num)

python/tests/test_account.py

@@ -11,6 +11,7 @@ from deltachat.hookspec import account_hookimpl
 from deltachat.capi import ffi, lib
 from deltachat.cutil import iter_array
 from datetime import datetime, timedelta, timezone
+from imap_tools import AND, U
 
 
 @pytest.mark.parametrize("msgtext,res", [
@@ -41,8 +42,8 @@ class TestOfflineAccountBasic:
     def test_wrong_db(self, tmpdir):
         p = tmpdir.join("hello.db")
         p.write("123")
-        with pytest.raises(ValueError):
-            Account(p.strpath)
+        account = Account(p.strpath)
+        assert not account.is_open()
 
     def test_os_name(self, tmpdir):
         p = tmpdir.join("hello.db")
@@ -57,7 +58,7 @@ class TestOfflineAccountBasic:
         alice_public = data.read_path("key/alice-public.asc")
         alice_secret = data.read_path("key/alice-secret.asc")
         assert alice_public and alice_secret
-        ac._preconfigure_keypair("alice@example.com", alice_public, alice_secret)
+        ac._preconfigure_keypair("alice@example.org", alice_public, alice_secret)
 
     def test_getinfo(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
@@ -265,23 +266,23 @@ class TestOfflineChat:
         assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft()
 
     def test_group_chat_creation_with_translation(self, ac1):
-        ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %1$s")
+        ac1.set_stock_translation(const.DC_STR_MSGGRPNAME, "abc %1$s xyz %2$s")
         ac1._evtracker.consume_events()
         with pytest.raises(ValueError):
-            ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %2$s")
+            ac1.set_stock_translation(const.DC_STR_FILE, "xyz %1$s")
+        ac1._evtracker.get_matching("DC_EVENT_WARNING")
+        with pytest.raises(ValueError):
+            ac1.set_stock_translation(const.DC_STR_CONTACT_NOT_VERIFIED, "xyz %2$s")
         ac1._evtracker.get_matching("DC_EVENT_WARNING")
         with pytest.raises(ValueError):
             ac1.set_stock_translation(500, "xyz %1$s")
         ac1._evtracker.get_matching("DC_EVENT_WARNING")
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact2 = ac1.create_contact("some2@example.org", name="some2")
-        chat = ac1.create_group_chat(name="title1", contacts=[contact1, contact2])
-        assert chat.get_name() == "title1"
-        assert contact1 in chat.get_contacts()
-        assert contact2 in chat.get_contacts()
-        assert not chat.is_promoted()
-        msg = chat.get_draft()
-        assert msg.text == "xyz title1"
+        chat = ac1.create_group_chat(name="homework", contacts=[])
+        assert chat.get_name() == "homework"
+        chat.send_text("Now we have a group for homework")
+        assert chat.is_promoted()
+        chat.set_name("Homework")
+        assert chat.get_messages()[-1].text == "abc homework xyz Homework by me."
 
     @pytest.mark.parametrize("verified", [True, False])
     def test_group_chat_qr(self, acfactory, ac1, verified):
@@ -652,8 +653,6 @@ class TestOnlineAccount:
             pre_generated_key=False,
             config={"key_gen_type": str(const.DC_KEY_GEN_ED25519)}
         )
-        # rsa key gen can be slow especially on CI, adjust timeout
-        ac1._evtracker.set_timeout(240)
         acfactory.wait_configure_and_start_io()
         chat = acfactory.get_accepted_chat(ac1, ac2)
 
@@ -737,7 +736,6 @@ class TestOnlineAccount:
         # make sure we are not sending message to ourselves
         assert self_addr not in ev.data2
         assert other_addr in ev.data2
-        ev = ac1._evtracker.get_matching("DC_EVENT_DELETED_BLOB_FILE")
 
         lp.sec("ac1: setting bcc_self=1")
         ac1.set_config("bcc_self", "1")
@@ -753,7 +751,6 @@ class TestOnlineAccount:
         # now make sure we are sending message to ourselves too
         assert self_addr in ev.data2
         assert other_addr in ev.data2
-        ev = ac1._evtracker.get_matching("DC_EVENT_DELETED_BLOB_FILE")
         assert ac1.direct_imap.idle_wait_for_seen()
 
         # Second client receives only second message, but not the first
@@ -860,7 +857,7 @@ class TestOnlineAccount:
 
     def test_mvbox_sentbox_threads(self, acfactory, lp):
         lp.sec("ac1: start with mvbox thread")
-        ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True, sentbox=True)
+        ac1 = acfactory.get_online_configuring_account(move=True, sentbox=True)
 
         lp.sec("ac2: start without mvbox/sentbox threads")
         ac2 = acfactory.get_online_configuring_account()
@@ -874,27 +871,31 @@ class TestOnlineAccount:
 
     def test_move_works(self, acfactory):
         ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account(mvbox=True, move=True)
+        ac2 = acfactory.get_online_configuring_account(move=True)
         acfactory.wait_configure_and_start_io()
         chat = acfactory.get_accepted_chat(ac1, ac2)
         chat.send_text("message1")
-        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
-        assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL
+
+        # Message is moved to the movebox
         ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
+        # Message is downloaded
+        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
+        assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL
+
     def test_move_works_on_self_sent(self, acfactory):
-        ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True)
+        ac1 = acfactory.get_online_configuring_account(move=True)
         ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure_and_start_io()
         ac1.set_config("bcc_self", "1")
 
         chat = acfactory.get_accepted_chat(ac1, ac2)
         chat.send_text("message1")
+        ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
         chat.send_text("message2")
+        ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
         chat.send_text("message3")
         ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-        ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-        ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
     def test_forward_messages(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -953,12 +954,24 @@ class TestOnlineAccount:
         assert msg_in.is_forwarded()
 
     def test_send_self_message(self, acfactory, lp):
-        ac1 = acfactory.get_one_online_account(mvbox=True, move=True)
+        ac1 = acfactory.get_one_online_account(move=True)
         lp.sec("ac1: create self chat")
         chat = ac1.get_self_contact().create_chat()
         chat.send_text("hello")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    def test_send_dot(self, acfactory, lp):
+        """Test that a single dot is properly escaped in SMTP protocol"""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("sending message")
+        msg_out = chat.send_text(".")
+
+        lp.sec("receiving message")
+        msg_in = ac2._evtracker.wait_next_incoming_message()
+        assert msg_in.text == msg_out.text
+
     def test_send_and_receive_message_markseen(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -983,7 +996,8 @@ class TestOnlineAccount:
         assert msg2 in chat2.get_messages()
         assert chat2.is_contact_request()
         assert chat2.count_fresh_messages() == 1
-        assert msg2.time_received >= msg1.time_sent
+        # Like it or not, this assert is flaky
+        # assert msg2.time_received >= msg1.time_sent
 
         lp.sec("create new chat with contact and verify it's proper")
         chat2b = msg2.create_chat()
@@ -1012,7 +1026,7 @@ class TestOnlineAccount:
         assert ev.data1 == msg2.chat.id
         assert ev.data2 == 0
 
-        ac2.direct_imap.idle_check(terminate=True)
+        ac2.direct_imap.idle_wait_for_new_message(terminate=True)
         lp.step("1")
         for i in range(2):
             ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
@@ -1034,30 +1048,88 @@ class TestOnlineAccount:
 
     def test_moved_markseen(self, acfactory, lp):
         """Test that message already moved to DeltaChat folder is marked as seen."""
-        ac1 = acfactory.get_online_configuring_account(mvbox=True, config={"inbox_watch": "0"})
-        ac2 = acfactory.get_online_configuring_account()
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account(move=True)
         acfactory.wait_configure_and_start_io([ac1, ac2])
-        ac1.set_config("bcc_self", "1")
 
-        ac1.direct_imap.idle_start()
+        ac2.stop_io()
+        ac2.direct_imap.idle_start()
+
         ac1.create_chat(ac2).send_text("Hello!")
-        ac1.direct_imap.idle_check(terminate=True)
-        ac1.stop_io()
+
+        ac2.direct_imap.idle_wait_for_new_message(terminate=True)
 
         # Emulate moving of the message to DeltaChat folder by Sieve rule.
         # mailcow server contains this rule by default.
-        ac1.direct_imap.conn.move(["*"], "DeltaChat")
+        ac2.direct_imap.conn.move(["*"], "DeltaChat")
 
-        ac1.direct_imap.select_folder("DeltaChat")
-        ac1.direct_imap.idle_start()
-        ac1.start_io()
-        ac1.direct_imap.idle_wait_for_seen()
-        ac1.direct_imap.idle_done()
+        ac2.direct_imap.select_folder("DeltaChat")
+        ac2.direct_imap.idle_start()
+        ac2.start_io()
+        msg = ac2._evtracker.wait_next_incoming_message()
 
-        fetch = list(ac1.direct_imap.conn.fetch("*", b'FLAGS').values())
-        flags = fetch[-1][b'FLAGS']
-        is_seen = b'\\Seen' in flags
-        assert is_seen
+        # Accept the contact request.
+        msg.chat.accept()
+        ac2.mark_seen_messages([msg])
+        uid = ac2.direct_imap.idle_wait_for_seen(terminate=True)
+
+        assert len([a for a in ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*")))]) == 1
+
+    def test_multidevice_sync_seen(self, acfactory, lp):
+        """Test that message marked as seen on one device is marked as seen on another."""
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        ac1_clone = acfactory.clone_online_account(ac1)
+        acfactory.wait_configure_and_start_io()
+
+        ac1.set_config("bcc_self", "1")
+        ac1_clone.set_config("bcc_self", "1")
+
+        ac1_chat = ac1.create_chat(ac2)
+        ac1_clone_chat = ac1_clone.create_chat(ac2)
+        ac2_chat = ac2.create_chat(ac1)
+
+        lp.sec("Send a message from ac2 to ac1 and check that it's 'fresh'")
+        ac2_chat.send_text("Hi")
+        ac1_message = ac1._evtracker.wait_next_incoming_message()
+        ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
+        assert ac1_chat.count_fresh_messages() == 1
+        assert ac1_clone_chat.count_fresh_messages() == 1
+        assert ac1_message.is_in_fresh
+        assert ac1_clone_message.is_in_fresh
+
+        lp.sec("ac1 marks message as seen on the first device")
+        ac1.mark_seen_messages([ac1_message])
+        assert ac1_message.is_in_seen
+
+        lp.sec("ac1 clone detects that message is marked as seen")
+        ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+        assert ev.data1 == ac1_clone_chat.id
+        assert ac1_clone_message.is_in_seen
+
+        lp.sec("Send an ephemeral message from ac2 to ac1")
+        ac2_chat.set_ephemeral_timer(60)
+        ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        ac1._evtracker.wait_next_incoming_message()
+        ac1_clone._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+        ac1_clone._evtracker.wait_next_incoming_message()
+
+        ac2_chat.send_text("Foobar")
+        ac1_message = ac1._evtracker.wait_next_incoming_message()
+        ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
+        assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
+        assert "Expires: " not in ac1_clone_message.get_message_info()
+        assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
+        assert "Expires: " not in ac1_clone_message.get_message_info()
+
+        ac1.mark_seen_messages([ac1_message])
+        assert ac1_message.is_in_seen
+        assert "Expires: " in ac1_message.get_message_info()
+        ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+        assert ev.data1 == ac1_clone_chat.id
+        assert ac1_clone_message.is_in_seen
+        # Test that the timer is started on the second device after synchronizing the seen status.
+        assert "Expires: " in ac1_clone_message.get_message_info()
 
     def test_message_override_sender_name(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1095,8 +1167,8 @@ class TestOnlineAccount:
     def test_markseen_message_and_mdn(self, acfactory, mvbox_move):
         # Please only change this test if you are very sure that it will still catch the issues it catches now.
         # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
-        ac1 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
-        ac2 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        ac1 = acfactory.get_online_configuring_account(move=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account(move=mvbox_move)
 
         acfactory.wait_configure_and_start_io()
         # Do not send BCC to self, we only want to test MDN on ac1.
@@ -1142,7 +1214,7 @@ class TestOnlineAccount:
         assert not msg_reply1.chat.is_group()
         assert msg_reply1.chat.id == private_chat1.id
 
-    def test_mdn_asymetric(self, acfactory, lp):
+    def test_mdn_asymmetric(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts(move=True)
 
         lp.sec("ac1: create chat with ac2")
@@ -1166,6 +1238,9 @@ class TestOnlineAccount:
 
         assert len(msg.chat.get_messages()) == 1
 
+        ac1.direct_imap.select_config_folder("mvbox")
+        ac1.direct_imap.idle_start()
+
         lp.sec("ac2: mark incoming message as seen")
         ac2.mark_seen_messages([msg])
 
@@ -1175,6 +1250,9 @@ class TestOnlineAccount:
 
         assert len(chat.get_messages()) == 1
 
+        # Wait for the message to be marked as seen on IMAP.
+        assert ac1.direct_imap.idle_wait_for_seen()
+
         # MDN is received even though MDNs are already disabled
         assert msg_out.is_out_mdn_received()
 
@@ -1352,18 +1430,22 @@ class TestOnlineAccount:
         assert not device_chat.can_send()
         assert device_chat.get_draft() is None
 
-    def test_dont_show_emails_in_draft_folder(self, acfactory, lp):
+    def test_dont_show_emails(self, acfactory, lp):
         """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them.
         So: If it's outgoing AND there is no Received header AND it's not in the sentbox, then ignore the email.
 
-        If the draft email is sent out later (i.e. moved to "Sent"), it must be shown."""
+        If the draft email is sent out later (i.e. moved to "Sent"), it must be shown.
+
+        Also, test that unknown emails in the Spam folder are not shown."""
         ac1 = acfactory.get_online_configuring_account()
         ac1.set_config("show_emails", "2")
-        ac1.create_contact("alice@example.com").create_chat()
+        ac1.create_contact("alice@example.org").create_chat()
 
         acfactory.wait_configure(ac1)
         ac1.direct_imap.create_folder("Drafts")
         ac1.direct_imap.create_folder("Sent")
+        ac1.direct_imap.create_folder("Spam")
+        ac1.direct_imap.create_folder("Junk")
 
         acfactory.wait_configure_and_start_io()
         # Wait until each folder was selected once and we are IDLEing again:
@@ -1373,7 +1455,7 @@ class TestOnlineAccount:
         ac1.direct_imap.append("Drafts", """
             From: ac1 <{}>
             Subject: subj
-            To: alice@example.com
+            To: alice@example.org
             Message-ID: <aepiors@example.org>
             Content-Type: text/plain; charset=utf-8
 
@@ -1382,12 +1464,30 @@ class TestOnlineAccount:
         ac1.direct_imap.append("Sent", """
             From: ac1 <{}>
             Subject: subj
-            To: alice@example.com
+            To: alice@example.org
             Message-ID: <hsabaeni@example.org>
             Content-Type: text/plain; charset=utf-8
 
             message in Sent
         """.format(ac1.get_config("configured_addr")))
+        ac1.direct_imap.append("Spam", """
+            From: unknown.address@junk.org
+            Subject: subj
+            To: {}
+            Message-ID: <spam.message@junk.org>
+            Content-Type: text/plain; charset=utf-8
+
+            Unknown message in Spam
+        """.format(ac1.get_config("configured_addr")))
+        ac1.direct_imap.append("Junk", """
+            From: unknown.address@junk.org
+            Subject: subj
+            To: {}
+            Message-ID: <spam.message@junk.org>
+            Content-Type: text/plain; charset=utf-8
+
+            Unknown message in Junk
+        """.format(ac1.get_config("configured_addr")))
 
         ac1.set_config("scan_all_folders_debounce_secs", "0")
         lp.sec("All prepared, now let DC find the message")
@@ -1401,6 +1501,10 @@ class TestOnlineAccount:
         assert msg.text == "subj – message in Sent"
         assert len(msg.chat.get_messages()) == 1
 
+        assert not any("unknown.address" in c.get_name() for c in ac1.get_chats())
+        ac1.direct_imap.select_folder("Spam")
+        assert ac1.direct_imap.get_uid_by_message_id("spam.message@junk.org")
+
         ac1.stop_io()
         lp.sec("'Send out' the draft, i.e. move it to the Sent folder, and wait for DC to display it this time")
         ac1.direct_imap.select_folder("Drafts")
@@ -1413,6 +1517,35 @@ class TestOnlineAccount:
         assert msg2.text == "subj – message in Drafts that is moved to Sent later"
         assert len(msg.chat.get_messages()) == 2
 
+    def test_no_old_msg_is_fresh(self, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        ac1_clone = acfactory.clone_online_account(ac1)
+        acfactory.wait_configure_and_start_io()
+
+        ac1.set_config("e2ee_enabled", "0")
+        ac1_clone.set_config("e2ee_enabled", "0")
+        ac2.set_config("e2ee_enabled", "0")
+
+        ac1_clone.set_config("bcc_self", "1")
+
+        ac1.create_chat(ac2)
+        ac1_clone.create_chat(ac2)
+
+        lp.sec("Send a first message from ac2 to ac1 and check that it's 'fresh'")
+        first_msg_id = ac2.create_chat(ac1).send_text("Hi")
+        ac1._evtracker.wait_next_incoming_message()
+        assert ac1.create_chat(ac2).count_fresh_messages() == 1
+        assert len(list(ac1.get_fresh_messages())) == 1
+
+        lp.sec("Send a message from ac1_clone to ac2 and check that ac1 marks the first message as 'noticed'")
+        ac1_clone.create_chat(ac2).send_text("Hi back")
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+
+        assert ev.data1 == first_msg_id.chat.id
+        assert ac1.create_chat(ac2).count_fresh_messages() == 0
+        assert len(list(ac1.get_fresh_messages())) == 0
+
     def test_prefer_encrypt(self, acfactory, lp):
         """Test quorum rule for encryption preference in 1:1 and group chat."""
         ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
@@ -1757,7 +1890,6 @@ class TestOnlineAccount:
         lp.sec("trigger ac setup message and return setupcode")
         assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]
         setup_code = ac1.initiate_key_transfer()
-        ac2._evtracker.set_timeout(30)
         ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
         msg = ac2.get_message_by_id(ev.data2)
         assert msg.is_setup_message()
@@ -1774,7 +1906,6 @@ class TestOnlineAccount:
     def test_ac_setup_message_twice(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
         ac2 = acfactory.clone_online_account(ac1)
-        ac2._evtracker.set_timeout(30)
         acfactory.wait_configure_and_start_io()
 
         lp.sec("trigger ac setup message but ignore")
@@ -1948,7 +2079,7 @@ class TestOnlineAccount:
 
         lp.sec("ac1: send a message to group chat to promote the group")
         chat.send_text("afterwards promoted")
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
         assert chat.is_promoted()
         assert sorted(x.addr for x in chat.get_contacts()) == \
@@ -1958,29 +2089,29 @@ class TestOnlineAccount:
         # note that if the above create_chat() would not
         # happen we would not receive a proper member_added event
         contact2 = chat.add_contact("devnull@testrun.org")
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "added"
         assert ev.message.get_sender_contact().addr == ac1_addr
         assert ev.contact.addr == "devnull@testrun.org"
 
         lp.sec("ac1: remove address2")
         chat.remove_contact(contact2)
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "removed"
         assert ev.contact.addr == contact2.addr
         assert ev.message.get_sender_contact().addr == ac1_addr
 
         lp.sec("ac1: remove ac2 contact from chat")
         chat.remove_contact(ac2)
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
+        ev = in_list.get()
         assert ev.action == "removed"
         assert ev.message.get_sender_contact().addr == ac1_addr
 
@@ -2077,10 +2208,8 @@ class TestOnlineAccount:
         assert msg_back.chat == chat
         assert chat.get_profile_image() is None
 
-    @pytest.mark.parametrize("inbox_watch", ["0", "1"])
-    def test_connectivity(self, acfactory, lp, inbox_watch):
+    def test_connectivity(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-        ac1.set_config("inbox_watch", inbox_watch)
         ac1.set_config("scan_all_folders_debounce_secs", "0")
 
         ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTED)
@@ -2100,7 +2229,7 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.create_chat(ac1).send_text("Hi")
 
-        ac1.direct_imap.idle_check(terminate=False)
+        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
         ac1.maybe_network()
 
         ac1._evtracker.wait_for_all_work_done()
@@ -2112,8 +2241,6 @@ class TestOnlineAccount:
 
         ac2.create_chat(ac1).send_text("Hi 2")
 
-        ac1.direct_imap.idle_check(terminate=True)
-        ac1.maybe_network()
         ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTED, const.DC_CONNECTIVITY_WORKING)
         ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_WORKING, const.DC_CONNECTIVITY_CONNECTED)
 
@@ -2137,7 +2264,7 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.create_chat(ac1).send_text("Hi")
 
-        ac1.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
         ac1.maybe_network()
 
         while 1:
@@ -2320,32 +2447,30 @@ class TestOnlineAccount:
 
     def test_immediate_autodelete(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
-        ac2 = acfactory.get_online_configuring_account(mvbox=False, move=False, sentbox=False)
+        ac2 = acfactory.get_online_configuring_account(move=False, sentbox=False)
 
         # "1" means delete immediately, while "0" means do not delete
         ac2.set_config("delete_server_after", "1")
 
         acfactory.wait_configure_and_start_io()
 
-        imap2 = ac2.direct_imap
-        imap2.idle_start()
-
         lp.sec("ac1: create chat with ac2")
         chat1 = ac1.create_chat(ac2)
         ac2.create_chat(ac1)
 
+        lp.sec("ac1: send message to ac2")
         sent_msg = chat1.send_text("hello")
-        imap2.idle_check(terminate=False)
 
         msg = ac2._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        imap2.idle_check(terminate=True)
+        lp.sec("ac2: wait for close/expunge on autodelete")
         ac2._evtracker.get_info_contains("close/expunge succeeded")
 
-        assert len(imap2.get_all_messages()) == 0
+        lp.sec("ac2: check that message was autodeleted on server")
+        assert len(ac2.direct_imap.get_all_messages()) == 0
 
-        # Mark deleted message as seen and check that read receipt arrives
+        lp.sec("ac2: Mark deleted message as seen and check that read receipt arrives")
         msg.mark_seen()
         ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
         assert ev.data1 == chat1.id
@@ -2429,15 +2554,13 @@ class TestOnlineAccount:
         lp.sec("ac2: deleting all messages except third")
         assert len(to_delete) == len(texts) - 1
         ac2.delete_messages(to_delete)
-        for msg in to_delete:
-            ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+        ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
 
         ac2._evtracker.get_info_contains("close/expunge succeeded")
 
-        lp.sec("imap2: test that only one message is left")
-        imap2 = ac2.direct_imap
-
-        assert len(imap2.get_all_messages()) == 1
+        lp.sec("ac2: test that only one message is left")
+        ac2.direct_imap.select_config_folder("inbox")
+        assert len(ac2.direct_imap.get_all_messages()) == 1
 
     def test_configure_error_msgs(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2581,38 +2704,33 @@ class TestOnlineAccount:
         assert received_reply.quoted_text == "hello"
         assert received_reply.quote.id == out_msg.id
 
-    @pytest.mark.parametrize("folder,move,expected_destination,inbox_watch,", [
-        ("xyz", False, "xyz", "1"),  # Test that emails are recognized in a random folder but not moved
-        ("xyz", True, "DeltaChat", "1"),  # ...emails are found in a random folder and moved to DeltaChat
-        ("Spam", False, "INBOX", "1"),  # ...emails are moved from the spam folder to the Inbox
-        ("INBOX", False, "INBOX", "0"),  # ...emails are found in the `Inbox` folder even if `inbox_watch` is "0"
+    @pytest.mark.parametrize("folder,move,expected_destination,", [
+        ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
+        ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
+        ("Spam", False, "INBOX"),  # ...emails are moved from the spam folder to the Inbox
     ])
     # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
     # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
-    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination, inbox_watch):
+    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination):
         """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
         variant = folder + "-" + str(move) + "-" + expected_destination
         lp.sec("Testing variant " + variant)
         ac1 = acfactory.get_online_configuring_account(move=move)
         ac2 = acfactory.get_online_configuring_account()
-        ac1.set_config("inbox_watch", inbox_watch)
 
         acfactory.wait_configure(ac1)
         ac1.direct_imap.create_folder(folder)
 
         acfactory.wait_configure_and_start_io()
         # Wait until each folder was selected once and we are IDLEing:
-        if inbox_watch == "1":
-            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
-        else:
-            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
         ac1.stop_io()
 
         # Send a message to ac1 and move it to the mvbox:
         ac1.direct_imap.select_config_folder("inbox")
         ac1.direct_imap.idle_start()
         acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
-        ac1.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.idle_wait_for_new_message(terminate=True)
         ac1.direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
 
         lp.sec("Everything prepared, now see if DeltaChat finds the message (" + variant + ")")
@@ -2621,11 +2739,7 @@ class TestOnlineAccount:
         msg = ac1._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        # Wait until the message was moved (if at all) and we are IDLEing again:
-        if inbox_watch == "1":
-            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
-        else:
-            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
+        # The message has been downloaded, which means it has reached its destination.
         ac1.direct_imap.select_folder(expected_destination)
         assert len(ac1.direct_imap.get_all_messages()) == 1
         if folder != expected_destination:
@@ -2648,8 +2762,7 @@ class TestOnlineAccount:
             if mvbox_move:
                 assert ac.get_config("configured_mvbox_folder")
 
-        ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
-        ac1.set_config("sentbox_move", "1")
+        ac1 = acfactory.get_online_configuring_account(move=mvbox_move)
         ac2 = acfactory.get_online_configuring_account()
 
         acfactory.wait_configure(ac1)
@@ -2666,10 +2779,7 @@ class TestOnlineAccount:
         acfactory.wait_configure_and_start_io()
         assert_folders_configured(ac1)
 
-        if mvbox_move:
-            ac1.direct_imap.select_config_folder("mvbox")
-        else:
-            ac1.direct_imap.select_config_folder("sentbox")
+        assert ac1.direct_imap.select_config_folder("mvbox" if mvbox_move else "inbox")
         ac1.direct_imap.idle_start()
 
         lp.sec("send out message with bcc to ourselves")
@@ -2678,22 +2788,24 @@ class TestOnlineAccount:
         chat.send_text("message text")
         assert_folders_configured(ac1)
 
-        # now wait until the bcc_self message arrives
-        # Also test that bcc_self messages moved to the mvbox are marked as read.
+        lp.sec("wait until the bcc_self message arrives in correct folder and is marked seen")
         assert ac1.direct_imap.idle_wait_for_seen()
         assert_folders_configured(ac1)
 
+        lp.sec("create a cloned ac1 and fetch contact history during configure")
         ac1_clone = acfactory.clone_online_account(ac1)
         ac1_clone.set_config("fetch_existing_msgs", "1")
         ac1_clone._configtracker.wait_finish()
         ac1_clone.start_io()
         assert_folders_configured(ac1_clone)
 
+        lp.sec("check that ac2 contact was fetchted during configure")
         ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
         ac2_addr = ac2.get_config("addr")
         assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
         assert_folders_configured(ac1_clone)
 
+        lp.sec("check that messages changed events arrive for the correct message")
         msg = ac1_clone._evtracker.wait_next_messages_changed()
         assert msg.text == "message text"
         assert_folders_configured(ac1)
@@ -2748,19 +2860,19 @@ class TestOnlineAccount:
 
     def test_delete_deltachat_folder(self, acfactory):
         """Test that DeltaChat folder is recreated if user deletes it manually."""
-        ac1 = acfactory.get_online_configuring_account(mvbox=True)
+        ac1 = acfactory.get_online_configuring_account(move=True)
         ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure(ac1)
 
-        ac1.direct_imap.conn.delete_folder("DeltaChat")
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 0
+        ac1.direct_imap.conn.folder.delete("DeltaChat")
+        assert "DeltaChat" not in ac1.direct_imap.list_folders()
         acfactory.wait_configure_and_start_io()
 
         ac2.create_chat(ac1).send_text("hello")
         msg = ac1._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 1
+        assert "DeltaChat" in ac1.direct_imap.list_folders()
 
 
 class TestGroupStressTests:
@@ -2876,7 +2988,8 @@ class TestGroupStressTests:
         lp.sec("ac2: check that ac3 is removed")
         msg = ac2._evtracker.wait_next_incoming_message()
 
-        assert msg.chat.num_contacts() == chat.num_contacts()
+        assert chat.num_contacts() == 2
+        assert msg.chat.num_contacts() == 2
         acfactory.dump_imap_summary(sys.stdout)
 
 

python/tests/test_lowlevel.py

@@ -36,7 +36,8 @@ def test_wrong_db(tmpdir):
     # write an invalid database file
     p.write("x123" * 10)
 
-    assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
+    context = lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
+    assert not lib.dc_context_is_open(context)
 
 
 def test_empty_blobdir(tmpdir):

python/tox.ini

@@ -1,12 +1,14 @@
 [tox]
+isolated_build = true
 envlist =
     py3
     lint
+    mypy
     auditwheels
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n1 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
@@ -42,6 +44,15 @@ commands =
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
+[testenv:mypy]
+deps =
+    mypy
+    typing
+    types-setuptools
+    types-requests
+commands =
+    mypy --no-incremental src/
+
 [testenv:doc]
 changedir=doc
 deps =
@@ -67,8 +78,8 @@ commands =
 addopts = -v -ra --strict-markers
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 90
-timeout_method = thread
+timeout = 150
+timeout_func_only = True
 markers = 
     ignored: ignore this test in default test runs, use --ignored to run.
 

rust-toolchain

@@ -1 +1 @@
-1.54.0
+1.60.0

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -2,13 +2,13 @@
 
 set -x -e
 
-# we use the python3.6 environment as the base environment 
-/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel 
+# we use the python3.7 environment as the base environment 
+/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel 
 
 pushd /usr/bin
 
-ln -s /opt/_internal/cpython-3.6.*/bin/tox
-ln -s /opt/_internal/cpython-3.6.*/bin/devpi
-ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
+ln -s /opt/_internal/cpython-3.7.*/bin/tox
+ln -s /opt/_internal/cpython-3.7.*/bin/devpi
+ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
 
 popd

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.54.0
+RUST_VERSION=1.60.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/deps/build_python.sh

@@ -2,13 +2,13 @@
 
 set -x -e
 
-# we use the python3.6 environment as the base environment
-/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel
+# we use the python3.7 environment as the base environment
+/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel
 
 pushd /usr/bin
 
-ln -s /opt/_internal/cpython-3.6.*/bin/tox
-ln -s /opt/_internal/cpython-3.6.*/bin/devpi
-ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
+ln -s /opt/_internal/cpython-3.7.*/bin/tox
+ln -s /opt/_internal/cpython-3.7.*/bin/devpi
+ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
 
 popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.54.0
+RUST_VERSION=1.60.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/run_all.sh

@@ -19,11 +19,9 @@ export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp36-cp36m/bin
+export PATH=$PATH:/opt/python/cp37-cp37m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.6
-ln -s /opt/python/cp36-cp36m/bin/python3.6
 rm -f python3.7
 ln -s /opt/python/cp37-cp37m/bin/python3.7
 rm -f python3.8

src/accounts.rs

@@ -29,21 +29,21 @@ pub struct Accounts {
 
 impl Accounts {
     /// Loads or creates an accounts folder at the given `dir`.
-    pub async fn new(os_name: String, dir: PathBuf) -> Result<Self> {
+    pub async fn new(dir: PathBuf) -> Result<Self> {
         if !dir.exists().await {
-            Accounts::create(os_name, &dir).await?;
+            Accounts::create(&dir).await?;
         }
 
         Accounts::open(dir).await
     }
 
     /// Creates a new default structure.
-    pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
+    pub async fn create(dir: &PathBuf) -> Result<()> {
         fs::create_dir_all(dir)
             .await
             .context("failed to create folder")?;
 
-        Config::new(os_name.clone(), dir).await?;
+        Config::new(dir).await?;
 
         Ok(())
     }
@@ -54,10 +54,19 @@ impl Accounts {
         ensure!(dir.exists().await, "directory does not exist");
 
         let config_file = dir.join(CONFIG_NAME);
-        ensure!(config_file.exists().await, "accounts.toml does not exist");
+        ensure!(
+            config_file.exists().await,
+            "{:?} does not exist",
+            config_file
+        );
 
-        let config = Config::from_file(config_file).await?;
-        let accounts = config.load_accounts().await?;
+        let config = Config::from_file(config_file)
+            .await
+            .context("failed to load accounts config")?;
+        let accounts = config
+            .load_accounts()
+            .await
+            .context("failed to load accounts")?;
 
         let emitter = EventEmitter::new();
 
@@ -66,7 +75,9 @@ impl Accounts {
         emitter.sender.send(events.get_emitter()).await?;
 
         for account in accounts.values() {
-            emitter.add_account(account).await?;
+            emitter.add_account(account).await.with_context(|| {
+                format!("failed to add account {} to event emitter", account.id)
+            })?;
         }
 
         Ok(Self {
@@ -104,12 +115,24 @@ impl Accounts {
         Ok(())
     }
 
-    /// Add a new account.
+    /// Add a new account and opens it.
+    ///
+    /// Returns account ID.
     pub async fn add_account(&mut self) -> Result<u32> {
-        let os_name = self.config.os_name().await;
         let account_config = self.config.new_account(&self.dir).await?;
 
-        let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
+        let ctx = Context::new(account_config.dbfile().into(), account_config.id).await?;
+        self.emitter.add_account(&ctx).await?;
+        self.accounts.insert(account_config.id, ctx);
+
+        Ok(account_config.id)
+    }
+
+    /// Adds a new closed account.
+    pub async fn add_closed_account(&mut self) -> Result<u32> {
+        let account_config = self.config.new_account(&self.dir).await?;
+
+        let ctx = Context::new_closed(account_config.dbfile().into(), account_config.id).await?;
         self.emitter.add_account(&ctx).await?;
         self.accounts.insert(account_config.id, ctx);
 
@@ -125,9 +148,27 @@ impl Accounts {
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id).await {
-            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
-                .await
-                .context("failed to remove account data")?;
+            // Spend up to 1 minute trying to remove the files.
+            // Files may remain locked up to 30 seconds due to r2d2 bug:
+            // https://github.com/sfackler/r2d2/issues/99
+            let mut counter = 0;
+            loop {
+                counter += 1;
+
+                if let Err(err) = fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                    .await
+                    .context("failed to remove account data")
+                {
+                    if counter > 60 {
+                        return Err(err);
+                    }
+
+                    // Wait 1 second and try again.
+                    async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
+                } else {
+                    break;
+                }
+            }
         }
         self.config.remove_account(id).await?;
 
@@ -183,13 +224,7 @@ impl Accounts {
 
         match res {
             Ok(_) => {
-                let ctx = Context::with_blobdir(
-                    self.config.os_name().await,
-                    new_dbfile,
-                    new_blobdir,
-                    account_config.id,
-                )
-                .await?;
+                let ctx = Context::new(new_dbfile, account_config.id).await?;
                 self.emitter.add_account(&ctx).await?;
                 self.accounts.insert(account_config.id, ctx);
                 Ok(account_config.id)
@@ -349,7 +384,6 @@ pub struct Config {
 /// This is serialized into TOML.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 struct InnerConfig {
-    pub os_name: String,
     /// The currently selected account.
     pub selected_account: u32,
     pub next_id: u32,
@@ -357,9 +391,8 @@ struct InnerConfig {
 }
 
 impl Config {
-    pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
+    pub async fn new(dir: &PathBuf) -> Result<Self> {
         let inner = InnerConfig {
-            os_name,
             accounts: Vec::new(),
             selected_account: 0,
             next_id: 1,
@@ -374,10 +407,6 @@ impl Config {
         Ok(cfg)
     }
 
-    pub async fn os_name(&self) -> String {
-        self.inner.os_name.clone()
-    }
-
     /// Sync the inmemory representation to disk.
     async fn sync(&self) -> Result<()> {
         fs::write(&self.file, toml::to_string_pretty(&self.inner)?)
@@ -396,12 +425,15 @@ impl Config {
     pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
         let mut accounts = BTreeMap::new();
         for account_config in &self.inner.accounts {
-            let ctx = Context::new(
-                self.inner.os_name.clone(),
-                account_config.dbfile().into(),
-                account_config.id,
-            )
-            .await?;
+            let ctx = Context::new(account_config.dbfile().into(), account_config.id)
+                .await
+                .with_context(|| {
+                    format!(
+                        "failed to create context from file {:?}",
+                        account_config.dbfile()
+                    )
+                })?;
+
             accounts.insert(account_config.id, ctx);
         }
 
@@ -426,8 +458,13 @@ impl Config {
 
         self.sync().await?;
 
-        self.select_account(id).await.expect("just added");
-        let cfg = self.get_account(id).await.expect("just added");
+        self.select_account(id)
+            .await
+            .context("failed to select just added account")?;
+        let cfg = self
+            .get_account(id)
+            .await
+            .context("failed to get just added account")?;
         Ok(cfg)
     }
 
@@ -498,7 +535,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts1").into();
 
-        let mut accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let mut accounts1 = Accounts::new(p.clone()).await.unwrap();
         accounts1.add_account().await.unwrap();
 
         let accounts2 = Accounts::open(p).await.unwrap();
@@ -516,7 +553,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let mut accounts = Accounts::new(p.clone()).await.unwrap();
         assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
@@ -543,7 +580,7 @@ mod tests {
         let dir = tempfile::tempdir()?;
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new("my_os".into(), p.clone()).await?;
+        let mut accounts = Accounts::new(p.clone()).await?;
         assert!(accounts.get_selected_account().await.is_none());
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
@@ -564,14 +601,12 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let mut accounts = Accounts::new(p.clone()).await.unwrap();
         assert_eq!(accounts.accounts.len(), 0);
         assert_eq!(accounts.config.get_selected_account().await, 0);
 
         let extern_dbfile: PathBuf = dir.path().join("other").into();
-        let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
-            .await
-            .unwrap();
+        let ctx = Context::new(extern_dbfile.clone(), 0).await.unwrap();
         ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
             .await
             .unwrap();
@@ -601,7 +636,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let mut accounts = Accounts::new(p.clone()).await.unwrap();
 
         for expected_id in 1..10 {
             let id = accounts.add_account().await.unwrap();
@@ -621,7 +656,7 @@ mod tests {
         let dummy_accounts = 10;
 
         let (id0, id1, id2) = {
-            let mut accounts = Accounts::new("my_os".into(), p.clone()).await?;
+            let mut accounts = Accounts::new(p.clone()).await?;
             accounts.add_account().await?;
             let ids = accounts.get_all().await;
             assert_eq!(ids.len(), 1);
@@ -656,7 +691,7 @@ mod tests {
         assert!(id2 > id1 + dummy_accounts);
 
         let (id0_reopened, id1_reopened, id2_reopened) = {
-            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+            let accounts = Accounts::new(p.clone()).await?;
             let ctx = accounts.get_selected_account().await.unwrap();
             assert_eq!(
                 ctx.get_config(crate::config::Config::Addr).await?,
@@ -701,7 +736,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+        let accounts = Accounts::new(p.clone()).await?;
 
         // Make sure there are no accounts.
         assert_eq!(accounts.accounts.len(), 0);
@@ -721,4 +756,49 @@ mod tests {
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_encrypted_account() -> Result<()> {
+        let dir = tempfile::tempdir().context("failed to create tempdir")?;
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let mut accounts = Accounts::new(p.clone())
+            .await
+            .context("failed to create accounts manager")?;
+
+        assert_eq!(accounts.accounts.len(), 0);
+        let account_id = accounts
+            .add_closed_account()
+            .await
+            .context("failed to add closed account")?;
+        let account = accounts
+            .get_selected_account()
+            .await
+            .context("failed to get account")?;
+        assert_eq!(account.id, account_id);
+        let passphrase_set_success = account
+            .open("foobar".to_string())
+            .await
+            .context("failed to set passphrase")?;
+        assert!(passphrase_set_success);
+        drop(accounts);
+
+        let accounts = Accounts::new(p.clone())
+            .await
+            .context("failed to create second accounts manager")?;
+        let account = accounts
+            .get_selected_account()
+            .await
+            .context("failed to get account")?;
+        assert_eq!(account.is_open().await, false);
+
+        // Try wrong passphrase.
+        assert_eq!(account.open("barfoo".to_string()).await?, false);
+        assert_eq!(account.open("".to_string()).await?, false);
+
+        assert_eq!(account.open("foobar".to_string()).await?, true);
+        assert_eq!(account.is_open().await, true);
+
+        Ok(())
+    }
 }

src/aheader.rs

@@ -2,7 +2,7 @@
 //!
 //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
 
-use anyhow::{bail, format_err, Error, Result};
+use anyhow::{bail, Context as _, Error, Result};
 use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
@@ -139,15 +139,14 @@ impl str::FromStr for Aheader {
         };
         let public_key: SignedPublicKey = attributes
             .remove("keydata")
-            .ok_or_else(|| format_err!("keydata attribute is not found"))
+            .context("keydata attribute is not found")
             .and_then(|raw| {
-                SignedPublicKey::from_base64(&raw)
-                    .map_err(|_| format_err!("Autocrypt key cannot be decoded"))
+                SignedPublicKey::from_base64(&raw).context("autocrypt key cannot be decoded")
             })
             .and_then(|key| {
                 key.verify()
                     .and(Ok(key))
-                    .map_err(|_| format_err!("Autocrypt key cannot be verified"))
+                    .context("autocrypt key cannot be verified")
             })?;
 
         let prefer_encrypt = attributes

src/blob.rs

@@ -3,28 +3,26 @@
 use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
+use std::io::Cursor;
 
 use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use anyhow::format_err;
-use anyhow::Context as _;
-use anyhow::Error;
-use image::DynamicImage;
-use image::GenericImageView;
-use image::ImageFormat;
+use anyhow::{format_err, Context as _, Error};
+use image::{DynamicImage, ImageFormat};
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::{
-    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
-    WORSE_IMAGE_SIZE,
+    MediaQuality, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE, WORSE_IMAGE_SIZE,
 };
 use crate::context::Context;
 use crate::events::EventType;
+use crate::log::LogExt;
 use crate::message;
+use crate::message::Viewtype;
 
 /// Represents a file in the blob directory.
 ///
@@ -63,7 +61,7 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name);
-        let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
+        let (name, mut file) = BlobObject::create_new_file(context, blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
             .map_err(|err| BlobError::WriteFailure {
@@ -87,13 +85,16 @@ impl<'a> BlobObject<'a> {
 
     // Creates a new file, returning a tuple of the name and the handle.
     async fn create_new_file(
+        context: &Context,
         dir: &Path,
         stem: &str,
         ext: &str,
     ) -> Result<(String, fs::File), BlobError> {
-        let max_attempt = 15;
+        const MAX_ATTEMPT: u32 = 16;
+        let mut attempt = 0;
         let mut name = format!("{}{}", stem, ext);
-        for attempt in 0..max_attempt {
+        loop {
+            attempt += 1;
             let path = dir.join(&name);
             match fs::OpenOptions::new()
                 .create_new(true)
@@ -103,24 +104,20 @@ impl<'a> BlobObject<'a> {
             {
                 Ok(file) => return Ok((name, file)),
                 Err(err) => {
-                    if attempt == max_attempt {
+                    if attempt >= MAX_ATTEMPT {
                         return Err(BlobError::CreateFailure {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
                         });
+                    } else if attempt == 1 && !dir.exists().await {
+                        fs::create_dir_all(dir).await.ok_or_log(context);
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
                     }
                 }
             }
         }
-        // This is supposed to be unreachable, but the compiler doesn't know.
-        Err(BlobError::CreateFailure {
-            blobdir: dir.to_path_buf(),
-            blobname: name,
-            cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
-        })
     }
 
     /// Creates a new blob object with unique name by copying an existing file.
@@ -149,7 +146,7 @@ impl<'a> BlobObject<'a> {
             })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
         let (name, mut dst_file) =
-            BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
+            BlobObject::create_new_file(context, context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
         if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
             {
@@ -292,7 +289,7 @@ impl<'a> BlobObject<'a> {
 
     /// Returns the filename of the blob.
     pub fn as_file_name(&self) -> &str {
-        self.name.rsplitn(2, '/').next().unwrap()
+        self.name.rsplit('/').next().unwrap()
     }
 
     /// The path relative in the blob directory.
@@ -305,7 +302,7 @@ impl<'a> BlobObject<'a> {
     /// If a blob's filename has an extension, it is always guaranteed
     /// to be lowercase.
     pub fn suffix(&self) -> Option<&str> {
-        let ext = self.name.rsplitn(2, '.').next();
+        let ext = self.name.rsplit('.').next();
         if ext == Some(&self.name) {
             None
         } else {
@@ -348,13 +345,30 @@ impl<'a> BlobObject<'a> {
         };
 
         let clean = sanitize_filename::sanitize_with_options(name, opts);
+        // Let's take the tricky filename
+        // "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz" as an example.
+        // Split it into "file" and "with_lots_of_characters_behind_point_and_double_ending.tar.gz":
         let mut iter = clean.splitn(2, '.');
+
         let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
-        let ext: String = iter.next().unwrap_or_default().chars().take(32).collect();
+        // stem == "file"
+
+        let ext_chars = iter.next().unwrap_or_default().chars();
+        let ext: String = ext_chars
+            .rev()
+            .take(32)
+            .collect::<Vec<_>>()
+            .iter()
+            .rev()
+            .collect();
+        // ext == "d_point_and_double_ending.tar.gz"
+
         if ext.is_empty() {
             (stem, "".to_string())
         } else {
             (stem, format!(".{}", ext).to_lowercase())
+            // Return ("file", ".d_point_and_double_ending.tar.gz")
+            // which is not perfect but acceptable.
         }
     }
 
@@ -449,7 +463,8 @@ impl<'a> BlobObject<'a> {
 
         fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
             encoded.clear();
-            img.write_to(encoded, image::ImageFormat::Jpeg)?;
+            let mut buf = Cursor::new(encoded);
+            img.write_to(&mut buf, image::ImageFormat::Jpeg)?;
             Ok(())
         }
         fn encoded_img_exceeds_bytes(
@@ -619,13 +634,14 @@ pub enum BlobError {
 mod tests {
     use fs::File;
 
-    use super::*;
+    use anyhow::Result;
+    use image::{GenericImageView, Pixel};
 
-    use crate::{
-        message::Message,
-        test_utils::{self, TestContext},
-    };
-    use image::Pixel;
+    use crate::chat::{self, create_group_chat, ProtectionStatus};
+    use crate::message::Message;
+    use crate::test_utils::{self, TestContext};
+
+    use super::*;
 
     #[async_std::test]
     async fn test_create() {
@@ -924,7 +940,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_recode_image() {
+    async fn test_recode_image_1() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
         send_image_check_mediaquality(Some("0"), bytes, 1000, 1000, 0, 1000, 1000)
@@ -941,7 +957,10 @@ mod tests {
         )
         .await
         .unwrap();
+    }
 
+    #[async_std::test]
+    async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(
@@ -957,22 +976,29 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
-        let mut bytes = vec![];
+        let mut buf = Cursor::new(vec![]);
         img_rotated
-            .write_to(&mut bytes, image::ImageFormat::Jpeg)
+            .write_to(&mut buf, image::ImageFormat::Jpeg)
             .unwrap();
-        let img_rotated = send_image_check_mediaquality(
-            Some("0"),
-            &bytes,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-            0,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-        assert_correct_rotation(&img_rotated);
+        let bytes = buf.into_inner();
+
+        // Do this in parallel to speed up the test a bit
+        // (it still takes very long though)
+        let bytes2 = bytes.clone();
+        let join_handle = async_std::task::spawn(async move {
+            let img_rotated = send_image_check_mediaquality(
+                Some("0"),
+                &bytes2,
+                BALANCED_IMAGE_SIZE * 1800 / 2000,
+                BALANCED_IMAGE_SIZE,
+                0,
+                BALANCED_IMAGE_SIZE * 1800 / 2000,
+                BALANCED_IMAGE_SIZE,
+            )
+            .await
+            .unwrap();
+            assert_correct_rotation(&img_rotated);
+        });
 
         let img_rotated = send_image_check_mediaquality(
             Some("1"),
@@ -987,6 +1013,11 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
+        join_handle.await;
+    }
+
+    #[async_std::test]
+    async fn test_recode_image_3() {
         let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
             .await
@@ -1066,4 +1097,38 @@ mod tests {
         assert_eq!(img.height() as u32, compressed_height);
         Ok(img)
     }
+
+    #[async_std::test]
+    async fn test_increation_in_blobdir() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
+
+        let file = t.get_blobdir().join("anyfile.dat");
+        File::create(&file).await?.write_all("bla".as_ref()).await?;
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file(file.to_str().unwrap(), None);
+        let prepared_id = chat::prepare_msg(&t, chat_id, &mut msg).await?;
+        assert_eq!(prepared_id, msg.id);
+        assert!(msg.is_increation());
+
+        let msg = Message::load_from_db(&t, prepared_id).await?;
+        assert!(msg.is_increation());
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_increation_not_blobdir() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
+        assert_ne!(t.get_blobdir().to_str(), t.dir.path().to_str());
+
+        let file = t.dir.path().join("anyfile.dat");
+        File::create(&file).await?.write_all("bla".as_ref()).await?;
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file(file.to_str().unwrap(), None);
+        assert!(chat::prepare_msg(&t, chat_id, &mut msg).await.is_err());
+
+        Ok(())
+    }
 }

src/chatlist.rs

@@ -1,16 +1,14 @@
 //! # Chat list module.
 
-use anyhow::{bail, ensure, Result};
+use anyhow::{ensure, Context as _, Result};
 
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
 use crate::constants::{
-    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CONTACT_ID_DEVICE,
-    DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT, DC_GCL_ARCHIVED_ONLY,
-    DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_GCL_ADD_ALLDONE_HINT,
+    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
 };
-use crate::contact::Contact;
+use crate::contact::{Contact, ContactId};
 use crate::context::Context;
-use crate::ephemeral::delete_expired_messages;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock_str;
 use crate::summary::Summary;
@@ -85,19 +83,13 @@ impl Chatlist {
         context: &Context,
         listflags: usize,
         query: Option<&str>,
-        query_contact_id: Option<u32>,
+        query_contact_id: Option<ContactId>,
     ) -> Result<Self> {
         let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
         let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
         let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
         let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
 
-        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
-        // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_expired_messages(context).await {
-            warn!(context, "Failed to hide expired messages: {}", err);
-        }
-
         let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
@@ -112,7 +104,7 @@ impl Chatlist {
         };
 
         let skip_id = if flag_for_forwarding {
-            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+            ChatId::lookup_by_contact(context, ContactId::DEVICE)
                 .await?
                 .unwrap_or_default()
         } else {
@@ -147,7 +139,7 @@ impl Chatlist {
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
                  GROUP BY c.id
                  ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                paramsv![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
+                paramsv![MessageState::OutDraft, query_contact_id, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             ).await?
@@ -216,7 +208,7 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+                ChatId::lookup_by_contact(context, ContactId::SELF)
                     .await?
                     .unwrap_or_default()
             } else {
@@ -271,21 +263,23 @@ impl Chatlist {
     /// Get a single chat ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_chat().
-    pub fn get_chat_id(&self, index: usize) -> ChatId {
-        match self.ids.get(index) {
-            Some((chat_id, _msg_id)) => *chat_id,
-            None => ChatId::new(0),
-        }
+    pub fn get_chat_id(&self, index: usize) -> Result<ChatId> {
+        let (chat_id, _msg_id) = self
+            .ids
+            .get(index)
+            .context("chatlist index is out of range")?;
+        Ok(*chat_id)
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
     pub fn get_msg_id(&self, index: usize) -> Result<Option<MsgId>> {
-        match self.ids.get(index) {
-            Some((_chat_id, msg_id)) => Ok(*msg_id),
-            None => bail!("Chatlist index out of range"),
-        }
+        let (_chat_id, msg_id) = self
+            .ids
+            .get(index)
+            .context("chatlist index is out of range")?;
+        Ok(*msg_id)
     }
 
     /// Returns a summary for a given chatlist index.
@@ -299,11 +293,10 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let (chat_id, lastmsg_id) = match self.ids.get(index) {
-            Some(ids) => ids,
-            None => bail!("Chatlist index out of range"),
-        };
-
+        let (chat_id, lastmsg_id) = self
+            .ids
+            .get(index)
+            .context("chatlist index is out of range")?;
         Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
     }
 
@@ -325,7 +318,7 @@ impl Chatlist {
 
         let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
             let lastmsg = Message::load_from_db(context, lastmsg_id).await?;
-            if lastmsg.from_id == DC_CONTACT_ID_SELF {
+            if lastmsg.from_id == ContactId::SELF {
                 (Some(lastmsg), None)
             } else {
                 match chat.typ {
@@ -342,7 +335,7 @@ impl Chatlist {
 
         if chat.id.is_archived_link() {
             Ok(Default::default())
-        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED) {
+        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != ContactId::UNDEFINED) {
             Ok(Summary::new(context, &lastmsg, chat, lastcontact.as_ref()).await)
         } else {
             Ok(Summary {
@@ -355,6 +348,10 @@ impl Chatlist {
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
         self.ids.iter().position(|(chat_id, _)| chat_id == &id)
     }
+
+    pub fn iter(&self) -> impl Iterator<Item = &(ChatId, Option<MsgId>)> {
+        self.ids.iter()
+    }
 }
 
 /// Returns the number of archived chats
@@ -374,8 +371,8 @@ mod tests {
     use super::*;
 
     use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus};
-    use crate::constants::Viewtype;
     use crate::dc_receive_imf::dc_receive_imf;
+    use crate::message::Viewtype;
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
 
@@ -395,16 +392,26 @@ mod tests {
         // check that the chatlist starts with the most recent message
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
-        assert_eq!(chats.get_chat_id(0), chat_id3);
-        assert_eq!(chats.get_chat_id(1), chat_id2);
-        assert_eq!(chats.get_chat_id(2), chat_id1);
+        assert_eq!(chats.get_chat_id(0).unwrap(), chat_id3);
+        assert_eq!(chats.get_chat_id(1).unwrap(), chat_id2);
+        assert_eq!(chats.get_chat_id(2).unwrap(), chat_id1);
+
+        // New drafts are sorted to the top
+        // We have to set a draft on the other two messages, too, as
+        // chat timestamps are only exact to the second and sorting by timestamp
+        // would not work.
+        // Message timestamps are "smeared" and unique, so we don't have this problem
+        // if we have any message (can be a draft) in all chats.
+        // Instead of setting drafts for chat_id1 and chat_id3, we could also sleep
+        // 2s here.
+        for chat_id in &[chat_id1, chat_id3, chat_id2] {
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_text(Some("hello".to_string()));
+            chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
+        }
 
-        // drafts are sorted to the top
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t, Some(&mut msg)).await.unwrap();
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.get_chat_id(0), chat_id2);
+        assert_eq!(chats.get_chat_id(0).unwrap(), chat_id2);
 
         // check chatlist query and archive functionality
         let chats = Chatlist::try_load(&t, 0, Some("b"), None).await.unwrap();
@@ -435,7 +442,7 @@ mod tests {
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0))
+        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0).unwrap())
             .await
             .unwrap()
             .is_self_talk());
@@ -444,7 +451,7 @@ mod tests {
             .await
             .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t, chats.get_chat_id(0))
+        assert!(Chat::load_from_db(&t, chats.get_chat_id(0).unwrap())
             .await
             .unwrap()
             .is_self_talk());
@@ -489,15 +496,13 @@ mod tests {
         dc_receive_imf(
             &t,
             b"From: Bob Authname <bob@example.org>\n\
-                 To: alice@example.com\n\
+                 To: alice@example.org\n\
                  Subject: foo\n\
                  Message-ID: <msg1234@example.org>\n\
                  Chat-Version: 1.0\n\
                  Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
                  \n\
                  hello foo\n",
-            "INBOX",
-            1,
             false,
         )
         .await?;
@@ -518,7 +523,7 @@ mod tests {
         // check, the one-to-one-chat can be found using chatlist search query
         let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0), chat_id);
+        assert_eq!(chats.get_chat_id(0).unwrap(), chat_id);
 
         // change the name of the contact; this also changes the name of the one-to-one-chat
         let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
@@ -551,15 +556,13 @@ mod tests {
         dc_receive_imf(
             &t,
             b"From: bob@example.org\n\
-                 To: alice@example.com\n\
+                 To: alice@example.org\n\
                  Subject: foo\n\
                  Message-ID: <msg5678@example.org>\n\
                  Chat-Version: 1.0\n\
                  Date: Sun, 22 Mar 2021 22:38:57 +0000\n\
                  \n\
                  hello foo\n",
-            "INBOX",
-            1,
             false,
         )
         .await?;
@@ -575,7 +578,7 @@ mod tests {
         // check, the one-to-one-chat can be found using chatlist search query
         let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0), chat_id);
+        assert_eq!(chats.get_chat_id(0)?, chat_id);
 
         // change the name of the contact; this also changes the name of the one-to-one-chat
         let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
@@ -586,7 +589,7 @@ mod tests {
         assert_eq!(chats.len(), 0); // email-addresses are searchable in contacts, not in chats
         let chats = Chatlist::try_load(&t, 0, Some("Bob Nickname"), None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0), chat_id);
+        assert_eq!(chats.get_chat_id(0)?, chat_id);
 
         // revert name change, this again changes the name of the one-to-one-chat to the email-address
         let test_id = Contact::create(&t, "", "bob@example.org").await?;

src/color.rs

@@ -35,6 +35,10 @@ pub(crate) fn str_to_color(s: &str) -> u32 {
     rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
 }
 
+pub fn color_int_to_hex_string(color: u32) -> String {
+    format!("{:#08x}", color).replace("0x", "#")
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/config.rs

@@ -1,20 +1,17 @@
 //! # Key-value configuration management.
 
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Context as _, Result};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
-use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
-use crate::job;
-use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
-use crate::stock_str;
 
 /// The available configuration keys.
 #[derive(
@@ -67,20 +64,18 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MdnsEnabled,
 
-    #[strum(props(default = "1"))]
-    InboxWatch,
-
-    #[strum(props(default = "1"))]
+    #[strum(props(default = "0"))]
     SentboxWatch,
 
-    #[strum(props(default = "1"))]
-    MvboxWatch,
-
     #[strum(props(default = "1"))]
     MvboxMove,
 
+    /// Watch for new messages in the "Mvbox" (aka DeltaChat folder) only.
+    ///
+    /// This will not entirely disable other folders, e.g. the spam folder will also still
+    /// be watched for new messages.
     #[strum(props(default = "0"))]
-    SentboxMove, // If `MvboxMove` is true, this config is ignored. Currently only used in tests.
+    OnlyFetchMvbox,
 
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
@@ -117,6 +112,7 @@ pub enum Config {
     DeleteDeviceAfter,
 
     SaveMimeHeaders,
+    /// The primary email address. Also see `SecondaryAddrs`.
     ConfiguredAddr,
     ConfiguredMailServer,
     ConfiguredMailUser,
@@ -135,11 +131,14 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
-    ConfiguredSpamFolder,
     ConfiguredTimestamp,
     ConfiguredProvider,
     Configured,
 
+    /// All secondary self addresses separated by spaces
+    /// (`addr1@example.org addr2@exapmle.org addr3@example.org`)
+    SecondaryAddrs,
+
     #[strum(serialize = "sys.version")]
     SysVersion,
 
@@ -206,7 +205,6 @@ impl Context {
 
         // Default values
         match key {
-            Config::Selfstatus => Ok(Some(stock_str::status_line(self).await)),
             Config::ConfiguredInboxFolder => Ok(Some("INBOX".to_owned())),
             _ => Ok(key.get_str("default").map(|s| s.to_string())),
         }
@@ -234,6 +232,11 @@ impl Context {
         Ok(self.get_config_int(key).await? != 0)
     }
 
+    pub(crate) async fn should_watch_mvbox(&self) -> Result<bool> {
+        Ok(self.get_config_bool(Config::MvboxMove).await?
+            || self.get_config_bool(Config::OnlyFetchMvbox).await?)
+    }
+
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
@@ -242,7 +245,7 @@ impl Context {
         match self.get_config_int(Config::DeleteServerAfter).await? {
             0 => Ok(None),
             1 => Ok(Some(0)),
-            x => Ok(Some(x as i64)),
+            x => Ok(Some(i64::from(x))),
         }
     }
 
@@ -264,7 +267,7 @@ impl Context {
     pub async fn get_config_delete_device_after(&self) -> Result<Option<i64>> {
         match self.get_config_int(Config::DeleteDeviceAfter).await? {
             0 => Ok(None),
-            x => Ok(Some(x as i64)),
+            x => Ok(Some(i64::from(x))),
         }
     }
 
@@ -290,55 +293,26 @@ impl Context {
                     }
                 }
                 self.emit_event(EventType::SelfavatarChanged);
-                Ok(())
-            }
-            Config::Selfstatus => {
-                let def = stock_str::status_line(self).await;
-                let val = if value.is_none() || value.unwrap() == def {
-                    None
-                } else {
-                    value
-                };
-
-                self.sql.set_raw_config(key, val).await?;
-                Ok(())
             }
             Config::DeleteDeviceAfter => {
-                let ret = self
-                    .sql
-                    .set_raw_config(key, value)
-                    .await
-                    .map_err(Into::into);
-                // Force chatlist reload to delete old messages immediately.
-                self.emit_event(EventType::MsgsChanged {
-                    msg_id: MsgId::new(0),
-                    chat_id: ChatId::new(0),
-                });
-                ret
+                let ret = self.sql.set_raw_config(key, value).await;
+                // Interrupt ephemeral loop to delete old messages immediately.
+                self.interrupt_ephemeral_task().await;
+                ret?
             }
             Config::Displayname => {
                 let value = value.map(improve_single_line_input);
                 self.sql.set_raw_config(key, value.as_deref()).await?;
-                Ok(())
-            }
-            Config::DeleteServerAfter => {
-                let ret = self
-                    .sql
-                    .set_raw_config(key, value)
-                    .await
-                    .map_err(Into::into);
-                job::schedule_resync(self).await?;
-                ret
             }
             _ => {
                 self.sql.set_raw_config(key, value).await?;
-                Ok(())
             }
         }
+        Ok(())
     }
 
     pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
-        self.set_config(key, if value { Some("1") } else { None })
+        self.set_config(key, if value { Some("1") } else { Some("0") })
             .await?;
         Ok(())
     }
@@ -359,6 +333,73 @@ impl Context {
     }
 }
 
+// Separate impl block for self address handling
+impl Context {
+    /// Determine whether the specified addr maps to the/a self addr.
+    /// Returns `false` if no addresses are configured.
+    pub(crate) async fn is_self_addr(&self, addr: &str) -> Result<bool> {
+        Ok(self
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .iter()
+            .any(|a| addr_cmp(addr, a))
+            || self
+                .get_secondary_self_addrs()
+                .await?
+                .iter()
+                .any(|a| addr_cmp(addr, a)))
+    }
+
+    /// Sets `primary_new` as the new primary self address and saves the old
+    /// primary address (if exists) as a secondary address.
+    ///
+    /// This should only be used by test code and during configure.
+    pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
+        // add old primary address (if exists) to secondary addresses
+        let mut secondary_addrs = self.get_all_self_addrs().await?;
+        // never store a primary address also as a secondary
+        secondary_addrs.retain(|a| !addr_cmp(a, primary_new));
+        self.set_config(
+            Config::SecondaryAddrs,
+            Some(secondary_addrs.join(" ").as_str()),
+        )
+        .await?;
+
+        self.set_config(Config::ConfiguredAddr, Some(primary_new))
+            .await?;
+
+        Ok(())
+    }
+
+    /// Returns all primary and secondary self addresses.
+    pub(crate) async fn get_all_self_addrs(&self) -> Result<Vec<String>> {
+        let primary_addrs = self.get_config(Config::ConfiguredAddr).await?.into_iter();
+        let secondary_addrs = self.get_secondary_self_addrs().await?.into_iter();
+
+        Ok(primary_addrs.chain(secondary_addrs).collect())
+    }
+
+    /// Returns all secondary self addresses.
+    pub(crate) async fn get_secondary_self_addrs(&self) -> Result<Vec<String>> {
+        let secondary_addrs = self
+            .get_config(Config::SecondaryAddrs)
+            .await?
+            .unwrap_or_default();
+        Ok(secondary_addrs
+            .split_ascii_whitespace()
+            .map(|s| s.to_string())
+            .collect())
+    }
+
+    /// Returns the primary self address.
+    /// Returns an error if no self addr is configured.
+    pub async fn get_primary_self_addr(&self) -> Result<String> {
+        self.get_config(Config::ConfiguredAddr)
+            .await?
+            .context("No self addr configured")
+    }
+}
+
 /// Returns all available configuration keys concated together.
 fn get_config_keys_string() -> String {
     let keys = Config::iter().fold(String::new(), |mut acc, key| {
@@ -430,4 +471,70 @@ mod tests {
 
         Ok(())
     }
+
+    /// Regression test for https://github.com/deltachat/deltachat-core-rust/issues/3012
+    #[async_std::test]
+    async fn test_set_config_bool() -> Result<()> {
+        let t = TestContext::new().await;
+
+        // We need some config that defaults to true
+        let c = Config::E2eeEnabled;
+        assert_eq!(t.get_config_bool(c).await?, true);
+        t.set_config_bool(c, false).await?;
+        assert_eq!(t.get_config_bool(c).await?, false);
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_self_addrs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        assert!(alice.is_self_addr("alice@example.org").await?);
+        assert_eq!(alice.get_all_self_addrs().await?, vec!["alice@example.org"]);
+        assert!(!alice.is_self_addr("alice@alice.com").await?);
+
+        // Test adding the same primary address
+        alice.set_primary_self_addr("alice@example.org").await?;
+        alice.set_primary_self_addr("Alice@Example.Org").await?;
+        assert_eq!(alice.get_all_self_addrs().await?, vec!["Alice@Example.Org"]);
+
+        // Test adding a new (primary) self address
+        // The address is trimmed during by `LoginParam::from_database()`,
+        // so `set_primary_self_addr()` doesn't have to trim it.
+        alice.set_primary_self_addr(" Alice@alice.com ").await?;
+        assert!(alice.is_self_addr("    aliCe@example.org").await?);
+        assert!(alice.is_self_addr("alice@alice.com").await?);
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec![" Alice@alice.com ", "Alice@Example.Org"]
+        );
+
+        // Check that the entry is not duplicated
+        alice.set_primary_self_addr("alice@alice.com").await?;
+        alice.set_primary_self_addr("alice@alice.com").await?;
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec!["alice@alice.com", "Alice@Example.Org"]
+        );
+
+        // Test switching back
+        alice.set_primary_self_addr("alice@example.org").await?;
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec!["alice@example.org", "alice@alice.com"]
+        );
+
+        // Test setting a new primary self address, the previous self address
+        // should be kept as a secondary self address
+        alice.set_primary_self_addr("alice@alice.xyz").await?;
+        assert_eq!(
+            alice.get_all_self_addrs().await?,
+            vec!["alice@alice.xyz", "alice@example.org", "alice@alice.com"]
+        );
+        assert!(alice.is_self_addr("alice@example.org").await?);
+        assert!(alice.is_self_addr("alice@alice.com").await?);
+        assert!(alice.is_self_addr("Alice@alice.xyz").await?);
+
+        Ok(())
+    }
 }

src/configure.rs

@@ -8,25 +8,23 @@ mod server_params;
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
 use async_std::task;
-use itertools::Itertools;
 use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
-use crate::dc_tools::EmailAddress;
+use crate::config::Config;
+use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2};
+use crate::context::Context;
+use crate::dc_tools::{time, EmailAddress};
 use crate::imap::Imap;
+use crate::job;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
-use crate::message::Message;
+use crate::message::{Message, Viewtype};
 use crate::oauth2::dc_get_oauth2_addr;
+use crate::param::Params;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
 use crate::stock_str;
 use crate::{chat, e2ee, provider};
-use crate::{config::Config, dc_tools::time};
-use crate::{
-    constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2},
-    job,
-};
-use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
@@ -87,7 +85,7 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let mut param = LoginParam::from_database(self, "").await?;
+        let mut param = LoginParam::load_candidate_params(self).await?;
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
@@ -222,7 +220,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             "checking internal provider-info for offline autoconfig"
         );
 
-        if let Some(provider) = provider::get_provider_info(&param_domain, socks5_enabled).await {
+        if let Some(provider) =
+            provider::get_provider_info(ctx, &param_domain, socks5_enabled).await
+        {
             param.provider = Some(provider);
             match provider.status {
                 provider::Status::Ok | provider::Status::Preparation => {
@@ -442,8 +442,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 900);
 
-    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await?
-        || ctx.get_config_bool(Config::MvboxMove).await?;
+    let create_mvbox = ctx.should_watch_mvbox().await?;
 
     imap.configure_folders(ctx, create_mvbox).await?;
 
@@ -454,11 +453,14 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     drop(imap);
 
     progress!(ctx, 910);
-    // configuration success - write back the configured parameters with the
-    // "configured_" prefix; also write the "configured"-flag */
+
+    if ctx.get_config(Config::ConfiguredAddr).await?.as_deref() != Some(&param.addr) {
+        // Switched account, all server UIDs we know are invalid
+        job::schedule_resync(ctx).await?;
+    }
+
     // the trailing underscore is correct
-    param.save_to_database(ctx, "configured_").await?;
-    ctx.sql.set_raw_config_bool("configured", true).await?;
+    param.save_as_configured_params(ctx).await?;
     ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
@@ -476,6 +478,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 940);
     update_device_chats_handle.await?;
 
+    ctx.sql.set_raw_config_bool("configured", true).await?;
+
     Ok(())
 }
 
@@ -682,7 +686,11 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
         return first_err.msg.to_string();
     }
 
-    errors.iter().map(|e| e.to_string()).join("\n\n")
+    errors
+        .iter()
+        .map(|e| e.to_string())
+        .collect::<Vec<String>>()
+        .join("\n\n")
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -697,11 +705,11 @@ pub enum Error {
         error: quick_xml::Error,
     },
 
-    #[error("Failed to get URL: {0}")]
-    ReadUrl(#[from] self::read_url::Error),
-
     #[error("Number of redirection is exceeded")]
     Redirection,
+
+    #[error("{0:#}")]
+    Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]

src/configure/read_url.rs

@@ -1,20 +1,40 @@
 use crate::context::Context;
 
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("URL request error")]
-    GetError(surf::Error),
-}
+use anyhow::format_err;
+use anyhow::Context as _;
 
-pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
-    info!(context, "Requesting URL {}", url);
-
-    match surf::get(url).recv_string().await {
-        Ok(res) => Ok(res),
-        Err(err) => {
-            info!(context, "Can\'t read URL {}: {}", url, err);
-
-            Err(Error::GetError(err))
+pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
+    match read_url_inner(context, url).await {
+        Ok(s) => {
+            info!(context, "Successfully read url {}", url);
+            Ok(s)
+        }
+        Err(e) => {
+            info!(context, "Can't read URL {}: {:#}", url, e);
+            Err(format_err!("Can't read URL {}: {:#}", url, e))
         }
     }
 }
+
+pub async fn read_url_inner(context: &Context, mut url: &str) -> anyhow::Result<String> {
+    let mut _temp; // For the borrow checker
+
+    // Follow up to 10 http-redirects
+    for _i in 0..10 {
+        let mut response = surf::get(url).send().await.map_err(|e| e.into_inner())?;
+        if response.status().is_redirection() {
+            _temp = response
+                .header("location")
+                .context("Redirection doesn't have a target location")?
+                .last()
+                .to_string();
+            info!(context, "Following redirect to {}", _temp);
+            url = &_temp;
+            continue;
+        }
+
+        return response.body_string().await.map_err(|e| e.into_inner());
+    }
+
+    Err(format_err!("Followed 10 redirections"))
+}

src/configure/server_params.rs

@@ -52,10 +52,8 @@ impl ServerParams {
     fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
         if self.hostname.is_empty() {
             vec![
-                Self {
-                    hostname: param_domain.to_string(),
-                    ..self.clone()
-                },
+                // Try "imap.ex.org"/"smtp.ex.org" and "mail.ex.org" first because if a server exists
+                // under this address, it's likely the correct one.
                 Self {
                     hostname: match self.protocol {
                         Protocol::Imap => "imap.".to_string() + param_domain,
@@ -65,6 +63,12 @@ impl ServerParams {
                 },
                 Self {
                     hostname: "mail.".to_string() + param_domain,
+                    ..self.clone()
+                },
+                // Try "ex.org" last because if it's wrong and the server is configured to
+                // not answer at all, configuration may be stuck for several minutes.
+                Self {
+                    hostname: param_domain.to_string(),
                     ..self
                 },
             ]
@@ -296,5 +300,48 @@ mod tests {
                 strict_tls: Some(true)
             }],
         );
+
+        // Test that "example.net" is tried after "*.example.net".
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Imap,
+                hostname: "".to_string(),
+                port: 10480,
+                socket: Socket::Ssl,
+                username: "foobar".to_string(),
+                strict_tls: Some(true),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+        assert_eq!(
+            v,
+            vec![
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "imap.example.net".to_string(),
+                    port: 10480,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                },
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "mail.example.net".to_string(),
+                    port: 10480,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                },
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "example.net".to_string(),
+                    port: 10480,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string(),
+                    strict_tls: Some(true)
+                }
+            ],
+        );
     }
 }

src/constants.rs

@@ -179,16 +179,6 @@ pub const DC_ELLIPSIS: &str = "[...]";
 /// `char`s), not Unicode Grapheme Clusters.
 pub const DC_DESIRED_TEXT_LEN: usize = 5000;
 
-pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
-pub const DC_CONTACT_ID_SELF: u32 = 1;
-pub const DC_CONTACT_ID_INFO: u32 = 2;
-pub const DC_CONTACT_ID_DEVICE: u32 = 5;
-pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
-
-// decorative address that is used for DC_CONTACT_ID_DEVICE
-// when an api that returns an email is called.
-pub const DC_CONTACT_ID_DEVICE_ADDR: &str = "device@localhost";
-
 // Flags for empty server job
 
 pub const DC_EMPTY_MVBOX: u32 = 0x01;
@@ -230,79 +220,6 @@ pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 // can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
 pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
 
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
-#[repr(u32)]
-pub enum Viewtype {
-    Unknown = 0,
-
-    /// Text message.
-    /// The text of the message is set using dc_msg_set_text()
-    /// and retrieved with dc_msg_get_text().
-    Text = 10,
-
-    /// Image message.
-    /// If the image is an animated GIF, the type DC_MSG_GIF should be used.
-    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension
-    /// and retrieved via dc_msg_set_file(), dc_msg_set_dimension().
-    Image = 20,
-
-    /// Animated GIF message.
-    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension()
-    /// and retrieved via dc_msg_get_file(), dc_msg_get_width(), dc_msg_get_height().
-    Gif = 21,
-
-    /// Message containing a sticker, similar to image.
-    /// If possible, the ui should display the image without borders in a transparent way.
-    /// A click on a sticker will offer to install the sticker set in some future.
-    Sticker = 23,
-
-    /// Message containing an Audio file.
-    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
-    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration().
-    Audio = 40,
-
-    /// A voice message that was directly recorded by the user.
-    /// For all other audio messages, the type #DC_MSG_AUDIO should be used.
-    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
-    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration()
-    Voice = 41,
-
-    /// Video messages.
-    /// File, width, height and durarion
-    /// are set via dc_msg_set_file(), dc_msg_set_dimension(), dc_msg_set_duration()
-    /// and retrieved via
-    /// dc_msg_get_file(), dc_msg_get_width(),
-    /// dc_msg_get_height(), dc_msg_get_duration().
-    Video = 50,
-
-    /// Message containing any file, eg. a PDF.
-    /// The file is set via dc_msg_set_file()
-    /// and retrieved via dc_msg_get_file().
-    File = 60,
-
-    /// Message is an invitation to a videochat.
-    VideochatInvitation = 70,
-}
-
-impl Default for Viewtype {
-    fn default() -> Self {
-        Viewtype::Unknown
-    }
-}
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
@@ -314,32 +231,9 @@ pub enum KeyType {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use num_traits::FromPrimitive;
 
-    #[test]
-    fn test_derive_display_works_as_expected() {
-        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
-    }
-
-    #[test]
-    fn test_viewtype_values() {
-        // values may be written to disk and must not change
-        assert_eq!(Viewtype::Unknown, Viewtype::default());
-        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
-        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
-        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
-        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
-        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
-        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
-        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
-        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
-        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
-        assert_eq!(
-            Viewtype::VideochatInvitation,
-            Viewtype::from_i32(70).unwrap()
-        );
-    }
+    use super::*;
 
     #[test]
     fn test_chattype_values() {

src/context.rs

@@ -10,7 +10,6 @@ use async_std::{
     channel::{self, Receiver, Sender},
     path::{Path, PathBuf},
     sync::{Arc, Mutex, RwLock},
-    task,
 };
 
 use crate::chat::{get_chat_cnt, ChatId};
@@ -24,7 +23,6 @@ use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
 use crate::quota::QuotaInfo;
 use crate::scheduler::Scheduler;
-use crate::securejoin::Bob;
 use crate::sql::Sql;
 
 #[derive(Clone, Debug)]
@@ -42,13 +40,9 @@ impl Deref for Context {
 
 #[derive(Debug)]
 pub struct InnerContext {
-    /// Database file path
-    pub(crate) dbfile: PathBuf,
     /// Blob directory path
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
-    pub(crate) os_name: Option<String>,
-    pub(crate) bob: Bob,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -61,7 +55,6 @@ pub struct InnerContext {
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
-    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
     /// Recently loaded quota information, if any.
     /// Set to `None` if quota was never tried to load.
@@ -85,7 +78,7 @@ pub struct InnerContext {
 
 #[derive(Debug)]
 pub struct RunningState {
-    pub ongoing_running: bool,
+    ongoing_running: bool,
     shall_stop_ongoing: bool,
     cancel_sender: Option<Sender<()>>,
 }
@@ -107,10 +100,19 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 }
 
 impl Context {
-    /// Creates new context.
-    pub async fn new(os_name: String, dbfile: PathBuf, id: u32) -> Result<Context> {
-        // pretty_env_logger::try_init_timed().ok();
+    /// Creates new context and opens the database.
+    pub async fn new(dbfile: PathBuf, id: u32) -> Result<Context> {
+        let context = Self::new_closed(dbfile, id).await?;
 
+        // Open the database if is not encrypted.
+        if context.check_passphrase("".to_string()).await? {
+            context.sql.open(&context, "".to_string()).await?;
+        }
+        Ok(context)
+    }
+
+    /// Creates new context without opening the database.
+    pub async fn new_closed(dbfile: PathBuf, id: u32) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
@@ -118,11 +120,38 @@ impl Context {
         if !blobdir.exists().await {
             async_std::fs::create_dir_all(&blobdir).await?;
         }
-        Context::with_blobdir(os_name, dbfile, blobdir, id).await
+        let context = Context::with_blobdir(dbfile, blobdir, id).await?;
+        Ok(context)
+    }
+
+    /// Opens the database with the given passphrase.
+    ///
+    /// Returns true if passphrase is correct, false is passphrase is not correct. Fails on other
+    /// errors.
+    pub async fn open(&self, passphrase: String) -> Result<bool> {
+        if self.sql.check_passphrase(passphrase.clone()).await? {
+            self.sql.open(self, passphrase).await?;
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    /// Returns true if database is open.
+    pub async fn is_open(&self) -> bool {
+        self.sql.is_open().await
+    }
+
+    /// Tests the database passphrase.
+    ///
+    /// Returns true if passphrase is correct.
+    ///
+    /// Fails if database is already open.
+    pub(crate) async fn check_passphrase(&self, passphrase: String) -> Result<bool> {
+        self.sql.check_passphrase(passphrase).await
     }
 
     pub(crate) async fn with_blobdir(
-        os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
         id: u32,
@@ -136,11 +165,8 @@ impl Context {
         let inner = InnerContext {
             id,
             blobdir,
-            dbfile,
-            os_name: Some(os_name),
             running_state: RwLock::new(Default::default()),
-            sql: Sql::new(),
-            bob: Default::default(),
+            sql: Sql::new(dbfile),
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
@@ -148,7 +174,6 @@ impl Context {
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
-            ephemeral_task: RwLock::new(None),
             quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
@@ -158,24 +183,20 @@ impl Context {
         let ctx = Context {
             inner: Arc::new(inner),
         };
-        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }
 
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
-        info!(self, "starting IO");
-        if self.inner.is_io_running().await {
-            info!(self, "IO is already running");
+        if let Ok(false) = self.is_configured().await {
+            warn!(self, "can not start io on a context that is not configured");
             return;
         }
 
-        {
-            let l = &mut *self.inner.scheduler.write().await;
-            if let Err(err) = l.start(self.clone()).await {
-                error!(self, "Failed to start IO: {}", err)
-            }
+        info!(self, "starting IO");
+        if let Err(err) = self.inner.scheduler.write().await.start(self.clone()).await {
+            error!(self, "Failed to start IO: {}", err)
         }
     }
 
@@ -183,7 +204,9 @@ impl Context {
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
 
-        self.inner.stop_io().await;
+        if let Err(err) = self.inner.stop_io().await {
+            warn!(self, "failed to stop IO: {}", err);
+        }
     }
 
     /// Returns a reference to the underlying SQL instance.
@@ -196,7 +219,7 @@ impl Context {
 
     /// Returns database file path.
     pub fn get_dbfile(&self) -> &Path {
-        self.dbfile.as_path()
+        self.sql.dbfile.as_path()
     }
 
     /// Returns blob directory path.
@@ -212,6 +235,24 @@ impl Context {
         });
     }
 
+    /// Emits a generic MsgsChanged event (without chat or message id)
+    pub fn emit_msgs_changed_without_ids(&self) {
+        self.emit_event(EventType::MsgsChanged {
+            chat_id: ChatId::new(0),
+            msg_id: MsgId::new(0),
+        });
+    }
+
+    /// Emits a MsgsChanged event with specified chat and message ids
+    pub fn emit_msgs_changed(&self, chat_id: ChatId, msg_id: MsgId) {
+        self.emit_event(EventType::MsgsChanged { chat_id, msg_id });
+    }
+
+    /// Emits an IncomingMsg event with specified chat and message ids
+    pub fn emit_incoming_msg(&self, chat_id: ChatId, msg_id: MsgId) {
+        self.emit_event(EventType::IncomingMsg { chat_id, msg_id });
+    }
+
     /// Returns a receiver for emitted events.
     ///
     /// Multiple emitters can be created, but note that in this case each emitted event will
@@ -227,7 +268,7 @@ impl Context {
 
     // Ongoing process allocation/free/check
 
-    pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
+    pub(crate) async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
         if self.has_ongoing().await {
             bail!("There is already another ongoing process running.");
         }
@@ -243,7 +284,7 @@ impl Context {
         Ok(receiver)
     }
 
-    pub async fn free_ongoing(&self) {
+    pub(crate) async fn free_ongoing(&self) {
         let s_a = &self.running_state;
         let mut s = s_a.write().await;
 
@@ -252,7 +293,7 @@ impl Context {
         s.cancel_sender.take();
     }
 
-    pub async fn has_ongoing(&self) -> bool {
+    pub(crate) async fn has_ongoing(&self) -> bool {
         let s_a = &self.running_state;
         let s = s_a.read().await;
 
@@ -277,7 +318,7 @@ impl Context {
         };
     }
 
-    pub async fn shall_stop_ongoing(&self) -> bool {
+    pub(crate) async fn shall_stop_ongoing(&self) -> bool {
         self.running_state.read().await.shall_stop_ongoing
     }
 
@@ -287,8 +328,9 @@ impl Context {
 
     pub async fn get_info(&self) -> Result<BTreeMap<&'static str, String>> {
         let unset = "0";
-        let l = LoginParam::from_database(self, "").await?;
-        let l2 = LoginParam::from_database(self, "configured_").await?;
+        let l = LoginParam::load_candidate_params(self).await?;
+        let l2 = LoginParam::load_configured_params(self).await?;
+        let secondary_addrs = self.get_secondary_self_addrs().await?.join(", ");
         let displayname = self.get_config(Config::Displayname).await?;
         let chats = get_chat_cnt(self).await? as usize;
         let unblocked_msgs = message::get_unblocked_msg_cnt(self).await as usize;
@@ -325,11 +367,9 @@ impl Context {
             Err(err) => format!("<key failure: {}>", err),
         };
 
-        let inbox_watch = self.get_config_int(Config::InboxWatch).await?;
         let sentbox_watch = self.get_config_int(Config::SentboxWatch).await?;
-        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await?;
         let mvbox_move = self.get_config_int(Config::MvboxMove).await?;
-        let sentbox_move = self.get_config_int(Config::SentboxMove).await?;
+        let only_fetch_mvbox = self.get_config_int(Config::OnlyFetchMvbox).await?;
         let folders_configured = self
             .sql
             .get_raw_config_int("folders_configured")
@@ -355,6 +395,13 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
+        res.insert(
+            "database_encrypted",
+            self.sql
+                .is_encrypted()
+                .await
+                .map_or_else(|| "closed".to_string(), |b| b.to_string()),
+        );
         res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
@@ -368,6 +415,7 @@ impl Context {
         res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
+        res.insert("secondary_addrs", secondary_addrs);
         res.insert(
             "fetch_existing_msgs",
             self.get_config_int(Config::FetchExistingMsgs)
@@ -384,11 +432,9 @@ impl Context {
                 .await?
                 .to_string(),
         );
-        res.insert("inbox_watch", inbox_watch.to_string());
         res.insert("sentbox_watch", sentbox_watch.to_string());
-        res.insert("mvbox_watch", mvbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
-        res.insert("sentbox_move", sentbox_move.to_string());
+        res.insert("only_fetch_mvbox", only_fetch_mvbox.to_string());
         res.insert("folders_configured", folders_configured.to_string());
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
@@ -576,19 +622,14 @@ impl Context {
         Ok(mvbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_spam_folder(&self, folder_name: &str) -> Result<bool> {
-        let spam = self.get_config(Config::ConfiguredSpamFolder).await?;
-        Ok(spam.as_deref() == Some(folder_name))
-    }
-
-    pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
+    pub(crate) fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         dbfile.with_file_name(blob_fname)
     }
 
-    pub fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
+    pub(crate) fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
         let mut wal_fname = OsString::new();
         wal_fname.push(dbfile.file_name().unwrap_or_default());
         wal_fname.push("-wal");
@@ -597,25 +638,9 @@ impl Context {
 }
 
 impl InnerContext {
-    async fn is_io_running(&self) -> bool {
-        self.scheduler.read().await.is_running()
-    }
-
-    async fn stop_io(&self) {
-        if self.is_io_running().await {
-            let token = {
-                let lock = &*self.scheduler.read().await;
-                lock.pre_stop().await
-            };
-            {
-                let lock = &mut *self.scheduler.write().await;
-                lock.stop(token).await;
-            }
-        }
-
-        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
-            ephemeral_task.cancel().await;
-        }
+    async fn stop_io(&self) -> Result<()> {
+        self.scheduler.write().await.stop().await?;
+        Ok(())
     }
 }
 
@@ -640,21 +665,26 @@ mod tests {
     use crate::chat::{
         get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
     };
-    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+    use crate::contact::ContactId;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::dc_tools::dc_create_outgoing_rfc724_mid;
-    use crate::message::Message;
+    use crate::message::{Message, Viewtype};
     use crate::test_utils::TestContext;
+    use anyhow::Context as _;
     use std::time::Duration;
     use strum::IntoEnumIterator;
+    use tempfile::tempdir;
 
     #[async_std::test]
-    async fn test_wrong_db() {
-        let tmp = tempfile::tempdir().unwrap();
+    async fn test_wrong_db() -> Result<()> {
+        let tmp = tempfile::tempdir()?;
         let dbfile = tmp.path().join("db.sqlite");
-        std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new("FakeOs".into(), dbfile.into(), 1).await;
-        assert!(res.is_err());
+        std::fs::write(&dbfile, b"123")?;
+        let res = Context::new(dbfile.into(), 1).await?;
+
+        // Broken database is indistinguishable from encrypted one.
+        assert_eq!(res.is_open().await, false);
+        Ok(())
     }
 
     #[async_std::test]
@@ -671,7 +701,7 @@ mod tests {
             .unwrap();
         let msg = format!(
             "From: {}\n\
-             To: alice@example.com\n\
+             To: alice@example.org\n\
              Message-ID: <{}>\n\
              Chat-Version: 1.0\n\
              Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
@@ -681,9 +711,7 @@ mod tests {
             dc_create_outgoing_rfc724_mid(None, contact.get_addr())
         );
         println!("{}", msg);
-        dc_receive_imf(t, msg.as_bytes(), "INBOX", 1, false)
-            .await
-            .unwrap();
+        dc_receive_imf(t, msg.as_bytes(), false).await.unwrap();
     }
 
     #[async_std::test]
@@ -804,9 +832,7 @@ mod tests {
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new("FakeOS".into(), dbfile.into(), 1)
-            .await
-            .unwrap();
+        Context::new(dbfile.into(), 1).await.unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -817,7 +843,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new("FakeOS".into(), dbfile.into(), 1).await;
+        let res = Context::new(dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -827,9 +853,7 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new("FakeOS".into(), dbfile.into(), 1)
-            .await
-            .unwrap();
+        Context::new(dbfile.into(), 1).await.unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -839,7 +863,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir, 1).await;
+        let res = Context::with_blobdir(dbfile.into(), blobdir, 1).await;
         assert!(res.is_err());
     }
 
@@ -848,7 +872,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
+        let res = Context::with_blobdir(dbfile.into(), blobdir.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -925,7 +949,7 @@ mod tests {
     #[async_std::test]
     async fn test_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
-        let self_talk = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
+        let self_talk = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
         let chat = alice
             .create_chat_with_contact("Bob", "bob@example.org")
             .await;
@@ -1010,4 +1034,29 @@ mod tests {
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_check_passphrase() -> Result<()> {
+        let dir = tempdir()?;
+        let dbfile = dir.path().join("db.sqlite");
+
+        let id = 1;
+        let context = Context::new_closed(dbfile.clone().into(), id)
+            .await
+            .context("failed to create context")?;
+        assert_eq!(context.open("foo".to_string()).await?, true);
+        assert_eq!(context.is_open().await, true);
+        drop(context);
+
+        let id = 2;
+        let context = Context::new(dbfile.into(), id)
+            .await
+            .context("failed to create context")?;
+        assert_eq!(context.is_open().await, false);
+        assert_eq!(context.check_passphrase("bar".to_string()).await?, false);
+        assert_eq!(context.open("false".to_string()).await?, false);
+        assert_eq!(context.open("foo".to_string()).await?, true);
+
+        Ok(())
+    }
 }

src/dc_tools.rs

@@ -5,6 +5,7 @@ use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
 use std::io::Cursor;
+use std::str::from_utf8;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
@@ -12,15 +13,18 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use anyhow::{bail, Error};
+use anyhow::Error;
 use chrono::{Local, TimeZone};
+use mailparse::dateparse;
+use mailparse::headers::Headers;
+use mailparse::MailHeaderMap;
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
-use crate::constants::{Viewtype, DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
+use crate::constants::{DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::events::EventType;
-use crate::message::Message;
+use crate::message::{Message, Viewtype};
 use crate::provider::get_provider_update_timestamp;
 use crate::stock_str;
 
@@ -67,7 +71,7 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
     let lt = Local::now();
-    lt.offset().local_minus_utc() as i64
+    i64::from(lt.offset().local_minus_utc())
 }
 
 // timesmearing
@@ -82,7 +86,7 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
 //   `last_smeared_timestamp` is again in sync with the normal time.
 // - however, we do not do all this for the far future,
 //   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
-const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
+pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 
 /// Returns the current smeared timestamp,
 ///
@@ -191,44 +195,29 @@ async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time
 }
 
 /* Message-ID tools */
+
+/// Generate an ID. The generated ID should be as short and as unique as possible:
+/// - short, because it may also used as part of Message-ID headers or in QR codes
+/// - unique as two IDs generated on two devices should not be the same. However, collisions are not world-wide but only by the few contacts.
+/// IDs generated by this function are 66 bit wide and are returned as 11 base64 characters.
+///
+/// Additional information when used as a message-id or group-id:
+/// - for OUTGOING messages this ID is written to the header as `Chat-Group-ID:` and is added to the message ID as Gr.<grpid>.<random>@<random>
+/// - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header
+/// - the group-id should be a string with the characters [a-zA-Z0-9\-_]
 pub(crate) fn dc_create_id() -> String {
-    /* generate an id. the generated ID should be as short and as unique as possible:
-    - short, because it may also used as part of Message-ID headers or in QR codes
-    - unique as two IDs generated on two devices should not be the same. However, collisions are not world-wide but only by the few contacts.
-    IDs generated by this function are 66 bit wide and are returned as 11 base64 characters.
-    If possible, RNG of OpenSSL is used.
-
-    Additional information when used as a message-id or group-id:
-    - for OUTGOING messages this ID is written to the header as `Chat-Group-ID:` and is added to the message ID as Gr.<grpid>.<random>@<random>
-    - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header
-    - the group-id should be a string with the characters [a-zA-Z0-9\-_] */
-
+    // ThreadRng implements CryptoRng trait and is supposed to be cryptographically secure.
     let mut rng = thread_rng();
-    let buf: [u32; 3] = [rng.gen(), rng.gen(), rng.gen()];
 
-    encode_66bits_as_base64(buf[0usize], buf[1usize], buf[2usize])
-}
+    // Generate 72 random bits.
+    let mut arr = [0u8; 9];
+    rng.fill(&mut arr[..]);
 
-/// Encode 66 bits as a base64 string.
-/// This is useful for ID generating with short strings as we save 5 character
-/// in each id compared to 64 bit hex encoding. For a typical group ID, these
-/// are 10 characters (grpid+msgid):
-///    hex:    64 bit, 4 bits/character, length = 64/4 = 16 characters
-///    base64: 64 bit, 6 bits/character, length = 64/6 = 11 characters (plus 2 additional bits)
-/// Only the lower 2 bits of `fill` are used.
-fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
-    use byteorder::{BigEndian, WriteBytesExt};
-
-    let mut wrapped_writer = Vec::new();
-    {
-        let mut enc = base64::write::EncoderWriter::new(&mut wrapped_writer, base64::URL_SAFE);
-        enc.write_u32::<BigEndian>(v1).unwrap();
-        enc.write_u32::<BigEndian>(v2).unwrap();
-        enc.write_u8(((fill & 0x3) as u8) << 6).unwrap();
-        enc.finish().unwrap();
-    }
-    assert_eq!(wrapped_writer.pop(), Some(b'A')); // Remove last "A"
-    String::from_utf8(wrapped_writer).unwrap()
+    // Take 11 base64 characters containing 66 random bits.
+    base64::encode_config(&arr, base64::URL_SAFE)
+        .chars()
+        .take(11)
+        .collect()
 }
 
 /// Function generates a Message-ID that can be used for a new outgoing message.
@@ -350,63 +339,6 @@ pub async fn dc_delete_files_in_dir(context: &Context, path: impl AsRef<Path>) {
     }
 }
 
-pub(crate) async fn dc_copy_file(
-    context: &Context,
-    src_path: impl AsRef<Path>,
-    dest_path: impl AsRef<Path>,
-) -> bool {
-    let src_abs = dc_get_abs_path(context, &src_path);
-    let mut src_file = match fs::File::open(&src_abs).await {
-        Ok(file) => file,
-        Err(err) => {
-            warn!(
-                context,
-                "failed to open for read '{}': {}",
-                src_abs.display(),
-                err
-            );
-            return false;
-        }
-    };
-
-    let dest_abs = dc_get_abs_path(context, &dest_path);
-    let mut dest_file = match fs::OpenOptions::new()
-        .create_new(true)
-        .write(true)
-        .open(&dest_abs)
-        .await
-    {
-        Ok(file) => file,
-        Err(err) => {
-            warn!(
-                context,
-                "failed to open for write '{}': {}",
-                dest_abs.display(),
-                err
-            );
-            return false;
-        }
-    };
-
-    match io::copy(&mut src_file, &mut dest_file).await {
-        Ok(_) => true,
-        Err(err) => {
-            error!(
-                context,
-                "Cannot copy \"{}\" to \"{}\": {}",
-                src_abs.display(),
-                dest_abs.display(),
-                err
-            );
-            {
-                // Attempt to remove the failed file, swallow errors resulting from that.
-                fs::remove_file(dest_abs).await.ok();
-            }
-            false
-        }
-    }
-}
-
 pub(crate) async fn dc_create_folder(
     context: &Context,
     path: impl AsRef<Path>,
@@ -504,33 +436,6 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
-/// it can be renamed to dest_path. This guarantees that the backup is complete.
-pub(crate) async fn get_next_backup_path(
-    folder: impl AsRef<Path>,
-    backup_time: i64,
-) -> Result<(PathBuf, PathBuf), Error> {
-    let folder = PathBuf::from(folder.as_ref());
-    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
-        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
-        .format("delta-chat-backup-%Y-%m-%d")
-        .to_string();
-
-    // 64 backup files per day should be enough for everyone
-    for i in 0..64 {
-        let mut tempfile = folder.clone();
-        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
-
-        let mut destfile = folder.clone();
-        destfile.push(format!("{}-{:02}.tar", stem, i));
-
-        if !tempfile.exists().await && !destfile.exists().await {
-            return Ok((tempfile, destfile));
-        }
-    }
-    bail!("could not create backup file, disk full?");
-}
-
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
@@ -632,8 +537,8 @@ impl rusqlite::types::ToSql for EmailAddress {
 /// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
 pub(crate) fn improve_single_line_input(input: &str) -> String {
     input
-        .replace("\n", " ")
-        .replace("\r", " ")
+        .replace('\n', " ")
+        .replace('\r', " ")
         .trim()
         .to_string()
 }
@@ -670,13 +575,144 @@ pub fn remove_subject_prefix(last_subject: &str) -> String {
         .to_string()
 }
 
+// Types and methods to create hop-info for message-info
+
+fn extract_address_from_receive_header<'a>(header: &'a str, start: &str) -> Option<&'a str> {
+    let header_len = header.len();
+    header.find(start).and_then(|mut begin| {
+        begin += start.len();
+        let end = header
+            .get(begin..)?
+            .find(|c: char| c.is_whitespace())
+            .unwrap_or(header_len);
+        header.get(begin..begin + end)
+    })
+}
+
+pub(crate) fn parse_receive_header(header: &str) -> String {
+    let header = header.replace(&['\r', '\n'][..], "");
+    let mut hop_info = String::from("Hop: ");
+
+    if let Some(from) = extract_address_from_receive_header(&header, "from ") {
+        hop_info += &format!("From: {}; ", from.trim());
+    }
+
+    if let Some(by) = extract_address_from_receive_header(&header, "by ") {
+        hop_info += &format!("By: {}; ", by.trim());
+    }
+
+    if let Ok(date) = dateparse(&header) {
+        // In tests, use the UTC timezone so that the test is reproducible
+        #[cfg(test)]
+        let date_obj = chrono::Utc.timestamp(date, 0);
+        #[cfg(not(test))]
+        let date_obj = Local.timestamp(date, 0);
+
+        hop_info += &format!("Date: {}", date_obj.to_rfc2822());
+    };
+
+    hop_info
+}
+
+/// parses "receive"-headers
+pub(crate) fn parse_receive_headers(headers: &Headers) -> String {
+    headers
+        .get_all_headers("Received")
+        .iter()
+        .rev()
+        .filter_map(|header_map_item| from_utf8(header_map_item.get_value_raw()).ok())
+        .map(parse_receive_header)
+        .collect::<Vec<_>>()
+        .join("\n")
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
 
-    use crate::test_utils::TestContext;
+    use crate::{
+        config::Config, dc_receive_imf::dc_receive_imf, message::get_msg_info,
+        test_utils::TestContext,
+    };
+
+    #[test]
+    fn test_parse_receive_headers() {
+        // Test `parse_receive_headers()` with some more-or-less random emails from the test-data
+        let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
+        let expected =
+            "Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000\n\
+             Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000";
+        check_parse_receive_headers(raw, expected);
+
+        let raw = include_bytes!("../test-data/message/wrong-html.eml");
+        let expected =
+            "Hop: From: oxbsltgw18.schlund.de; By: mrelayeu.kundenserver.de; Date: Thu, 06 Aug 2020 16:40:31 +0000\n\
+             Hop: From: mout.kundenserver.de; By: dd37930.kasserver.com; Date: Thu, 06 Aug 2020 16:40:32 +0000";
+        check_parse_receive_headers(raw, expected);
+
+        let raw = include_bytes!("../test-data/message/posteo_ndn.eml");
+        let expected =
+            "Hop: By: mout01.posteo.de; Date: Tue, 09 Jun 2020 18:44:22 +0000\n\
+             Hop: From: mout01.posteo.de; By: mx04.posteo.de; Date: Tue, 09 Jun 2020 18:44:22 +0000\n\
+             Hop: From: mx04.posteo.de; By: mailin06.posteo.de; Date: Tue, 09 Jun 2020 18:44:23 +0000\n\
+             Hop: From: mailin06.posteo.de; By: proxy02.posteo.de; Date: Tue, 09 Jun 2020 18:44:23 +0000\n\
+             Hop: From: proxy02.posteo.de; By: proxy02.posteo.name; Date: Tue, 09 Jun 2020 18:44:23 +0000\n\
+             Hop: From: proxy02.posteo.name; By: dovecot03.posteo.local; Date: Tue, 09 Jun 2020 18:44:24 +0000";
+        check_parse_receive_headers(raw, expected);
+    }
+
+    fn check_parse_receive_headers(raw: &[u8], expected: &str) {
+        let mail = mailparse::parse_mail(raw).unwrap();
+        let hop_info = parse_receive_headers(&mail.get_headers());
+        assert_eq!(hop_info, expected)
+    }
+
+    #[async_std::test]
+    async fn test_parse_receive_headers_integration() {
+        let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
+        let expected = r"State: Fresh
+
+hi
+
+Message-ID: 2dfdbde7@example.org
+
+Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000
+Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000";
+        check_parse_receive_headers_integration(raw, expected).await;
+
+        let raw = include_bytes!("../test-data/message/encrypted_with_received_headers.eml");
+        let expected = "State: Fresh, Encrypted
+
+Re: Message from alice@example.org
+
+hi back\r\n\
+\r\n\
+-- \r\n\
+Sent with my Delta Chat Messenger: https://delta.chat
+
+Message-ID: Mr.adQpEwndXLH.LPDdlFVJ7wG@example.net
+
+Hop: From: [127.0.0.1]; By: mail.example.org; Date: Mon, 27 Dec 2021 11:21:21 +0000
+Hop: From: mout.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
+Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000";
+        check_parse_receive_headers_integration(raw, expected).await;
+    }
+
+    async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
+        let t = TestContext::new_alice().await;
+        t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
+        dc_receive_imf(&t, raw, false).await.unwrap();
+        let msg = t.get_last_msg().await;
+        let msg_info = get_msg_info(&t, msg.id).await.unwrap();
+
+        // Ignore the first rows of the msg_info because they contain a
+        // received time that depends on the test time which makes it impossible to
+        // compare with a static string
+        let capped_result = &msg_info[msg_info.find("State").unwrap()..];
+        assert_eq!(expected, capped_result);
+    }
 
     #[test]
     fn test_rust_ftoa() {
@@ -730,23 +766,12 @@ mod tests {
     }
 
     #[test]
-    fn test_encode_66bits_as_base64() {
-        assert_eq!(
-            encode_66bits_as_base64(0x01234567, 0x89abcdef, 0),
-            "ASNFZ4mrze8"
-        );
-        assert_eq!(
-            encode_66bits_as_base64(0x01234567, 0x89abcdef, 1),
-            "ASNFZ4mrze9"
-        );
-        assert_eq!(
-            encode_66bits_as_base64(0x01234567, 0x89abcdef, 2),
-            "ASNFZ4mrze-"
-        );
-        assert_eq!(
-            encode_66bits_as_base64(0x01234567, 0x89abcdef, 3),
-            "ASNFZ4mrze_"
-        );
+    fn test_dc_create_id_invalid_chars() {
+        for _ in 1..1000 {
+            let buf = dc_create_id();
+            assert!(!buf.contains('/')); // `/` must not be used to be URL-safe
+            assert!(!buf.contains('.')); // `.` is used as a delimiter when extracting grpid from Message-ID
+        }
     }
 
     #[test]
@@ -890,20 +915,7 @@ mod tests {
 
         assert!(dc_file_exist!(context, &abs_path).await);
 
-        assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
-
-        // attempting to copy a second time should fail
-        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
-
-        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada").await, 7);
-
-        let buf = dc_read_file(context, "$BLOBDIR/dada").await.unwrap();
-
-        assert_eq!(buf.len(), 7);
-        assert_eq!(&buf, b"content");
-
         assert!(dc_delete_file(context, "$BLOBDIR/foobar").await);
-        assert!(dc_delete_file(context, "$BLOBDIR/dada").await);
         assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder")
             .await
             .is_ok());
@@ -1029,7 +1041,7 @@ mod tests {
         maybe_warn_on_bad_time(&t, timestamp_past, get_provider_update_timestamp()).await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0);
+        let device_chat_id = chats.get_chat_id(0).unwrap();
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1067,7 +1079,7 @@ mod tests {
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        assert_eq!(device_chat_id, chats.get_chat_id(0));
+        assert_eq!(device_chat_id, chats.get_chat_id(0).unwrap());
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1099,7 +1111,7 @@ mod tests {
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0);
+        let device_chat_id = chats.get_chat_id(0).unwrap();
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1121,7 +1133,7 @@ mod tests {
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0);
+        let device_chat_id = chats.get_chat_id(0).unwrap();
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1138,7 +1150,7 @@ mod tests {
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0);
+        let device_chat_id = chats.get_chat_id(0).unwrap();
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();

src/dehtml.rs

@@ -36,9 +36,9 @@ impl Dehtml {
             ""
         }
     }
-    fn append_prefix(&self, line_end: impl AsRef<str>) -> String {
+    fn append_prefix(&self, line_end: &str) -> String {
         // line_end is e.g. "\n\n". We add "> " if necessary.
-        line_end.as_ref().to_owned() + self.line_prefix()
+        line_end.to_string() + self.line_prefix()
     }
     fn get_add_text(&self) -> AddText {
         if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {

src/download.rs

@@ -3,14 +3,14 @@
 use anyhow::{anyhow, Result};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
 
 use crate::config::Config;
-use crate::constants::Viewtype;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::imap::{Imap, ImapActionResult};
 use crate::job::{self, Action, Job, Status};
-use crate::message::{Message, MsgId};
+use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
 use crate::param::Params;
 use crate::{job_try, stock_str, EventType};
@@ -129,24 +129,48 @@ impl Job {
         }
 
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let server_folder = msg.server_folder.unwrap_or_default();
-        match imap
-            .fetch_single_msg(context, &server_folder, msg.server_uid)
-            .await
-        {
-            ImapActionResult::RetryLater | ImapActionResult::Failed => {
-                job_try!(
-                    msg.id
-                        .update_download_state(context, DownloadState::Failure)
-                        .await
-                );
-                Status::Finished(Err(anyhow!("Call download_full() again to try over.")))
-            }
-            ImapActionResult::Success | ImapActionResult::AlreadyDone => {
-                // update_download_state() not needed as receive_imf() already
-                // set the state and emitted the event.
-                Status::Finished(Ok(()))
+        let row = job_try!(
+            context
+                .sql
+                .query_row_optional(
+                    "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''",
+                    paramsv![msg.rfc724_mid],
+                    |row| {
+                        let server_uid: u32 = row.get(0)?;
+                        let server_folder: String = row.get(1)?;
+                        Ok((server_uid, server_folder))
+                    }
+                )
+                .await
+        );
+
+        if let Some((server_uid, server_folder)) = row {
+            match imap
+                .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone())
+                .await
+            {
+                ImapActionResult::RetryLater | ImapActionResult::Failed => {
+                    job_try!(
+                        msg.id
+                            .update_download_state(context, DownloadState::Failure)
+                            .await
+                    );
+                    Status::Finished(Err(anyhow!("Call download_full() again to try over.")))
+                }
+                ImapActionResult::Success => {
+                    // update_download_state() not needed as receive_imf() already
+                    // set the state and emitted the event.
+                    Status::Finished(Ok(()))
+                }
             }
+        } else {
+            // No IMAP record found, we don't know the UID and folder.
+            job_try!(
+                msg.id
+                    .update_download_state(context, DownloadState::Failure)
+                    .await
+            );
+            Status::Finished(Err(anyhow!("Call download_full() again to try over.")))
         }
     }
 }
@@ -161,6 +185,7 @@ impl Imap {
         context: &Context,
         folder: &str,
         uid: u32,
+        rfc724_mid: String,
     ) -> ImapActionResult {
         if let Some(imapresult) = self
             .prepare_imap_operation_on_msg(context, folder, uid)
@@ -172,14 +197,20 @@ impl Imap {
         // we are connected, and the folder is selected
         info!(context, "Downloading message {}/{} fully...", folder, uid);
 
-        let (_, error_cnt) = self
-            .fetch_many_msgs(context, folder, vec![uid], false, false)
-            .await;
-        if error_cnt > 0 {
-            return ImapActionResult::Failed;
+        let mut uid_message_ids: BTreeMap<u32, String> = BTreeMap::new();
+        uid_message_ids.insert(uid, rfc724_mid);
+        let (last_uid, _received) = match self
+            .fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, false, false)
+            .await
+        {
+            Ok(res) => res,
+            Err(_) => return ImapActionResult::Failed,
+        };
+        if last_uid.is_none() {
+            ImapActionResult::Failed
+        } else {
+            ImapActionResult::Success
         }
-
-        ImapActionResult::Success
     }
 }
 
@@ -223,13 +254,16 @@ impl MimeMessage {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use crate::chat::send_msg;
-    use crate::constants::Viewtype;
-    use crate::dc_receive_imf::dc_receive_imf_inner;
-    use crate::test_utils::TestContext;
     use num_traits::FromPrimitive;
 
+    use crate::chat::send_msg;
+    use crate::dc_receive_imf::dc_receive_imf_inner;
+    use crate::ephemeral::Timer;
+    use crate::message::Viewtype;
+    use crate::test_utils::TestContext;
+
+    use super::*;
+
     #[test]
     fn test_downloadstate_values() {
         // values may be written to disk and must not change
@@ -310,9 +344,8 @@ mod tests {
 
         dc_receive_imf_inner(
             &t,
+            "Mr.12345678901@example.com",
             header.as_bytes(),
-            "INBOX",
-            1,
             false,
             Some(100000),
             false,
@@ -328,9 +361,8 @@ mod tests {
 
         dc_receive_imf_inner(
             &t,
+            "Mr.12345678901@example.com",
             format!("{}\n\n100k text...", header).as_bytes(),
-            "INBOX",
-            1,
             false,
             None,
             false,
@@ -343,4 +375,39 @@ mod tests {
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_partial_download_and_ephemeral() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = t
+            .create_chat_with_contact("bob", "bob@example.org")
+            .await
+            .id;
+        chat_id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
+            .await?;
+
+        // download message from bob partially, this must not change the ephemeral timer
+        dc_receive_imf_inner(
+            &t,
+            "first@example.org",
+            b"From: Bob <bob@example.org>\n\
+                    To: Alice <alice@example.org>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: subject\n\
+                    Message-ID: <first@example.org>\n\
+                    Date: Sun, 14 Nov 2021 00:10:00 +0000\
+                    Content-Type: text/plain",
+            false,
+            Some(100000),
+            false,
+        )
+        .await?;
+        assert_eq!(
+            chat_id.get_ephemeral_timer(&t).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        Ok(())
+    }
 }

src/e2ee.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashSet;
 
-use anyhow::{bail, ensure, format_err, Result};
+use anyhow::{format_err, Context as _, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
@@ -28,13 +28,7 @@ impl EncryptHelper {
         let prefer_encrypt =
             EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await?)
                 .unwrap_or_default();
-        let addr = match context.get_config(Config::ConfiguredAddr).await? {
-            None => {
-                bail!("addr not configured!");
-            }
-            Some(addr) => addr,
-        };
-
+        let addr = context.get_primary_self_addr().await?;
         let public_key = SignedPublicKey::load_self(context).await?;
 
         Ok(EncryptHelper {
@@ -121,9 +115,9 @@ impl EncryptHelper {
             .into_iter()
             .filter_map(|(state, addr)| state.map(|s| (s, addr)))
         {
-            let key = peerstate.take_key(min_verified).ok_or_else(|| {
-                format_err!("proper enc-key for {} missing, cannot encrypt", addr)
-            })?;
+            let key = peerstate
+                .take_key(min_verified)
+                .with_context(|| format!("proper enc-key for {} missing, cannot encrypt", addr))?;
             keyring.add(key);
         }
         keyring.add(self.public_key.clone());
@@ -179,7 +173,6 @@ pub async fn try_decrypt(
     // Possibly perform decryption
     let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
     let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
-    let mut signatures = HashSet::default();
 
     if let Some(ref mut peerstate) = peerstate {
         peerstate
@@ -192,14 +185,17 @@ pub async fn try_decrypt(
         }
     }
 
-    let out_mail = decrypt_if_autocrypt_message(
+    let (out_mail, signatures) = match decrypt_if_autocrypt_message(
         context,
         mail,
         private_keyring,
         public_keyring_for_validate,
-        &mut signatures,
     )
-    .await?;
+    .await?
+    {
+        Some((out_mail, signatures)) => (Some(out_mail), signatures),
+        None => (None, Default::default()),
+    };
 
     if let Some(mut peerstate) = peerstate {
         // If message is not encrypted and it is not a read receipt, degrade encryption.
@@ -275,8 +271,7 @@ async fn decrypt_if_autocrypt_message(
     mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<Fingerprint>,
-) -> Result<Option<Vec<u8>>> {
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
     let encrypted_data_part = match get_autocrypt_mime(mail).or_else(|| get_mixed_up_mime(mail)) {
         None => {
             // not an autocrypt mime message, abort and ignore
@@ -290,36 +285,60 @@ async fn decrypt_if_autocrypt_message(
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
-        ret_valid_signatures,
     )
     .await
 }
 
+/// Validates signatures of Multipart/Signed message part, as defined in RFC 1847.
+///
+/// Returns `None` if the part is not a Multipart/Signed part, otherwise retruns the set of key
+/// fingerprints for which there is a valid signature.
+async fn validate_detached_signature(
+    mail: &ParsedMail<'_>,
+    public_keyring_for_validate: &Keyring<SignedPublicKey>,
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
+    if mail.ctype.mimetype != "multipart/signed" {
+        return Ok(None);
+    }
+
+    if let [first_part, second_part] = &mail.subparts[..] {
+        // First part is the content, second part is the signature.
+        let content = first_part.raw_bytes;
+        let signature = second_part.get_body_raw()?;
+        let ret_valid_signatures =
+            pgp::pk_validate(content, &signature, public_keyring_for_validate).await?;
+
+        Ok(Some((content.to_vec(), ret_valid_signatures)))
+    } else {
+        Ok(None)
+    }
+}
+
 /// Returns Ok(None) if nothing encrypted was found.
 async fn decrypt_part(
     mail: &ParsedMail<'_>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<Fingerprint>,
-) -> Result<Option<Vec<u8>>> {
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
     let data = mail.get_body_raw()?;
 
     if has_decrypted_pgp_armor(&data) {
-        // we should only have one decryption happening
-        ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
+        let (plain, ret_valid_signatures) =
+            pgp::pk_decrypt(data, private_keyring, &public_keyring_for_validate).await?;
 
-        let plain = pgp::pk_decrypt(
-            data,
-            private_keyring,
-            public_keyring_for_validate,
-            Some(ret_valid_signatures),
-        )
-        .await?;
+        // Check for detached signatures.
+        // If decrypted part is a multipart/signed, then there is a detached signature.
+        let decrypted_part = mailparse::parse_mail(&plain)?;
+        if let Some((content, valid_detached_signatures)) =
+            validate_detached_signature(&decrypted_part, &public_keyring_for_validate).await?
+        {
+            return Ok(Some((content, valid_detached_signatures)));
+        } else {
+            // If the message was wrongly or not signed, still return the plain text.
+            // The caller has to check the signatures then.
 
-        // If the message was wrongly or not signed, still return the plain text.
-        // The caller has to check the signatures then.
-
-        return Ok(Some(plain));
+            return Ok(Some((plain, ret_valid_signatures)));
+        }
     }
 
     Ok(None)
@@ -362,38 +381,31 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 /// [Config::ConfiguredAddr] is configured, this address is returned.
 // TODO, remove this once deltachat::key::Key no longer exists.
 pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await?
-        .ok_or_else(|| {
-            format_err!(concat!(
-                "Failed to get self address, ",
-                "cannot ensure secret key if not configured."
-            ))
-        })?;
+    let self_addr = context.get_primary_self_addr().await?;
     SignedPublicKey::load_self(context).await?;
     Ok(self_addr)
 }
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-
     use crate::chat;
-    use crate::constants::Viewtype;
-    use crate::message::Message;
+    use crate::message::{Message, Viewtype};
     use crate::param::Param;
     use crate::peerstate::ToSave;
     use crate::test_utils::{bob_keypair, TestContext};
 
+    use super::*;
+
     mod ensure_secret_key_exists {
         use super::*;
 
         #[async_std::test]
         async fn test_prexisting() {
-            let t = TestContext::new().await;
-            let test_addr = t.configure_alice().await;
-            assert_eq!(ensure_secret_key_exists(&t).await.unwrap(), test_addr);
+            let t = TestContext::new_alice().await;
+            assert_eq!(
+                ensure_secret_key_exists(&t).await.unwrap(),
+                "alice@example.org"
+            );
         }
 
         #[async_std::test]
@@ -464,7 +476,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert!(!msg.was_encrypted());
 
         // Parsing a message is enough to update peerstate
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
@@ -495,28 +507,28 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
         let msg = bob.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
         // Alice sends plaintext message with Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
-        msg.param.set_int(Param::ForcePlaintext, 1);
+        msg.force_plaintext();
         chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
         chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
         let sent = alice.pop_sent_msg().await;
 
         let msg = bob.parse_msg(&sent).await;
         assert!(!msg.was_encrypted());
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
         // Alice sends plaintext message without Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
-        msg.param.set_int(Param::ForcePlaintext, 1);
+        msg.force_plaintext();
         msg.param.set_int(Param::SkipAutocrypt, 1);
         chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
         chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
@@ -524,7 +536,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
         let msg = bob.parse_msg(&sent).await;
         assert!(!msg.was_encrypted());
-        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Reset);

src/ephemeral.rs

@@ -48,11 +48,11 @@
 //!
 //! ## When messages are deleted
 //!
-//! Local deletion happens when the chatlist or chat is loaded. A
-//! `MsgsChanged` event is emitted when a message deletion is due, to
-//! make UI reload displayed messages and cause actual deletion.
+//! The `ephemeral_loop` task schedules the next due running of
+//! `delete_expired_messages` which in turn emits `MsgsChanged` events
+//! when deleting local messages to make UIs reload displayed messages.
 //!
-//! Server deletion happens by generating IMAP deletion jobs based on
+//! Server deletion happens by updating the `imap` table based on
 //! the database entries which are expired either according to their
 //! ephemeral message timers or global `delete_server_after` setting.
 
@@ -62,20 +62,21 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{ensure, Context as _, Result};
-use async_std::task;
+use async_std::channel::Receiver;
+use async_std::future::timeout;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{send_msg, ChatId};
-use crate::constants::{
-    Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
-};
+use crate::constants::{DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH};
+use crate::contact::ContactId;
 use crate::context::Context;
-use crate::dc_tools::time;
+use crate::dc_tools::{duration_to_str, time};
 use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
-use crate::job;
-use crate::message::{Message, MessageState, MsgId};
+use crate::log::LogExt;
+use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
+use crate::sql;
 use crate::stock_str;
 use std::cmp::max;
 
@@ -197,7 +198,7 @@ impl ChatId {
         }
         self.inner_set_ephemeral_timer(context, timer).await?;
         let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, ContactId::SELF).await);
         msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
         if let Err(err) = send_msg(context, self, &mut msg).await {
             error!(
@@ -213,7 +214,7 @@ impl ChatId {
 pub(crate) async fn stock_ephemeral_timer_changed(
     context: &Context,
     timer: Timer,
-    from_id: u32,
+    from_id: ContactId,
 ) -> String {
     match timer {
         Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
@@ -263,7 +264,7 @@ pub(crate) async fn stock_ephemeral_timer_changed(
 
 impl MsgId {
     /// Returns ephemeral message timer value for the message.
-    pub(crate) async fn ephemeral_timer(self, context: &Context) -> anyhow::Result<Timer> {
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> Result<Timer> {
         let res = match context
             .sql
             .query_get_value(
@@ -279,7 +280,7 @@ impl MsgId {
     }
 
     /// Starts ephemeral message timer for the message if it is not started yet.
-    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> anyhow::Result<()> {
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> Result<()> {
         if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
             let ephemeral_timestamp = time().saturating_add(duration.into());
 
@@ -292,12 +293,43 @@ impl MsgId {
                     paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
                 )
                 .await?;
-            schedule_ephemeral_task(context).await;
+            context.interrupt_ephemeral_task().await;
         }
         Ok(())
     }
 }
 
+pub(crate) async fn start_ephemeral_timers_msgids(
+    context: &Context,
+    msg_ids: &[MsgId],
+) -> Result<()> {
+    let msg_ids: Vec<&dyn crate::ToSql> = msg_ids
+        .iter()
+        .map(|msg_id| msg_id as &dyn crate::ToSql)
+        .collect();
+    let now = time();
+    let count = context
+        .sql
+        .execute(
+            format!(
+                "UPDATE msgs SET ephemeral_timestamp = ? + ephemeral_timer
+         WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ? + ephemeral_timer) AND ephemeral_timer > 0
+         AND id IN ({})",
+                sql::repeat_vars(msg_ids.len())
+            ),
+            rusqlite::params_from_iter(
+                std::iter::once(&now as &dyn crate::ToSql)
+                    .chain(std::iter::once(&now as &dyn crate::ToSql))
+                    .chain(msg_ids),
+            ),
+        )
+        .await?;
+    if count > 0 {
+        context.interrupt_ephemeral_task().await;
+    }
+    Ok(())
+}
+
 /// Deletes messages which are expired according to
 /// `delete_device_after` setting or `ephemeral_timestamp` column.
 ///
@@ -306,7 +338,7 @@ impl MsgId {
 /// false. This function does not emit the MsgsChanged event itself,
 /// because it is also called when chatlist is reloaded, and emitting
 /// MsgsChanged there will cause infinite reload loop.
-pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool> {
+pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Result<()> {
     let mut updated = context
         .sql
         .execute(
@@ -322,21 +354,21 @@ WHERE
   AND ephemeral_timestamp <= ?
   AND chat_id != ?
 "#,
-            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
+            paramsv![DC_CHAT_ID_TRASH, now, DC_CHAT_ID_TRASH],
         )
         .await
         .context("update failed")?
         > 0;
 
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
             .await?
             .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
             .await?
             .unwrap_or_default();
 
-        let threshold_timestamp = time() - delete_device_after;
+        let threshold_timestamp = now.saturating_sub(delete_device_after);
 
         // Delete expired messages
         //
@@ -346,7 +378,8 @@ WHERE
             .sql
             .execute(
                 "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
+             SET chat_id = ?, txt = '', subject='', txt_raw='', \
+                 mime_headers='', from_id=0, to_id=0, param='' \
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
@@ -365,80 +398,121 @@ WHERE
         updated |= rows_modified > 0;
     }
 
-    schedule_ephemeral_task(context).await;
-    Ok(updated)
+    if updated {
+        context.emit_msgs_changed_without_ids();
+    }
+
+    Ok(())
 }
 
-/// Schedule a task to emit MsgsChanged event when the next local
-/// deletion happens. Existing task is cancelled to make sure at most
-/// one such task is scheduled at a time.
+/// Calculates the next timestamp when a message will be deleted due to
+/// `delete_device_after` setting being set.
+async fn next_delete_device_after_timestamp(context: &Context) -> Result<Option<i64>> {
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
+        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+            .await?
+            .unwrap_or_default();
+        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
+            .await?
+            .unwrap_or_default();
+
+        let oldest_message_timestamp: Option<i64> = context
+            .sql
+            .query_get_value(
+                r#"
+                SELECT min(timestamp)
+                FROM msgs
+                WHERE chat_id > ?
+                  AND chat_id != ?
+                  AND chat_id != ?;
+                "#,
+                paramsv![DC_CHAT_ID_TRASH, self_chat_id, device_chat_id],
+            )
+            .await?;
+
+        Ok(oldest_message_timestamp.map(|x| x.saturating_add(delete_device_after)))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Calculates next timestamp when expiration of some message will happen.
 ///
-/// UI is expected to reload the chatlist or the chat in response to
-/// MsgsChanged event, this will trigger actual deletion.
-///
-/// This takes into account only per-chat timeouts, because global device
-/// timeouts are at least one hour long and deletion is triggered often enough
-/// by user actions.
-pub async fn schedule_ephemeral_task(context: &Context) {
+/// Expiration can happen either because user has set `delete_device_after` setting or because the
+/// message itself has an ephemeral timer.
+async fn next_expiration_timestamp(context: &Context) -> Option<i64> {
     let ephemeral_timestamp: Option<i64> = match context
         .sql
         .query_get_value(
             r#"
-    SELECT ephemeral_timestamp
-    FROM msgs
-    WHERE ephemeral_timestamp != 0
-      AND chat_id != ?
-    ORDER BY ephemeral_timestamp ASC
-    LIMIT 1;
-    "#,
+            SELECT min(ephemeral_timestamp)
+            FROM msgs
+            WHERE ephemeral_timestamp != 0
+              AND chat_id != ?;
+            "#,
             paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
         )
         .await
     {
         Err(err) => {
             warn!(context, "Can't calculate next ephemeral timeout: {}", err);
-            return;
+            None
         }
         Ok(ephemeral_timestamp) => ephemeral_timestamp,
     };
 
-    // Cancel existing task, if any
-    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
-        ephemeral_task.cancel().await;
-    }
+    let delete_device_after_timestamp: Option<i64> =
+        match next_delete_device_after_timestamp(context).await {
+            Err(err) => {
+                warn!(
+                    context,
+                    "Can't calculate timestamp of the next message expiration: {}", err
+                );
+                None
+            }
+            Ok(timestamp) => timestamp,
+        };
+
+    ephemeral_timestamp
+        .into_iter()
+        .chain(delete_device_after_timestamp.into_iter())
+        .min()
+}
+
+pub(crate) async fn ephemeral_loop(context: &Context, interrupt_receiver: Receiver<()>) {
+    loop {
+        let ephemeral_timestamp = next_expiration_timestamp(context).await;
 
-    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
         let now = SystemTime::now();
-        let until = UNIX_EPOCH
-            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
-            + Duration::from_secs(1);
+        let until = if let Some(ephemeral_timestamp) = ephemeral_timestamp {
+            UNIX_EPOCH
+                + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+                + Duration::from_secs(1)
+        } else {
+            // no messages to be deleted for now, wait long for one to occur
+            now + Duration::from_secs(86400)
+        };
 
         if let Ok(duration) = until.duration_since(now) {
-            // Schedule a task, ephemeral_timestamp is in the future
-            let context1 = context.clone();
-            let ephemeral_task = task::spawn(async move {
-                async_std::task::sleep(duration).await;
-                context1.emit_event(EventType::MsgsChanged {
-                    chat_id: ChatId::new(0),
-                    msg_id: MsgId::new(0),
-                });
-            });
-            *context.ephemeral_task.write().await = Some(ephemeral_task);
-        } else {
-            // Emit event immediately
-            context.emit_event(EventType::MsgsChanged {
-                chat_id: ChatId::new(0),
-                msg_id: MsgId::new(0),
-            });
+            info!(
+                context,
+                "Ephemeral loop waiting for deletion in {} or interrupt",
+                duration_to_str(duration)
+            );
+            if timeout(duration, interrupt_receiver.recv()).await.is_ok() {
+                // received an interruption signal, recompute waiting time (if any)
+                continue;
+            }
         }
+
+        delete_expired_messages(context, time())
+            .await
+            .ok_or_log(context);
     }
 }
 
-/// Returns ID of any expired message that should be deleted from the server.
-///
-/// It looks up the trash chat too, to find messages that are already
-/// deleted locally, but not deleted on the server.
-pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
+/// Schedules expired IMAP messages for deletion.
+pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<()> {
     let now = time();
 
     let (threshold_timestamp, threshold_timestamp_extended) =
@@ -452,27 +526,20 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Resul
 
     context
         .sql
-        .query_row_optional(
-            "SELECT id FROM msgs \
-         WHERE ( \
-         ((download_state = 0 AND timestamp < ?) OR (download_state != 0 AND timestamp < ?)) \
-         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
-         ) \
-         AND server_uid != 0 \
-         AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
-         LIMIT 1",
-            paramsv![
-                threshold_timestamp,
-                threshold_timestamp_extended,
-                now,
-                job::Action::DeleteMsgOnImap
-            ],
-            |row| {
-                let msg_id: MsgId = row.get(0)?;
-                Ok(msg_id)
-            },
+        .execute(
+            "UPDATE imap
+             SET target=''
+             WHERE rfc724_mid IN (
+               SELECT rfc724_mid FROM msgs
+               WHERE ((download_state = 0 AND timestamp < ?) OR
+                      (download_state != 0 AND timestamp < ?) OR
+                      (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?))
+             )",
+            paramsv![threshold_timestamp, threshold_timestamp_extended, now],
         )
-        .await
+        .await?;
+
+    Ok(())
 }
 
 /// Start ephemeral timers for seen messages if they are not started
@@ -507,12 +574,10 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
 
 #[cfg(test)]
 mod tests {
-    use crate::param::Params;
-    use async_std::task::sleep;
-
     use super::*;
     use crate::config::Config;
     use crate::dc_receive_imf::dc_receive_imf;
+    use crate::dc_tools::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
@@ -525,7 +590,7 @@ mod tests {
         let context = TestContext::new().await;
 
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, ContactId::SELF).await,
             "Message deletion timer is disabled by me."
         );
 
@@ -533,7 +598,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 1 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 s by me."
@@ -542,7 +607,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 30 s by me."
@@ -551,7 +616,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 minute by me."
@@ -560,7 +625,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 90 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1.5 minutes by me."
@@ -569,7 +634,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 * 60 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 30 minutes by me."
@@ -578,7 +643,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 * 60 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 hour by me."
@@ -587,7 +652,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 5400 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1.5 hours by me."
@@ -598,7 +663,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 2 hours by me."
@@ -609,7 +674,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 day by me."
@@ -620,7 +685,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 2 days by me."
@@ -631,7 +696,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 1 week by me."
@@ -642,7 +707,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                ContactId::SELF
             )
             .await,
             "Message deletion timer is set to 4 weeks by me."
@@ -725,7 +790,7 @@ mod tests {
     /// Test that Alice replying to the chat without a timer at the same time as Bob enables the
     /// timer does not result in disabling the timer on the Bob's side.
     #[async_std::test]
-    async fn test_ephemeral_timer_rollback() -> anyhow::Result<()> {
+    async fn test_ephemeral_timer_rollback() -> Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
@@ -799,57 +864,108 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_ephemeral_delete_msgs() {
+    async fn test_ephemeral_delete_msgs() -> Result<()> {
         let t = TestContext::new_alice().await;
-        let chat = t.get_self_chat().await;
+        let self_chat = t.get_self_chat().await;
 
-        t.send_text(chat.id, "Saved message, which we delete manually")
+        assert_eq!(next_expiration_timestamp(&t).await, None);
+
+        t.send_text(self_chat.id, "Saved message, which we delete manually")
             .await;
-        let msg = t.get_last_msg_in(chat.id).await;
-        msg.id.delete_from_db(&t).await.unwrap();
-        check_msg_was_deleted(&t, &chat, msg.id).await;
+        let msg = t.get_last_msg_in(self_chat.id).await;
+        msg.id.delete_from_db(&t).await?;
+        check_msg_is_deleted(&t, &self_chat, msg.id).await;
 
-        chat.id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
+        self_chat
+            .id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 3600 })
             .await
             .unwrap();
-        let msg = t
-            .send_text(chat.id, "Saved message, disappearing after 1s")
-            .await;
 
-        sleep(Duration::from_millis(1100)).await;
+        // Send a saved message which will be deleted after 3600s
+        let now = time();
+        let msg = t.send_text(self_chat.id, "Message text").await;
 
-        // Check checks that the msg was deleted locally
-        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
-
-        // Check that the msg will be deleted on the server
-        // First of all, set a server_uid so that DC thinks that it's actually possible to delete
-        t.sql
-            .execute(
-                "UPDATE msgs SET server_uid=1 WHERE id=?",
-                paramsv![msg.sender_msg_id],
-            )
+        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3599, time() + 3601)
             .await
             .unwrap();
-        let job = job::load_imap_deletion_job(&t).await.unwrap();
-        assert_eq!(
-            job,
-            Some(job::Job::new(
-                job::Action::DeleteMsgOnImap,
-                msg.sender_msg_id.to_u32(),
-                Params::new(),
-                0,
-            ))
-        );
-        // Let's assume that executing the job fails on first try and the job is saved to the db
-        job.unwrap().save(&t).await.unwrap();
 
-        // Make sure that we don't get yet another job when loading from db
-        let job2 = job::load_imap_deletion_job(&t).await.unwrap();
-        assert_eq!(job2, None);
+        // Set DeleteDeviceAfter to 1800s. Thend send a saved message which will
+        // still be deleted after 3600s because DeleteDeviceAfter doesn't apply to saved messages.
+        t.set_config(Config::DeleteDeviceAfter, Some("1800"))
+            .await?;
+
+        let now = time();
+        let msg = t.send_text(self_chat.id, "Message text").await;
+
+        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3559, time() + 3601)
+            .await
+            .unwrap();
+
+        // Send a message to Bob which will be deleted after 1800s because of DeleteDeviceAfter.
+        let bob_chat = t.create_chat_with_contact("", "bob@example.net").await;
+        let now = time();
+        let msg = t.send_text(bob_chat.id, "Message text").await;
+
+        check_msg_will_be_deleted(
+            &t,
+            msg.sender_msg_id,
+            &bob_chat,
+            now + 1799,
+            // The message may appear to be sent MAX_SECONDS_TO_LEND_FROM_FUTURE later and
+            // therefore be deleted MAX_SECONDS_TO_LEND_FROM_FUTURE later.
+            time() + 1801 + MAX_SECONDS_TO_LEND_FROM_FUTURE,
+        )
+        .await
+        .unwrap();
+
+        // Enable ephemeral messages with Bob -> message will be deleted after 60s.
+        // This tests that the message is deleted at min(ephemeral deletion time, DeleteDeviceAfter deletion time).
+        bob_chat
+            .id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
+            .await?;
+
+        let now = time();
+        let msg = t.send_text(bob_chat.id, "Message text").await;
+
+        check_msg_will_be_deleted(&t, msg.sender_msg_id, &bob_chat, now + 59, time() + 61)
+            .await
+            .unwrap();
+
+        Ok(())
     }
 
-    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+    async fn check_msg_will_be_deleted(
+        t: &TestContext,
+        msg_id: MsgId,
+        chat: &Chat,
+        not_deleted_at: i64,
+        deleted_at: i64,
+    ) -> Result<()> {
+        let next_expiration = next_expiration_timestamp(t).await.unwrap();
+
+        assert!(next_expiration > not_deleted_at);
+        delete_expired_messages(t, not_deleted_at).await?;
+
+        let loaded = Message::load_from_db(t, msg_id).await?;
+        assert_eq!(loaded.text.unwrap(), "Message text");
+        assert_eq!(loaded.chat_id, chat.id);
+
+        assert!(next_expiration < deleted_at);
+        delete_expired_messages(t, deleted_at).await?;
+
+        let loaded = Message::load_from_db(t, msg_id).await?;
+        assert_eq!(loaded.text.unwrap(), "");
+        assert_eq!(loaded.chat_id, DC_CHAT_ID_TRASH);
+
+        // Check that the msg was deleted locally.
+        check_msg_is_deleted(t, chat, msg_id).await;
+
+        Ok(())
+    }
+
+    async fn check_msg_is_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
         let chat_items = chat::get_chat_msgs(t, chat.id, 0, None).await.unwrap();
         // Check that the chat is empty except for possibly info messages:
         for item in &chat_items {
@@ -861,8 +977,8 @@ mod tests {
 
         // Check that if there is a message left, the text and metadata are gone
         if let Ok(msg) = Message::load_from_db(t, msg_id).await {
-            assert_eq!(msg.from_id, 0);
-            assert_eq!(msg.to_id, 0);
+            assert_eq!(msg.from_id, ContactId::UNDEFINED);
+            assert_eq!(msg.to_id, ContactId::UNDEFINED);
             assert!(msg.text.is_none_or_empty(), "{:?}", msg.text);
             let rawtxt: Option<String> = t
                 .sql
@@ -874,7 +990,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_load_imap_deletion_msgid() -> Result<()> {
+    async fn test_delete_expired_imap_messages() -> Result<()> {
         let t = TestContext::new_alice().await;
         const HOUR: i64 = 60 * 60;
         let now = time();
@@ -887,42 +1003,98 @@ mod tests {
             (2000, now - 18 * HOUR, now - HOUR),
             (2020, now - 17 * HOUR, now + HOUR),
         ] {
+            let message_id = id.to_string();
             t.sql
-                .execute(
-                    "INSERT INTO msgs (id, server_uid, timestamp, ephemeral_timestamp) VALUES (?,?,?,?);",
-                    paramsv![id, id, timestamp, ephemeral_timestamp],
-                )
-                .await?;
+                   .execute(
+                       "INSERT INTO msgs (id, rfc724_mid, timestamp, ephemeral_timestamp) VALUES (?,?,?,?);",
+                       paramsv![id, message_id, timestamp, ephemeral_timestamp],
+                   )
+                   .await?;
+            t.sql
+                   .execute(
+                       "INSERT INTO imap (rfc724_mid, folder, uid, target) VALUES (?,'INBOX',?, 'INBOX');",
+                       paramsv![message_id, id],
+                   )
+                   .await?;
         }
 
-        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(2000)));
+        async fn test_marked_for_deletion(context: &Context, id: u32) -> Result<()> {
+            assert_eq!(
+                context
+                    .sql
+                    .count(
+                        "SELECT COUNT(*) FROM imap WHERE target='' AND rfc724_mid=?",
+                        paramsv![id.to_string()],
+                    )
+                    .await?,
+                1
+            );
+            Ok(())
+        }
 
-        MsgId::new(2000).delete_from_db(&t).await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
+        async fn remove_uid(context: &Context, id: u32) -> Result<()> {
+            context
+                .sql
+                .execute(
+                    "DELETE FROM imap WHERE rfc724_mid=?",
+                    paramsv![id.to_string()],
+                )
+                .await?;
+            Ok(())
+        }
+
+        // This should mark message 2000 for deletion.
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 2000).await?;
+        remove_uid(&t, 2000).await?;
+        // No other messages are marked for deletion.
+        assert_eq!(
+            t.sql
+                .count("SELECT COUNT(*) FROM imap WHERE target=''", paramsv![],)
+                .await?,
+            0
+        );
 
         t.set_config(Config::DeleteServerAfter, Some(&*(25 * HOUR).to_string()))
             .await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1000)));
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 1000).await?;
 
         MsgId::new(1000)
             .update_download_state(&t, DownloadState::Available)
             .await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1000))); // delete downloadable anyway
-
-        MsgId::new(1000).delete_from_db(&t).await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
+        t.sql
+            .execute(
+                "UPDATE imap SET target=folder WHERE rfc724_mid='1000'",
+                paramsv![],
+            )
+            .await?;
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 1000).await?; // Delete downloadable anyway.
+        remove_uid(&t, 1000).await?;
 
         t.set_config(Config::DeleteServerAfter, Some(&*(22 * HOUR).to_string()))
             .await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1010)));
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 1010).await?;
+        t.sql
+            .execute(
+                "UPDATE imap SET target=folder WHERE rfc724_mid='1010'",
+                paramsv![],
+            )
+            .await?;
 
         MsgId::new(1010)
             .update_download_state(&t, DownloadState::Available)
             .await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, None); // keep downloadable for now
-
-        MsgId::new(1010).delete_from_db(&t).await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
+        delete_expired_imap_messages(&t).await?;
+        // Keep downloadable for now.
+        assert_eq!(
+            t.sql
+                .count("SELECT COUNT(*) FROM imap WHERE target=''", paramsv![],)
+                .await?,
+            0
+        );
 
         Ok(())
     }
@@ -936,15 +1108,13 @@ mod tests {
         dc_receive_imf(
             &alice,
             b"From: Bob <bob@example.com>\n\
-                    To: Alice <alice@example.com>\n\
+                    To: Alice <alice@example.org>\n\
                     Chat-Version: 1.0\n\
                     Subject: Subject\n\
                     Message-ID: <first@example.com>\n\
                     Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
                     \n\
                     hello\n",
-            "INBOX",
-            1,
             false,
         )
         .await?;
@@ -957,7 +1127,7 @@ mod tests {
         dc_receive_imf(
             &alice,
             b"From: Bob <bob@example.com>\n\
-                    To: Alice <alice@example.com>\n\
+                    To: Alice <alice@example.org>\n\
                     Chat-Version: 1.0\n\
                     Subject: Subject\n\
                     Message-ID: <second@example.com>\n\
@@ -965,8 +1135,6 @@ mod tests {
                     Ephemeral-Timer: 60\n\
                     \n\
                     second message\n",
-            "INBOX",
-            2,
             false,
         )
         .await?;
@@ -994,7 +1162,7 @@ mod tests {
         dc_receive_imf(
             &alice,
             b"From: Bob <bob@example.com>\n\
-                    To: Alice <alice@example.com>\n\
+                    To: Alice <alice@example.org>\n\
                     Chat-Version: 1.0\n\
                     Subject: Subject\n\
                     Message-ID: <third@example.com>\n\
@@ -1003,8 +1171,6 @@ mod tests {
                     In-Reply-To: <first@example.com>\n\
                     \n\
                     > hello\n",
-            "INBOX",
-            3,
             false,
         )
         .await?;

src/events.rs

@@ -7,8 +7,10 @@ use async_std::path::PathBuf;
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
+use crate::contact::ContactId;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
+use crate::webxdc::StatusUpdateSerial;
 
 #[derive(Debug)]
 pub struct Events {
@@ -252,7 +254,7 @@ pub enum EventType {
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
     #[strum(props(id = "2030"))]
-    ContactsChanged(Option<u32>),
+    ContactsChanged(Option<ContactId>),
 
     /// Location of one or more contact has changed.
     ///
@@ -260,7 +262,7 @@ pub enum EventType {
     ///     If the locations of several contacts have been changed,
     ///     eg. after calling dc_delete_all_locations(), this parameter is set to `None`.
     #[strum(props(id = "2035"))]
-    LocationChanged(Option<u32>),
+    LocationChanged(Option<ContactId>),
 
     /// Inform about the configuration progress started by configure().
     #[strum(props(id = "2041"))]
@@ -304,7 +306,10 @@ pub enum EventType {
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
     #[strum(props(id = "2060"))]
-    SecurejoinInviterProgress { contact_id: u32, progress: usize },
+    SecurejoinInviterProgress {
+        contact_id: ContactId,
+        progress: usize,
+    },
 
     /// Progress information of a secure-join handshake from the view of the joiner
     /// (Bob, the person who scans the QR code).
@@ -315,7 +320,10 @@ pub enum EventType {
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[strum(props(id = "2061"))]
-    SecurejoinJoinerProgress { contact_id: u32, progress: usize },
+    SecurejoinJoinerProgress {
+        contact_id: ContactId,
+        progress: usize,
+    },
 
     /// The connectivity to the server changed.
     /// This means that you should refresh the connectivity view
@@ -326,4 +334,10 @@ pub enum EventType {
 
     #[strum(props(id = "2110"))]
     SelfavatarChanged,
+
+    #[strum(props(id = "2120"))]
+    WebxdcStatusUpdate {
+        msg_id: MsgId,
+        status_update_serial: StatusUpdateSerial,
+    },
 }

src/headerdef.rs

@@ -33,6 +33,7 @@ pub enum HeaderDef {
     XMozillaDraftInfo,
 
     ListId,
+    ListPost,
     References,
     InReplyTo,
     Precedence,

src/html.rs

@@ -279,9 +279,9 @@ mod tests {
     use crate::chat;
     use crate::chat::forward_msgs;
     use crate::config::Config;
-    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+    use crate::contact::ContactId;
     use crate::dc_receive_imf::dc_receive_imf;
-    use crate::message::MessengerMessage;
+    use crate::message::{MessengerMessage, Viewtype};
     use crate::test_utils::TestContext;
 
     #[async_std::test]
@@ -365,7 +365,7 @@ test some special html-characters as < > and & but also " and &#x
         // however, rust multiline-strings use just `\n`;
         // therefore, we just remove `\r` before comparison.
         assert_eq!(
-            parser.html.replace("\r", ""),
+            parser.html.replace('\r', ""),
             r##"
 <html>
   <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
@@ -379,7 +379,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         let raw = include_bytes!("../test-data/message/text_alt_html.eml");
         let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
-            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
             r##"<html>
   <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
 </html>
@@ -394,7 +394,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
         let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
-            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
             r##"<html>
   <p>
     this is <b>html</b>
@@ -440,11 +440,9 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", 1, false)
-            .await
-            .unwrap();
+        dc_receive_imf(&alice, raw, false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_ne!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
         assert!(!msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -458,7 +456,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .await
             .unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -468,10 +466,10 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
 
         // bob: check that bob also got the html-part of the forwarded message
         let bob = TestContext::new_bob().await;
-        let chat = bob.create_chat_with_contact("", "alice@example.com").await;
+        let chat = bob.create_chat_with_contact("", "alice@example.org").await;
         bob.recv_msg(&alice.pop_sent_msg().await).await;
         let msg = bob.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_ne!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -491,9 +489,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, "INBOX", 1, false)
-            .await
-            .unwrap();
+        dc_receive_imf(&alice, raw, false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
 
         // forward the message to saved-messages,
@@ -510,7 +506,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         alice.recv_msg(&msg).await;
         let chat = alice.get_self_chat().await;
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.get_from_id(), ContactId::SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.get_showpadlock());
         assert!(msg.is_forwarded());
@@ -559,8 +555,6 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/cp1252-html.eml"),
-            "INBOX",
-            0,
             false,
         )
         .await?;

src/imap/idle.rs

@@ -1,6 +1,6 @@
 use super::Imap;
 
-use anyhow::{bail, format_err, Result};
+use anyhow::{bail, Context as _, Result};
 use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
@@ -31,7 +31,7 @@ impl Imap {
         let timeout = Duration::from_secs(23 * 60);
         let mut info = Default::default();
 
-        if self.server_sent_unsolicited_exists(context) {
+        if self.server_sent_unsolicited_exists(context)? {
             return Ok(info);
         }
 
@@ -90,8 +90,8 @@ impl Imap {
             let session = handle
                 .done()
                 .timeout(Duration::from_secs(15))
-                .await
-                .map_err(|err| format_err!("IMAP IDLE protocol timed out: {}", err))??;
+                .await?
+                .context("IMAP IDLE protocol timed out")?;
             self.session = Some(Session { inner: session });
         } else {
             warn!(context, "Attempted to idle without a session");
@@ -150,19 +150,21 @@ impl Imap {
                     }
                     if self.config.can_idle {
                         // we only fake-idled because network was gone during IDLE, probably
-                        break InterruptInfo::new(false, None);
+                        break InterruptInfo::new(false);
                     }
                     info!(context, "fake_idle is connected");
                     // we are connected, let's see if fetching messages results
                     // in anything.  If so, we behave as if IDLE had data but
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
-
-                    match self.fetch_new_messages(context, &watch_folder, false).await {
+                    match self
+                        .fetch_new_messages(context, &watch_folder, false, false)
+                        .await
+                    {
                         Ok(res) => {
                             info!(context, "fetch_new_messages returned {:?}", res);
                             if res {
-                                break InterruptInfo::new(false, None);
+                                break InterruptInfo::new(false);
                             }
                         }
                         Err(err) => {

src/imap/scan_folders.rs

@@ -2,15 +2,18 @@ use std::{collections::BTreeMap, time::Instant};
 
 use anyhow::{Context as _, Result};
 
+use crate::config::Config;
 use crate::imap::Imap;
-use crate::{config::Config, log::LogExt};
+use crate::log::LogExt;
 use crate::{context::Context, imap::FolderMeaning};
-use async_std::prelude::*;
+
+use async_std::stream::StreamExt;
 
 use super::{get_folder_meaning, get_folder_meaning_by_name};
 
 impl Imap {
-    pub async fn scan_folders(&mut self, context: &Context) -> Result<()> {
+    /// Returns true if folders were scanned, false if scanning was postponed.
+    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<bool> {
         // First of all, debounce to once per minute:
         let mut last_scan = context.last_full_folder_scan.lock().await;
         if let Some(last_scan) = *last_scan {
@@ -20,28 +23,18 @@ impl Imap {
                 .await?;
 
             if elapsed_secs < debounce_secs {
-                return Ok(());
+                return Ok(false);
             }
         }
         info!(context, "Starting full folder scan");
 
         self.prepare(context).await?;
-        let session = self.session.as_mut();
-        let session = session.context("scan_folders(): IMAP No Connection established")?;
-        let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
-        let watched_folders = get_watched_folders(context).await;
+        let folders = self.list_folders(context).await?;
+        let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
 
         for folder in folders {
-            let folder = match folder {
-                Ok(f) => f,
-                Err(e) => {
-                    warn!(context, "Can't get folder: {}", e);
-                    continue;
-                }
-            };
-
             // Gmail labels are not folders and should be skipped. For example,
             // emails appear in the inbox and under "All Mail" as soon as it is
             // received. The code used to wrongly conclude that the email had
@@ -67,54 +60,74 @@ impl Imap {
             let is_drafts = folder_meaning == FolderMeaning::Drafts
                 || (folder_meaning == FolderMeaning::Unknown
                     && folder_name_meaning == FolderMeaning::Drafts);
+            let is_spam_folder = folder_meaning == FolderMeaning::Spam
+                || (folder_meaning == FolderMeaning::Unknown
+                    && folder_name_meaning == FolderMeaning::Spam);
 
             // Don't scan folders that are watched anyway
             if !watched_folders.contains(&folder.name().to_string()) && !is_drafts {
                 // Drain leftover unsolicited EXISTS messages
-                self.server_sent_unsolicited_exists(context);
+                self.server_sent_unsolicited_exists(context)?;
 
                 loop {
-                    self.fetch_new_messages(context, folder.name(), false)
+                    self.fetch_move_delete(context, folder.name(), is_spam_folder)
                         .await
                         .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
 
                     // If the server sent an unsocicited EXISTS during the fetch, we need to fetch again
-                    if !self.server_sent_unsolicited_exists(context) {
+                    if !self.server_sent_unsolicited_exists(context)? {
                         break;
                     }
                 }
             }
         }
 
-        // We iterate over both folder meanings to make sure that if e.g. the "Sent" folder was deleted,
-        // `ConfiguredSentboxFolder` is set to `None`:
-        for config in &[
-            Config::ConfiguredSentboxFolder,
-            Config::ConfiguredSpamFolder,
-        ] {
-            context
-                .set_config(*config, folder_configs.get(config).map(|s| s.as_str()))
-                .await?;
-        }
+        // Set the `ConfiguredSentboxFolder` or set it to `None` if the folder was deleted.
+        context
+            .set_config(
+                Config::ConfiguredSentboxFolder,
+                folder_configs
+                    .get(&Config::ConfiguredSentboxFolder)
+                    .map(|s| s.as_str()),
+            )
+            .await?;
 
         last_scan.replace(Instant::now());
-        Ok(())
+        Ok(true)
+    }
+
+    /// Returns the names of all folders on the IMAP server.
+    pub async fn list_folders(
+        self: &mut Imap,
+        context: &Context,
+    ) -> Result<Vec<async_imap::types::Name>> {
+        let session = self.session.as_mut();
+        let session = session.context("No IMAP connection")?;
+        let list = session
+            .list(Some(""), Some("*"))
+            .await?
+            .filter_map(|f| f.ok_or_log_msg(context, "list_folders() can't get folder"));
+        Ok(list.collect().await)
     }
 }
 
-pub(crate) async fn get_watched_folders(context: &Context) -> Vec<String> {
+pub(crate) async fn get_watched_folder_configs(context: &Context) -> Result<Vec<Config>> {
+    let mut res = vec![Config::ConfiguredInboxFolder];
+    if context.get_config_bool(Config::SentboxWatch).await? {
+        res.push(Config::ConfiguredSentboxFolder);
+    }
+    if context.should_watch_mvbox().await? {
+        res.push(Config::ConfiguredMvboxFolder);
+    }
+    Ok(res)
+}
+
+pub(crate) async fn get_watched_folders(context: &Context) -> Result<Vec<String>> {
     let mut res = Vec::new();
-    let folder_watched_configured = &[
-        (Config::SentboxWatch, Config::ConfiguredSentboxFolder),
-        (Config::MvboxWatch, Config::ConfiguredMvboxFolder),
-        (Config::InboxWatch, Config::ConfiguredInboxFolder),
-    ];
-    for (watched, configured) in folder_watched_configured {
-        if context.get_config_bool(*watched).await.unwrap_or_default() {
-            if let Ok(Some(folder)) = context.get_config(*configured).await {
-                res.push(folder);
-            }
+    for folder_config in get_watched_folder_configs(context).await? {
+        if let Some(folder) = context.get_config(folder_config).await? {
+            res.push(folder);
         }
     }
-    res
+    Ok(res)
 }

src/imap/select_folder.rs

@@ -93,7 +93,11 @@ impl Imap {
         // select new folder
         if let Some(folder) = folder {
             if let Some(ref mut session) = &mut self.session {
-                let res = session.select(folder).await;
+                let res = if self.config.can_condstore {
+                    session.select_condstore(folder).await
+                } else {
+                    session.select(folder).await
+                };
 
                 // <https://tools.ietf.org/html/rfc3501#section-6.3.1>
                 // says that if the server reports select failure we are in

src/imex.rs

@@ -16,21 +16,21 @@ use rand::{thread_rng, Rng};
 use crate::blob::BlobObject;
 use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
 use crate::config::Config;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+use crate::contact::ContactId;
 use crate::context::Context;
 use crate::dc_tools::{
-    dc_copy_file, dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
-    dc_open_file_std, dc_read_file, dc_write_file, get_next_backup_path, time, EmailAddress,
+    dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
+    dc_open_file_std, dc_read_file, dc_write_file, time, EmailAddress,
 };
 use crate::e2ee;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::log::LogExt;
-use crate::message::{Message, MsgId};
+use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::pgp;
-use crate::sql::{self, Sql};
+use crate::sql;
 use crate::stock_str;
 
 // Name of the database file in the backup.
@@ -41,24 +41,24 @@ const BLOBS_BACKUP_NAME: &str = "blobs_backup";
 #[repr(u32)]
 pub enum ImexMode {
     /// Export all private keys and all public keys of the user to the
-    /// directory given as `param1`.  The default key is written to the files `public-key-default.asc`
+    /// directory given as `path`.  The default key is written to the files `public-key-default.asc`
     /// and `private-key-default.asc`, if there are more keys, they are written to files as
     /// `public-key-<id>.asc` and `private-key-<id>.asc`
     ExportSelfKeys = 1,
 
-    /// Import private keys found in the directory given as `param1`.
+    /// Import private keys found in the directory given as `path`.
     /// The last imported key is made the default keys unless its name contains the string `legacy`.
     /// Public keys are not imported.
     ImportSelfKeys = 2,
 
-    /// Export a backup to the directory given as `param1`.
+    /// Export a backup to the directory given as `path` with the given `passphrase`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
     /// The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
     /// the format is `delta-chat-<day>-<number>.tar`
     ExportBackup = 11,
 
-    /// `param1` is the file (not: directory) to import. The file is normally
+    /// `path` is the file (not: directory) to import. The file is normally
     /// created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup
     /// is only possible as long as the context is not configured or used in another way.
     ImportBackup = 12,
@@ -78,11 +78,16 @@ pub enum ImexMode {
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
-pub async fn imex(context: &Context, what: ImexMode, param1: &Path) -> Result<()> {
+pub async fn imex(
+    context: &Context,
+    what: ImexMode,
+    path: &Path,
+    passphrase: Option<String>,
+) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
 
     let res = async {
-        let success = imex_inner(context, what, param1).await;
+        let success = imex_inner(context, what, path, passphrase).await;
         match success {
             Ok(()) => {
                 info!(context, "IMEX successfully completed");
@@ -115,15 +120,10 @@ async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
         dc_delete_file(context, context.get_dbfile()).await;
         dc_delete_files_in_dir(context, context.get_blobdir()).await;
     }
-    if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
-        if let Err(e) = context.sql.open(context, context.get_dbfile(), false).await {
-            warn!(context, "Re-opening db after imex failed: {}", e);
-        }
-    }
 }
 
 /// Returns the filename of the backup found (otherwise an error)
-pub async fn has_backup(context: &Context, dir_name: &Path) -> Result<String> {
+pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result<String> {
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_name = "".to_string();
     let mut newest_backup_path: Option<PathBuf> = None;
@@ -145,59 +145,6 @@ pub async fn has_backup(context: &Context, dir_name: &Path) -> Result<String> {
         }
     }
 
-    match newest_backup_path {
-        Some(path) => Ok(path.to_string_lossy().into_owned()),
-        None => has_backup_old(context, dir_name).await,
-        // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`.
-    }
-}
-
-/// Returns the filename of the backup found (otherwise an error)
-pub async fn has_backup_old(context: &Context, dir_name: &Path) -> Result<String> {
-    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
-    let mut newest_backup_time = 0;
-    let mut newest_backup_name = "".to_string();
-    let mut newest_backup_path: Option<PathBuf> = None;
-    while let Some(dirent) = dir_iter.next().await {
-        if let Ok(dirent) = dirent {
-            let path = dirent.path();
-            let name = dirent.file_name();
-            let name = name.to_string_lossy();
-            if name.starts_with("delta-chat") && name.ends_with(".bak") {
-                let sql = Sql::new();
-                match sql.open(context, &path, true).await {
-                    Ok(_) => {
-                        let curr_backup_time = sql
-                            .get_raw_config_int("backup_time")
-                            .await?
-                            .unwrap_or_default();
-                        if curr_backup_time > newest_backup_time {
-                            newest_backup_path = Some(path);
-                            newest_backup_time = curr_backup_time;
-                        }
-                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
-                        sql.close().await;
-                    }
-                    Err(e) => {
-                        warn!(
-                            context,
-                            "Found backup file {} which could not be opened: {}", name, e
-                        );
-                        // On some Android devices we can't open sql files that are not in our private directory
-                        // (see <https://github.com/deltachat/deltachat-android/issues/1768>). So, compare names
-                        // to still find the newest backup.
-                        let name: String = name.into();
-                        if newest_backup_time == 0
-                            && (newest_backup_name.is_empty() || name > newest_backup_name)
-                        {
-                            newest_backup_path = Some(path);
-                            newest_backup_name = name;
-                        }
-                    }
-                }
-            }
-        }
-    }
     match newest_backup_path {
         Some(path) => Ok(path.to_string_lossy().into_owned()),
         None => bail!("no backup found in {}", dir_name.display()),
@@ -218,7 +165,6 @@ pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
 }
 
 async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
-    let mut msg: Message;
     let setup_code = create_setup_code(context);
     /* this may require a keypair to be created. this may take a second ... */
     let setup_file_content = render_setup_file(context, &setup_code).await?;
@@ -230,15 +176,17 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     )
     .await?;
 
-    let chat_id = ChatId::create_for_contact(context, DC_CONTACT_ID_SELF).await?;
-    msg = Message::default();
-    msg.viewtype = Viewtype::File;
+    let chat_id = ChatId::create_for_contact(context, ContactId::SELF).await?;
+    let mut msg = Message {
+        viewtype: Viewtype::File,
+        ..Default::default()
+    };
     msg.param.set(Param::File, setup_file_blob.as_name());
     msg.subject = stock_str::ac_setup_msg_subject(context).await;
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.param.set_int(Param::ForcePlaintext, 1);
+    msg.force_plaintext();
     msg.param.set_int(Param::SkipAutocrypt, 1);
 
     let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
@@ -289,7 +237,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
 
     let msg_subj = stock_str::ac_setup_msg_subject(context).await;
     let msg_body = stock_str::ac_setup_msg_body(context).await;
-    let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
+    let msg_body_html = msg_body.replace('\r', "").replace('\n', "<br>");
     Ok(format!(
         concat!(
             "<!DOCTYPE html>\r\n",
@@ -403,9 +351,8 @@ async fn set_self_key(
         }
     };
 
-    let self_addr = context.get_config(Config::ConfiguredAddr).await?;
-    ensure!(self_addr.is_some(), "Missing self addr");
-    let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
+    let self_addr = context.get_primary_self_addr().await?;
+    let addr = EmailAddress::new(&self_addr)?;
     let keypair = pgp::KeyPair {
         addr,
         public: public_key,
@@ -449,7 +396,12 @@ fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-async fn imex_inner(context: &Context, what: ImexMode, path: &Path) -> Result<()> {
+async fn imex_inner(
+    context: &Context,
+    what: ImexMode,
+    path: &Path,
+    passphrase: Option<String>,
+) -> Result<()> {
     info!(context, "Import/export dir: {}", path.display());
     ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
@@ -467,26 +419,27 @@ async fn imex_inner(context: &Context, what: ImexMode, path: &Path) -> Result<()
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
 
-        ImexMode::ExportBackup => export_backup(context, path).await,
-        // import_backup() will call import_backup_old() if this is an old backup.
-        ImexMode::ImportBackup => import_backup(context, path).await,
+        ImexMode::ExportBackup => {
+            export_backup(context, path, passphrase.unwrap_or_default()).await
+        }
+        ImexMode::ImportBackup => {
+            import_backup(context, path, passphrase.unwrap_or_default()).await?;
+            context.sql.run_migrations(context).await
+        }
     }
 }
 
-/// Import Backup
-async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()> {
-    if backup_to_import.to_string_lossy().ends_with(".bak") {
-        // Backwards compability
-        return import_backup_old(context, backup_to_import).await;
-    }
-
-    info!(
-        context,
-        "Import \"{}\" to \"{}\".",
-        backup_to_import.display(),
-        context.get_dbfile().display()
-    );
-
+/// Imports backup into the currently open database.
+///
+/// The contents of the currently open database will be lost.
+///
+/// `passphrase` is the passphrase used to open backup database. If backup is unencrypted, pass
+/// empty string here.
+async fn import_backup(
+    context: &Context,
+    backup_to_import: &Path,
+    passphrase: String,
+) -> Result<()> {
     ensure!(
         !context.is_configured().await?,
         "Cannot import backups to accounts in use."
@@ -495,15 +448,19 @@ async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()>
         !context.scheduler.read().await.is_running(),
         "cannot import backup, IO already running"
     );
-    context.sql.close().await;
-    dc_delete_file(context, context.get_dbfile()).await;
-    ensure!(
-        !context.get_dbfile().exists().await,
-        "Cannot delete old database."
-    );
 
     let backup_file = File::open(backup_to_import).await?;
     let file_size = backup_file.metadata().await?.len();
+    info!(
+        context,
+        "Import \"{}\" ({} bytes) to \"{}\".",
+        backup_to_import.display(),
+        file_size,
+        context.get_dbfile().display()
+    );
+
+    context.sql.config_cache.write().await.clear();
+
     let archive = Archive::new(backup_file);
 
     let mut entries = archive.entries()?;
@@ -522,11 +479,15 @@ async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()>
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
             // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
             f.unpack_in(context.get_blobdir()).await?;
-            fs::rename(
-                context.get_blobdir().join(DBFILE_BACKUP_NAME),
-                context.get_dbfile(),
-            )
-            .await?;
+            let unpacked_database = context.get_blobdir().join(DBFILE_BACKUP_NAME);
+            context
+                .sql
+                .import(&unpacked_database, passphrase.clone())
+                .await
+                .context("cannot import unpacked database")?;
+            fs::remove_file(unpacked_database)
+                .await
+                .context("cannot remove unpacked database")?;
         } else {
             // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
             f.unpack_in(context.get_blobdir()).await?;
@@ -541,136 +502,52 @@ async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()>
         }
     }
 
-    context
-        .sql
-        .open(context, context.get_dbfile(), false)
-        .await
-        .context("Could not re-open db")?;
-
     delete_and_reset_all_device_msgs(context).await?;
 
     Ok(())
 }
 
-async fn import_backup_old(context: &Context, backup_to_import: &Path) -> Result<()> {
-    info!(
-        context,
-        "Import \"{}\" to \"{}\".",
-        backup_to_import.display(),
-        context.get_dbfile().display()
-    );
-
-    ensure!(
-        !context.is_configured().await?,
-        "Cannot import backups to accounts in use."
-    );
-    ensure!(
-        !context.scheduler.read().await.is_running(),
-        "cannot import backup, IO already running"
-    );
-    context.sql.close().await;
-    dc_delete_file(context, context.get_dbfile()).await;
-    ensure!(
-        !context.get_dbfile().exists().await,
-        "Cannot delete old database."
-    );
-
-    ensure!(
-        dc_copy_file(context, backup_to_import, context.get_dbfile()).await,
-        "could not copy file"
-    );
-    /* error already logged */
-    /* re-open copied database file */
-    context
-        .sql
-        .open(context, context.get_dbfile(), false)
-        .await
-        .context("Could not re-open db")?;
-
-    delete_and_reset_all_device_msgs(context).await?;
-
-    let total_files_cnt = context
-        .sql
-        .count("SELECT COUNT(*) FROM backup_blobs;", paramsv![])
-        .await?;
-    info!(
-        context,
-        "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
-    );
-
-    // Load IDs only for now, without the file contents, to avoid
-    // consuming too much memory.
-    let file_ids = context
-        .sql
-        .query_map(
-            "SELECT id FROM backup_blobs ORDER BY id",
-            paramsv![],
-            |row| row.get(0),
-            |ids| {
-                ids.collect::<std::result::Result<Vec<i64>, _>>()
-                    .map_err(Into::into)
-            },
-        )
-        .await?;
-
-    let mut all_files_extracted = true;
-    for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
-        // Load a single blob into memory
-        let (file_name, file_blob) = context
-            .sql
-            .query_row(
-                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
-                paramsv![file_id],
-                |row| {
-                    let file_name: String = row.get(0)?;
-                    let file_blob: Vec<u8> = row.get(1)?;
-                    Ok((file_name, file_blob))
-                },
-            )
-            .await?;
-
-        if context.shall_stop_ongoing().await {
-            all_files_extracted = false;
-            break;
-        }
-        let mut permille = processed_files_cnt * 1000 / total_files_cnt;
-        if permille < 10 {
-            permille = 10
-        }
-        if permille > 990 {
-            permille = 990
-        }
-        context.emit_event(EventType::ImexProgress(permille));
-        if file_blob.is_empty() {
-            continue;
-        }
-
-        let path_filename = context.get_blobdir().join(file_name);
-        dc_write_file(context, &path_filename, &file_blob).await?;
-    }
-
-    if all_files_extracted {
-        // only delete backup_blobs if all files were successfully extracted
-        context
-            .sql
-            .execute("DROP TABLE backup_blobs;", paramsv![])
-            .await?;
-        context.sql.execute("VACUUM;", paramsv![]).await.ok();
-        Ok(())
-    } else {
-        bail!("received stop signal");
-    }
-}
-
 /*******************************************************************************
  * Export backup
  ******************************************************************************/
-#[allow(unused)]
-async fn export_backup(context: &Context, dir: &Path) -> Result<()> {
+
+/// Returns Ok((temp_db_path, temp_path, dest_path)) on success. Unencrypted database can be
+/// written to temp_db_path. The backup can then be written to temp_path. If the backup succeeded,
+/// it can be renamed to dest_path. This guarantees that the backup is complete.
+async fn get_next_backup_path(
+    folder: &Path,
+    backup_time: i64,
+) -> Result<(PathBuf, PathBuf, PathBuf)> {
+    let folder = PathBuf::from(folder);
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
+        .format("delta-chat-backup-%Y-%m-%d")
+        .to_string();
+
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut tempdbfile = folder.clone();
+        tempdbfile.push(format!("{}-{:02}.db", stem, i));
+
+        let mut tempfile = folder.clone();
+        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
+
+        let mut destfile = folder.clone();
+        destfile.push(format!("{}-{:02}.tar", stem, i));
+
+        if !tempdbfile.exists().await && !tempfile.exists().await && !destfile.exists().await {
+            return Ok((tempdbfile, tempfile, destfile));
+        }
+    }
+    bail!("could not create backup file, disk full?");
+}
+
+async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
-    let (temp_path, dest_path) = get_next_backup_path(dir, now).await?;
-    let _d = DeleteOnDrop(temp_path.clone());
+    let (temp_db_path, temp_path, dest_path) = get_next_backup_path(dir, now).await?;
+    let _d1 = DeleteOnDrop(temp_db_path.clone());
+    let _d2 = DeleteOnDrop(temp_path.clone());
 
     context
         .sql
@@ -682,16 +559,14 @@ async fn export_backup(context: &Context, dir: &Path) -> Result<()> {
         .sql
         .execute("VACUUM;", paramsv![])
         .await
-        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
+        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e))
+        .ok();
 
     ensure!(
         !context.scheduler.read().await.is_running(),
         "cannot export backup, IO already running"
     );
 
-    // we close the database during the export
-    context.sql.close().await;
-
     info!(
         context,
         "Backup '{}' to '{}'.",
@@ -699,10 +574,13 @@ async fn export_backup(context: &Context, dir: &Path) -> Result<()> {
         dest_path.display(),
     );
 
-    let res = export_backup_inner(context, &temp_path).await;
+    context
+        .sql
+        .export(&temp_db_path, passphrase)
+        .await
+        .with_context(|| format!("failed to backup plaintext database to {:?}", temp_db_path))?;
 
-    // we re-open the database after export is finished
-    context.sql.open(context, context.get_dbfile(), false).await;
+    let res = export_backup_inner(context, &temp_db_path, &temp_path).await;
 
     match &res {
         Ok(_) => {
@@ -721,18 +599,21 @@ impl Drop for DeleteOnDrop {
     fn drop(&mut self) {
         let file = self.0.clone();
         // Not using dc_delete_file() here because it would send a DeletedBlobFile event
-        async_std::task::block_on(async move { fs::remove_file(file).await.ok() });
+        async_std::task::block_on(fs::remove_file(file)).ok();
     }
 }
 
-async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
+async fn export_backup_inner(
+    context: &Context,
+    temp_db_path: &Path,
+    temp_path: &Path,
+) -> Result<()> {
     let file = File::create(temp_path).await?;
 
     let mut builder = async_tar::Builder::new(file);
 
-    // append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument.
     builder
-        .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
+        .append_path_with_name(temp_db_path, DBFILE_BACKUP_NAME)
         .await?;
 
     let read_dir: Vec<_> = fs::read_dir(context.get_blobdir()).await?.collect().await;
@@ -936,9 +817,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file() {
-        let t = TestContext::new().await;
-
-        t.configure_alice().await;
+        let t = TestContext::new_alice().await;
         let msg = render_setup_file(&t, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
@@ -955,11 +834,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
-        t.configure_alice().await;
         let msg = render_setup_file(&t, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
@@ -1012,20 +890,66 @@ mod tests {
 
     #[async_std::test]
     async fn test_export_and_import_key() {
-        let context = TestContext::new().await;
-        context.configure_alice().await;
+        let context = TestContext::new_alice().await;
         let blobdir = context.ctx.get_blobdir();
-        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir).await {
+        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir, None).await {
             panic!("got error on export: {:?}", err);
         }
 
-        let context2 = TestContext::new().await;
-        context2.configure_alice().await;
-        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir).await {
+        let context2 = TestContext::new_alice().await;
+        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir, None).await {
             panic!("got error on import: {:?}", err);
         }
     }
 
+    #[async_std::test]
+    async fn test_export_and_import_backup() -> Result<()> {
+        let backup_dir = tempfile::tempdir().unwrap();
+
+        let context1 = TestContext::new_alice().await;
+        assert!(context1.is_configured().await?);
+
+        let context2 = TestContext::new().await;
+        assert!(!context2.is_configured().await?);
+        assert!(has_backup(&context2, backup_dir.path().as_ref())
+            .await
+            .is_err());
+
+        // export from context1
+        assert!(imex(
+            &context1,
+            ImexMode::ExportBackup,
+            backup_dir.path().as_ref(),
+            None,
+        )
+        .await
+        .is_ok());
+        let _event = context1
+            .evtracker
+            .get_matching(|evt| matches!(evt, EventType::ImexProgress(1000)))
+            .await;
+
+        // import to context2
+        let backup = has_backup(&context2, backup_dir.path().as_ref()).await?;
+        assert!(
+            imex(&context2, ImexMode::ImportBackup, backup.as_ref(), None)
+                .await
+                .is_ok()
+        );
+        let _event = context2
+            .evtracker
+            .get_matching(|evt| matches!(evt, EventType::ImexProgress(1000)))
+            .await;
+
+        assert!(context2.is_configured().await?);
+        assert_eq!(
+            context2.get_config(Config::Addr).await?,
+            Some("alice@example.org".to_string())
+        );
+
+        Ok(())
+    }
+
     #[test]
     fn test_normalize_setup_code() {
         let norm = normalize_setup_code("123422343234423452346234723482349234");

src/key.rs

@@ -4,7 +4,7 @@ use std::collections::BTreeMap;
 use std::fmt;
 use std::io::Cursor;
 
-use anyhow::{format_err, Result};
+use anyhow::{format_err, Context as _, Result};
 use async_trait::async_trait;
 use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
@@ -50,8 +50,7 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     /// the ASCII-armored representation.
     fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
         let bytes = data.as_bytes();
-        Self::KeyType::from_armor_single(Cursor::new(bytes))
-            .map_err(|err| format_err!("rPGP error: {}", err))
+        Self::KeyType::from_armor_single(Cursor::new(bytes)).context("rPGP error")
     }
 
     /// Load the users' default key from the database.
@@ -92,16 +91,17 @@ impl DcKey for SignedPublicKey {
     type KeyType = SignedPublicKey;
 
     async fn load_self(context: &Context) -> Result<Self::KeyType> {
+        let addr = context.get_primary_self_addr().await?;
         match context
             .sql
             .query_row_optional(
                 r#"
             SELECT public_key
               FROM keypairs
-             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
+             WHERE addr=?
                AND is_default=1;
             "#,
-                paramsv![],
+                paramsv![addr],
                 |row| {
                     let bytes: Vec<u8> = row.get(0)?;
                     Ok(bytes)
@@ -199,10 +199,7 @@ impl DcSecretKey for SignedSecretKey {
 }
 
 async fn generate_keypair(context: &Context) -> Result<KeyPair> {
-    let addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await?
-        .ok_or_else(|| format_err!("No address configured"))?;
+    let addr = context.get_primary_self_addr().await?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -289,17 +286,17 @@ pub async fn store_self_keypair(
             paramsv![public_key, secret_key],
         )
         .await
-        .map_err(|err| err.context("failed to remove old use of key"))?;
+        .context("failed to remove old use of key")?;
     if default == KeyPairUse::Default {
         context
             .sql
             .execute("UPDATE keypairs SET is_default=0;", paramsv![])
             .await
-            .map_err(|err| err.context("failed to clear default"))?;
+            .context("failed to clear default")?;
     }
     let is_default = match default {
-        KeyPairUse::Default => true as i32,
-        KeyPairUse::ReadOnly => false as i32,
+        KeyPairUse::Default => i32::from(true),
+        KeyPairUse::ReadOnly => i32::from(false),
     };
 
     let addr = keypair.addr.to_string();
@@ -313,13 +310,13 @@ pub async fn store_self_keypair(
             paramsv![addr, is_default, public_key, secret_key, t],
         )
         .await
-        .map_err(|err| err.context("failed to insert keypair"))?;
+        .context("failed to insert keypair")?;
 
     Ok(())
 }
 
 /// A key fingerprint
-#[derive(Clone, Eq, PartialEq, Hash)]
+#[derive(Clone, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)]
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {
@@ -510,8 +507,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
-        let t = TestContext::new().await;
-        t.configure_alice().await;
+        let t = TestContext::new_alice().await;
         let pubkey = SignedPublicKey::load_self(&t).await.unwrap();
         assert_eq!(alice.public, pubkey);
         let seckey = SignedSecretKey::load_self(&t).await.unwrap();
@@ -521,7 +517,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_generate_public() {
         let t = TestContext::new().await;
-        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
             .await
             .unwrap();
         let key = SignedPublicKey::load_self(&t).await;
@@ -531,7 +527,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[async_std::test]
     async fn test_load_self_generate_secret() {
         let t = TestContext::new().await;
-        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
             .await
             .unwrap();
         let key = SignedSecretKey::load_self(&t).await;
@@ -543,7 +539,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         use std::thread;
 
         let t = TestContext::new().await;
-        t.set_config(Config::ConfiguredAddr, Some("alice@example.com"))
+        t.set_config(Config::ConfiguredAddr, Some("alice@example.org"))
             .await
             .unwrap();
         let thr0 = {
@@ -589,27 +585,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(nrows().await, 1);
     }
 
-    // Convenient way to create a new key if you need one, run with
-    // `cargo test key::tests::gen_key`.
-    // #[test]
-    // fn gen_key() {
-    //     let name = "fiona";
-    //     let keypair = crate::pgp::create_keypair(
-    //         EmailAddress::new(&format!("{}@example.net", name)).unwrap(),
-    //     )
-    //     .unwrap();
-    //     std::fs::write(
-    //         format!("test-data/key/{}-public.asc", name),
-    //         keypair.public.to_base64(),
-    //     )
-    //     .unwrap();
-    //     std::fs::write(
-    //         format!("test-data/key/{}-secret.asc", name),
-    //         keypair.secret.to_base64(),
-    //     )
-    //     .unwrap();
-    // }
-
     #[test]
     fn test_fingerprint_from_str() {
         let res = Fingerprint::new(vec![

src/keyring.rs

@@ -79,8 +79,7 @@ mod tests {
     #[async_std::test]
     async fn test_keyring_load_self() {
         // new_self() implies load_self()
-        let t = TestContext::new().await;
-        t.configure_alice().await;
+        let t = TestContext::new_alice().await;
         let alice = alice_keypair();
 
         let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t).await.unwrap();

src/lib.rs

@@ -7,12 +7,14 @@
     clippy::all,
     clippy::indexing_slicing,
     clippy::wildcard_imports,
-    clippy::needless_borrow
+    clippy::needless_borrow,
+    clippy::cast_lossless
 )]
 #![allow(
     clippy::match_bool,
     clippy::eval_order_dependence,
-    clippy::bool_assert_comparison
+    clippy::bool_assert_comparison,
+    clippy::manual_split_once
 )]
 
 #[macro_use]
@@ -77,6 +79,7 @@ pub mod peerstate;
 pub mod pgp;
 pub mod provider;
 pub mod qr;
+pub mod qr_code_generator;
 pub mod quota;
 pub mod securejoin;
 mod simplify;
@@ -85,6 +88,7 @@ pub mod stock_str;
 mod sync;
 mod token;
 mod update_helper;
+pub mod webxdc;
 #[macro_use]
 mod dehtml;
 mod color;

src/location.rs

@@ -1,20 +1,20 @@
 //! Location handling.
 use std::convert::TryFrom;
 
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Context as _, Result};
+use async_std::channel::Receiver;
+use async_std::future::timeout;
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+use std::time::Duration;
 
 use crate::chat::{self, ChatId};
-use crate::config::Config;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+use crate::contact::ContactId;
 use crate::context::Context;
-use crate::dc_tools::time;
+use crate::dc_tools::{duration_to_str, time};
 use crate::events::EventType;
-use crate::job::{self, Job};
-use crate::message::{Message, MsgId};
+use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
-use crate::param::Params;
 use crate::stock_str;
 
 /// Location record
@@ -25,7 +25,7 @@ pub struct Location {
     pub longitude: f64,
     pub accuracy: f64,
     pub timestamp: i64,
-    pub contact_id: u32,
+    pub contact_id: ContactId,
     pub msg_id: u32,
     pub chat_id: ChatId,
     pub marker: Option<String>,
@@ -101,10 +101,10 @@ impl Kml {
             let val = event.unescape_and_decode(reader).unwrap_or_default();
 
             let val = val
-                .replace("\n", "")
-                .replace("\r", "")
-                .replace("\t", "")
-                .replace(" ", "");
+                .replace('\n', "")
+                .replace('\r', "")
+                .replace('\t', "")
+                .replace(' ', "");
 
             if self.tag.contains(KmlTag::WHEN) && val.len() >= 19 {
                 // YYYY-MM-DDTHH:MM:SSZ
@@ -223,36 +223,15 @@ pub async fn send_locations_to_chat(
             .unwrap_or_default();
     } else if 0 == seconds && is_sending_locations_before {
         let stock_str = stock_str::msg_location_disabled(context).await;
-        chat::add_info_msg(context, chat_id, stock_str, now).await?;
+        chat::add_info_msg(context, chat_id, &stock_str, now).await?;
     }
     context.emit_event(EventType::ChatModified(chat_id));
     if 0 != seconds {
-        schedule_maybe_send_locations(context, false).await?;
-        job::add(
-            context,
-            job::Job::new(
-                job::Action::MaybeSendLocationsEnded,
-                chat_id.to_u32(),
-                Params::new(),
-                seconds + 1,
-            ),
-        )
-        .await?;
+        context.interrupt_location().await;
     }
     Ok(())
 }
 
-async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) -> Result<()> {
-    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await? {
-        job::add(
-            context,
-            job::Job::new(job::Action::MaybeSendLocations, 0, Params::new(), 60),
-        )
-        .await?;
-    };
-    Ok(())
-}
-
 /// Returns whether `chat_id` or any chat is sending locations.
 ///
 /// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
@@ -314,18 +293,18 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                         accuracy,
                         time(),
                         chat_id,
-                        DC_CONTACT_ID_SELF,
+                        ContactId::SELF,
                     ]
             ).await {
                 warn!(context, "failed to store location {:?}", err);
             } else {
+                info!(context, "stored location for chat {}", chat_id);
                 continue_streaming = true;
             }
         }
         if continue_streaming {
-            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+            context.emit_event(EventType::LocationChanged(Some(ContactId::SELF)));
         };
-        schedule_maybe_send_locations(context, false).await.ok();
     }
 
     continue_streaming
@@ -424,10 +403,7 @@ pub async fn delete_all(context: &Context) -> Result<()> {
 pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)> {
     let mut last_added_location_id = 0;
 
-    let self_addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await?
-        .unwrap_or_default();
+    let self_addr = context.get_primary_self_addr().await?;
 
     let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
         "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
@@ -462,10 +438,10 @@ pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)
              GROUP BY timestamp \
              ORDER BY timestamp;",
                 paramsv![
-                    DC_CONTACT_ID_SELF,
+                    ContactId::SELF,
                     locations_send_begin,
                     locations_last_sent,
-                    DC_CONTACT_ID_SELF
+                    ContactId::SELF
                 ],
                 |row| {
                     let location_id: i32 = row.get(0)?;
@@ -558,7 +534,7 @@ pub async fn set_msg_location_id(context: &Context, msg_id: MsgId, location_id:
 pub(crate) async fn save(
     context: &Context,
     chat_id: ChatId,
-    contact_id: u32,
+    contact_id: ContactId,
     locations: &[Location],
     independent: bool,
 ) -> Result<Option<u32>> {
@@ -585,12 +561,12 @@ pub(crate) async fn save(
             conn.prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?;
         let mut stmt_insert = conn.prepare_cached(stmt_insert)?;
 
-        let exists = stmt_test.exists(paramsv![timestamp, contact_id as i32])?;
+        let exists = stmt_test.exists(paramsv![timestamp, contact_id])?;
 
         if independent || !exists {
             stmt_insert.execute(paramsv![
                 timestamp,
-                contact_id as i32,
+                contact_id,
                 chat_id,
                 latitude,
                 longitude,
@@ -611,147 +587,140 @@ pub(crate) async fn save(
     Ok(newest_location_id)
 }
 
-pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> job::Status {
-    let now = time();
-    let mut continue_streaming = false;
-    info!(
-        context,
-        " ----------------- MAYBE_SEND_LOCATIONS -------------- ",
-    );
+pub(crate) async fn location_loop(context: &Context, interrupt_receiver: Receiver<()>) {
+    loop {
+        let next_event = match maybe_send_locations(context).await {
+            Err(err) => {
+                warn!(context, "maybe_send_locations failed: {}", err);
+                Some(60) // Retry one minute later.
+            }
+            Ok(next_event) => next_event,
+        };
 
+        let duration = if let Some(next_event) = next_event {
+            Duration::from_secs(next_event)
+        } else {
+            Duration::from_secs(86400)
+        };
+
+        info!(
+            context,
+            "Location loop is waiting for {} or interrupt",
+            duration_to_str(duration)
+        );
+        timeout(duration, interrupt_receiver.recv()).await.ok();
+    }
+}
+
+/// Returns number of seconds until the next time location streaming for some chat ends
+/// automatically.
+async fn maybe_send_locations(context: &Context) -> Result<Option<u64>> {
+    let mut next_event: Option<u64> = None;
+
+    let now = time();
     let rows = context
         .sql
         .query_map(
-            "SELECT id, locations_send_begin, locations_last_sent \
-         FROM chats \
-         WHERE locations_send_until>?;",
-            paramsv![now],
+            "SELECT id, locations_send_begin, locations_send_until, locations_last_sent
+             FROM chats
+             WHERE locations_send_until>0",
+            [],
             |row| {
                 let chat_id: ChatId = row.get(0)?;
                 let locations_send_begin: i64 = row.get(1)?;
-                let locations_last_sent: i64 = row.get(2)?;
-                continue_streaming = true;
-
-                // be a bit tolerant as the timer may not align exactly with time(NULL)
-                if now - locations_last_sent < (60 - 3) {
-                    Ok(None)
-                } else {
-                    Ok(Some((chat_id, locations_send_begin, locations_last_sent)))
-                }
+                let locations_send_until: i64 = row.get(2)?;
+                let locations_last_sent: i64 = row.get(3)?;
+                Ok((
+                    chat_id,
+                    locations_send_begin,
+                    locations_send_until,
+                    locations_last_sent,
+                ))
             },
             |rows| {
-                rows.filter_map(|v| v.transpose())
-                    .collect::<std::result::Result<Vec<_>, _>>()
+                rows.collect::<std::result::Result<Vec<_>, _>>()
                     .map_err(Into::into)
             },
         )
-        .await;
+        .await
+        .context("failed to query location streaming chats")?;
 
-    if let Ok(rows) = rows {
-        let mut msgs = Vec::new();
-
-        {
-            let conn = job_try!(context.sql.get_conn().await);
-
-            let mut stmt_locations = job_try!(conn.prepare_cached(
-                "SELECT id \
+    for (chat_id, locations_send_begin, locations_send_until, locations_last_sent) in rows {
+        if locations_send_begin > 0 && locations_send_until > now {
+            let can_send = now > locations_last_sent + 60;
+            let has_locations = context
+                .sql
+                .exists(
+                    "SELECT COUNT(id) \
      FROM locations \
      WHERE from_id=? \
      AND timestamp>=? \
      AND timestamp>? \
-     AND independent=0 \
-             ORDER BY timestamp;",
-            ));
+     AND independent=0",
+                    paramsv![ContactId::SELF, locations_send_begin, locations_last_sent,],
+                )
+                .await?;
 
-            for (chat_id, locations_send_begin, locations_last_sent) in &rows {
-                if !stmt_locations
-                    .exists(paramsv![
-                        DC_CONTACT_ID_SELF,
-                        *locations_send_begin,
-                        *locations_last_sent,
-                    ])
-                    .unwrap_or_default()
-                {
-                    // if there is no new location, there's nothing to send.
-                    // however, maybe we want to bypass this test eg. 15 minutes
-                } else {
-                    // pending locations are attached automatically to every message,
+            next_event = next_event
+                .into_iter()
+                .chain(u64::try_from(locations_send_until - now).into_iter())
+                .min();
+
+            if has_locations {
+                if can_send {
+                    // Send location-only message.
+                    // Pending locations are attached automatically to every message,
                     // so also to this empty text message.
-                    // DC_CMD_LOCATION is only needed to create a nicer subject.
-                    //
-                    // for optimisation and to avoid flooding the sending queue,
-                    // we could sending these messages only if we're really online.
-                    // the easiest way to determine this, is to check for an empty message queue.
-                    // (might not be 100%, however, as positions are sent combined later
-                    // and dc_set_location() is typically called periodically, this is ok)
+                    info!(
+                        context,
+                        "Chat {} has pending locations, sending them.", chat_id
+                    );
                     let mut msg = Message::new(Viewtype::Text);
                     msg.hidden = true;
                     msg.param.set_cmd(SystemMessage::LocationOnly);
-                    msgs.push((*chat_id, msg));
+                    chat::send_msg(context, chat_id, &mut msg).await?;
+                } else {
+                    // Wait until pending locations can be sent.
+                    info!(
+                        context,
+                        "Chat {} has pending locations, but they can't be sent yet.", chat_id
+                    );
+                    next_event = next_event
+                        .into_iter()
+                        .chain(u64::try_from(locations_last_sent + 61 - now).into_iter())
+                        .min();
                 }
+            } else {
+                info!(
+                    context,
+                    "Chat {} has location streaming enabled, but no pending locations.", chat_id
+                );
             }
-        }
-
-        for (chat_id, mut msg) in msgs.into_iter() {
-            // TODO: better error handling
-            chat::send_msg(context, chat_id, &mut msg)
-                .await
-                .unwrap_or_default();
-        }
-    }
-
-    if continue_streaming {
-        job_try!(schedule_maybe_send_locations(context, true).await);
-    }
-    job::Status::Finished(Ok(()))
-}
-
-pub(crate) async fn job_maybe_send_locations_ended(
-    context: &Context,
-    job: &mut Job,
-) -> job::Status {
-    // this function is called when location-streaming _might_ have ended for a chat.
-    // the function checks, if location-streaming is really ended;
-    // if so, a device-message is added if not yet done.
-
-    let chat_id = ChatId::new(job.foreign_id);
-
-    let (send_begin, send_until) = job_try!(
-        context
-            .sql
-            .query_row(
-                "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
-                paramsv![chat_id],
-                |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
-            )
-            .await
-    );
-
-    let now = time();
-    if !(send_begin != 0 && now <= send_until) {
-        // still streaming -
-        // may happen as several calls to dc_send_locations_to_chat()
-        // do not un-schedule pending DC_MAYBE_SEND_LOC_ENDED jobs
-        if !(send_begin == 0 && send_until == 0) {
-            // not streaming, device-message already sent
-            job_try!(
-                context
-                    .sql
-                    .execute(
-                        "UPDATE chats \
-                             SET locations_send_begin=0, locations_send_until=0 \
- WHERE id=?",
-                        paramsv![chat_id],
-                    )
-                    .await
+        } else {
+            // Location streaming was either explicitly disabled (locations_send_begin = 0) or
+            // locations_send_until is in the past.
+            info!(
+                context,
+                "Disabling location streaming for chat {}.", chat_id
             );
+            context
+                .sql
+                .execute(
+                    "UPDATE chats \
+                         SET locations_send_begin=0, locations_send_until=0 \
+                         WHERE id=?",
+                    paramsv![chat_id],
+                )
+                .await
+                .context("failed to disable location streaming")?;
 
             let stock_str = stock_str::msg_location_disabled(context).await;
-            job_try!(chat::add_info_msg(context, chat_id, stock_str, now).await);
+            chat::add_info_msg(context, chat_id, &stock_str, now).await?;
             context.emit_event(EventType::ChatModified(chat_id));
         }
     }
-    job::Status::Finished(Ok(()))
+
+    Ok(next_event)
 }
 
 #[cfg(test)]
@@ -759,6 +728,7 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
+    use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils::TestContext;
 
     #[async_std::test]
@@ -819,4 +789,68 @@ mod tests {
         assert!(!is_marker(" "));
         assert!(!is_marker("\t"));
     }
+
+    /// Tests that location.kml is hidden.
+    #[async_std::test]
+    async fn receive_location_kml() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        dc_receive_imf(
+            &alice,
+            br#"Subject: Hello
+Message-ID: hello@example.net
+To: Alice <alice@example.org>
+From: Bob <bob@example.net>
+Date: Mon, 20 Dec 2021 00:00:00 +0000
+Chat-Version: 1.0
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+Text message."#,
+            false,
+        )
+        .await?;
+        let received_msg = alice.get_last_msg().await;
+        assert_eq!(received_msg.text.unwrap(), "Text message.");
+
+        dc_receive_imf(
+            &alice,
+            br#"Subject: locations
+MIME-Version: 1.0
+To: <alice@example.org>
+From: <bob@example.net>
+Date: Tue, 21 Dec 2021 00:00:00 +0000
+Chat-Version: 1.0
+Message-ID: <foobar@example.net>
+Content-Type: multipart/mixed; boundary="U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF"
+
+
+--U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+
+
+--U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
+Content-Type: application/vnd.google-earth.kml+xml
+Content-Disposition: attachment; filename="location.kml"
+
+<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2">
+<Document addr="bob@example.net">
+<Placemark><Timestamp><when>2021-11-21T00:00:00Z</when></Timestamp><Point><coordinates accuracy="1.0000000000000000">10.00000000000000,20.00000000000000</coordinates></Point></Placemark>
+</Document>
+</kml>
+
+--U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF--"#,
+            false,
+        )
+        .await?;
+
+        // Received location message is not visible, last message stays the same.
+        let received_msg2 = alice.get_last_msg().await;
+        assert_eq!(received_msg2.id, received_msg.id);
+
+        let locations = get_range(&alice, None, None, 0, 0).await?;
+        assert_eq!(locations.len(), 1);
+        Ok(())
+    }
 }

src/login_param.rs

@@ -139,8 +139,18 @@ pub struct LoginParam {
 }
 
 impl LoginParam {
+    /// Load entered (candidate) account settings
+    pub async fn load_candidate_params(context: &Context) -> Result<Self> {
+        LoginParam::from_database(context, "").await
+    }
+
+    /// Load configured (working) account settings
+    pub async fn load_configured_params(context: &Context) -> Result<Self> {
+        LoginParam::from_database(context, "configured_").await
+    }
+
     /// Read the login parameters from the database.
-    pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
+    async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
@@ -242,18 +252,18 @@ impl LoginParam {
     }
 
     /// Save this loginparam to the database.
-    pub async fn save_to_database(&self, context: &Context, prefix: impl AsRef<str>) -> Result<()> {
-        let prefix = prefix.as_ref();
+    pub async fn save_as_configured_params(&self, context: &Context) -> Result<()> {
+        let prefix = "configured_";
         let sql = &context.sql;
 
-        let key = format!("{}addr", prefix);
-        sql.set_raw_config(key, Some(&self.addr)).await?;
+        context.set_primary_self_addr(&self.addr).await?;
 
         let key = format!("{}mail_server", prefix);
         sql.set_raw_config(key, Some(&self.imap.server)).await?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_raw_config_int(key, self.imap.port as i32).await?;
+        sql.set_raw_config_int(key, i32::from(self.imap.port))
+            .await?;
 
         let key = format!("{}mail_user", prefix);
         sql.set_raw_config(key, Some(&self.imap.user)).await?;
@@ -273,7 +283,8 @@ impl LoginParam {
         sql.set_raw_config(key, Some(&self.smtp.server)).await?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_raw_config_int(key, self.smtp.port as i32).await?;
+        sql.set_raw_config_int(key, i32::from(self.smtp.port))
+            .await?;
 
         let key = format!("{}send_user", prefix);
         sql.set_raw_config(key, Some(&self.smtp.user)).await?;
@@ -413,7 +424,7 @@ mod tests {
         let t = TestContext::new().await;
 
         let param = LoginParam {
-            addr: "alice@example.com".to_string(),
+            addr: "alice@example.org".to_string(),
             imap: ServerLoginParam {
                 server: "imap.example.com".to_string(),
                 user: "alice".to_string(),
@@ -424,7 +435,7 @@ mod tests {
             },
             smtp: ServerLoginParam {
                 server: "smtp.example.com".to_string(),
-                user: "alice@example.com".to_string(),
+                user: "alice@example.org".to_string(),
                 password: "bar".to_string(),
                 port: 456,
                 security: Socket::Ssl,
@@ -436,8 +447,8 @@ mod tests {
             socks5_config: None,
         };
 
-        param.save_to_database(&t, "foobar_").await?;
-        let loaded = LoginParam::from_database(&t, "foobar_").await?;
+        param.save_as_configured_params(&t).await?;
+        let loaded = LoginParam::load_configured_params(&t).await?;
 
         assert_eq!(param, loaded);
         Ok(())

src/mimefactory.rs

@@ -2,14 +2,14 @@
 
 use std::convert::TryInto;
 
-use anyhow::{bail, ensure, format_err, Result};
+use anyhow::{bail, ensure, Context as _, Result};
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
 
 use crate::blob::BlobObject;
-use crate::chat::{self, Chat};
+use crate::chat::Chat;
 use crate::config::Config;
-use crate::constants::{Chattype, Viewtype, DC_FROM_HANDSHAKE};
+use crate::constants::{Chattype, DC_FROM_HANDSHAKE};
 use crate::contact::Contact;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::IsNoneOrEmpty;
@@ -22,7 +22,7 @@ use crate::ephemeral::Timer as EphemeralTimer;
 use crate::format_flowed::{format_flowed, format_flowed_quote};
 use crate::html::new_html_mimepart;
 use crate::location;
-use crate::message::{self, Message, MsgId};
+use crate::message::{self, Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
@@ -81,7 +81,7 @@ pub struct MimeFactory<'a> {
 /// Result of rendering a message, ready to be submitted to a send job.
 #[derive(Debug, Clone)]
 pub struct RenderedEmail {
-    pub message: Vec<u8>,
+    pub message: String,
     // pub envelope: Envelope,
     pub is_encrypted: bool,
     pub is_gossiped: bool,
@@ -133,11 +133,7 @@ impl<'a> MimeFactory<'a> {
     ) -> Result<MimeFactory<'a>> {
         let chat = Chat::load_from_db(context, msg.chat_id).await?;
 
-        let from_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
-
+        let from_addr = context.get_primary_self_addr().await?;
         let config_displayname = context
             .get_config(Config::Displayname)
             .await?
@@ -154,6 +150,12 @@ impl<'a> MimeFactory<'a> {
 
         if chat.is_self_talk() {
             recipients.push((from_displayname.to_string(), from_addr.to_string()));
+        } else if chat.is_mailing_list() {
+            let list_post = chat
+                .param
+                .get(Param::ListPost)
+                .context("Can't write to mailinglist without ListPost param")?;
+            recipients.push(("".to_string(), list_post.to_string()));
         } else {
             context
                 .sql
@@ -201,7 +203,6 @@ impl<'a> MimeFactory<'a> {
             )
             .await?;
 
-        let default_str = stock_str::status_line(context).await;
         let factory = MimeFactory {
             from_addr,
             from_displayname,
@@ -209,7 +210,7 @@ impl<'a> MimeFactory<'a> {
             selfstatus: context
                 .get_config(Config::Selfstatus)
                 .await?
-                .unwrap_or(default_str),
+                .unwrap_or_default(),
             recipients,
             timestamp: msg.timestamp_sort,
             loaded: Loaded::Message { chat },
@@ -232,19 +233,15 @@ impl<'a> MimeFactory<'a> {
         ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
         let contact = Contact::load_from_db(context, msg.from_id).await?;
-        let from_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default();
+        let from_addr = context.get_primary_self_addr().await?;
         let from_displayname = context
             .get_config(Config::Displayname)
             .await?
             .unwrap_or_default();
-        let default_str = stock_str::status_line(context).await;
         let selfstatus = context
             .get_config(Config::Selfstatus)
             .await?
-            .unwrap_or(default_str);
+            .unwrap_or_default();
         let timestamp = dc_create_smeared_timestamp(context).await;
 
         let res = MimeFactory::<'a> {
@@ -274,10 +271,7 @@ impl<'a> MimeFactory<'a> {
         &self,
         context: &Context,
     ) -> Result<Vec<(Option<Peerstate>, &str)>> {
-        let self_addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .ok_or_else(|| format_err!("Not configured"))?;
+        let self_addr = context.get_primary_self_addr().await?;
 
         let mut res = Vec::new();
         for (_, addr) in self
@@ -458,49 +452,121 @@ impl<'a> MimeFactory<'a> {
             self.from_addr.clone(),
         );
 
+        let undisclosed_recipients = match &self.loaded {
+            Loaded::Message { chat } => chat.typ == Chattype::Broadcast,
+            Loaded::Mdn { .. } => false,
+        };
+
         let mut to = Vec::new();
-        for (name, addr) in self.recipients.iter() {
-            if name.is_empty() {
-                to.push(Address::new_mailbox(addr.clone()));
-            } else {
-                to.push(Address::new_mailbox_with_name(
-                    name.to_string(),
-                    addr.clone(),
-                ));
+        if undisclosed_recipients {
+            to.push(Address::new_group(
+                "hidden-recipients".to_string(),
+                Vec::new(),
+            ));
+        } else {
+            let email_to_remove =
+                if self.msg.param.get_cmd() == SystemMessage::MemberRemovedFromGroup {
+                    self.msg.param.get(Param::Arg)
+                } else {
+                    None
+                };
+
+            for (name, addr) in self.recipients.iter() {
+                if let Some(email_to_remove) = email_to_remove {
+                    if email_to_remove == addr {
+                        continue;
+                    }
+                }
+
+                if name.is_empty() {
+                    to.push(Address::new_mailbox(addr.clone()));
+                } else {
+                    to.push(Address::new_mailbox_with_name(
+                        name.to_string(),
+                        addr.clone(),
+                    ));
+                }
+            }
+
+            if to.is_empty() {
+                to.push(from.clone());
             }
         }
 
-        if to.is_empty() {
-            to.push(from.clone());
+        // Start with Internet Message Format headers in the order of the standard example
+        // <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.1.1>.
+        headers
+            .unprotected
+            .push(Header::new_with_value("From".into(), vec![from]).unwrap());
+        if let Some(sender_displayname) = &self.sender_displayname {
+            let sender =
+                Address::new_mailbox_with_name(sender_displayname.clone(), self.from_addr.clone());
+            headers
+                .unprotected
+                .push(Header::new_with_value("Sender".into(), vec![sender]).unwrap());
+        }
+        headers
+            .unprotected
+            .push(Header::new_with_value("To".into(), to).unwrap());
+
+        let subject_str = self.subject_str(context).await?;
+        let encoded_subject = if subject_str
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
+        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
+        // but we do not want to encode all subjects just because they contain a space.
+        {
+            subject_str.clone()
+        } else {
+            encode_words(&subject_str)
+        };
+        headers
+            .protected
+            .push(Header::new("Subject".into(), encoded_subject));
+
+        let date = chrono::Utc
+            .from_local_datetime(&chrono::NaiveDateTime::from_timestamp(self.timestamp, 0))
+            .unwrap()
+            .to_rfc2822();
+        headers.unprotected.push(Header::new("Date".into(), date));
+
+        let rfc724_mid = match self.loaded {
+            Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
+            Loaded::Mdn { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
+        };
+        let rfc724_mid_headervalue = render_rfc724_mid(&rfc724_mid);
+
+        // Amazon's SMTP servers change the `Message-ID`, just as Outlook's SMTP servers do.
+        // Outlook's servers add an `X-Microsoft-Original-Message-ID` header with the original `Message-ID`,
+        // and when downloading messages we look for this header in order to correctly identify
+        // messages.
+        // Amazon's servers do not add such a header, so we just add it ourselves.
+        if let Some(server) = context.get_config(Config::ConfiguredSendServer).await? {
+            if server.ends_with(".amazonaws.com") {
+                headers.unprotected.push(Header::new(
+                    "X-Microsoft-Original-Message-ID".into(),
+                    rfc724_mid_headervalue.clone(),
+                ))
+            }
         }
 
         headers
             .unprotected
-            .push(Header::new("MIME-Version".into(), "1.0".into()));
+            .push(Header::new("Message-ID".into(), rfc724_mid_headervalue));
 
+        // Reply headers as in <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.2>.
+        if !self.in_reply_to.is_empty() {
+            headers
+                .unprotected
+                .push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
+        }
         if !self.references.is_empty() {
             headers
                 .unprotected
                 .push(Header::new("References".into(), self.references.clone()));
         }
 
-        if !self.in_reply_to.is_empty() {
-            headers
-                .unprotected
-                .push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
-        }
-
-        let date = chrono::Utc
-            .from_local_datetime(&chrono::NaiveDateTime::from_timestamp(self.timestamp, 0))
-            .unwrap()
-            .to_rfc2822();
-
-        headers.unprotected.push(Header::new("Date".into(), date));
-
-        headers
-            .unprotected
-            .push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
-
+        // Automatic Response headers <https://www.rfc-editor.org/rfc/rfc3834>
         if let Loaded::Mdn { .. } = self.loaded {
             headers.unprotected.push(Header::new(
                 "Auto-Submitted".to_string(),
@@ -513,6 +579,11 @@ impl<'a> MimeFactory<'a> {
             ));
         }
 
+        // Non-standard headers.
+        headers
+            .unprotected
+            .push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
+
         if self.req_mdn {
             // we use "Chat-Disposition-Notification-To"
             // because replies to "Disposition-Notification-To" are weird in many cases
@@ -527,21 +598,9 @@ impl<'a> MimeFactory<'a> {
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
         let skip_autocrypt = self.should_skip_autocrypt();
-        let subject_str = self.subject_str(context).await?;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let encrypt_helper = EncryptHelper::new(context).await?;
 
-        let encoded_subject = if subject_str
-            .chars()
-            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
-        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
-        // but we do not want to encode all subjects just because they contain a space.
-        {
-            subject_str.clone()
-        } else {
-            encode_words(&subject_str)
-        };
-
         if !skip_autocrypt {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
@@ -550,15 +609,6 @@ impl<'a> MimeFactory<'a> {
                 .push(Header::new("Autocrypt".into(), aheader));
         }
 
-        headers
-            .protected
-            .push(Header::new("Subject".into(), encoded_subject));
-
-        let rfc724_mid = match self.loaded {
-            Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
-            Loaded::Mdn { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
-        };
-
         let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(context).await?;
         if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
             headers.protected.push(Header::new(
@@ -567,36 +617,11 @@ impl<'a> MimeFactory<'a> {
             ));
         }
 
-        headers.unprotected.push(Header::new(
-            "Message-ID".into(),
-            render_rfc724_mid(&rfc724_mid),
-        ));
-
-        let undisclosed_recipients = match &self.loaded {
-            Loaded::Message { chat } => chat.typ == Chattype::Broadcast,
-            Loaded::Mdn { .. } => false,
-        };
-
-        if undisclosed_recipients {
-            headers
-                .unprotected
-                .push(Header::new("To".into(), "hidden-recipients: ;".to_string()));
-        } else {
-            headers
-                .unprotected
-                .push(Header::new_with_value("To".into(), to).unwrap());
-        }
-
+        // MIME header <https://datatracker.ietf.org/doc/html/rfc2045>.
+        // Content-Type
         headers
             .unprotected
-            .push(Header::new_with_value("From".into(), vec![from]).unwrap());
-        if let Some(sender_displayname) = &self.sender_displayname {
-            let sender =
-                Address::new_mailbox_with_name(sender_displayname.clone(), self.from_addr.clone());
-            headers
-                .unprotected
-                .push(Header::new_with_value("Sender".into(), vec![sender]).unwrap());
-        }
+            .push(Header::new("MIME-Version".into(), "1.0".into()));
 
         let mut is_gossiped = false;
 
@@ -623,6 +648,11 @@ impl<'a> MimeFactory<'a> {
                     "Content-Type".to_string(),
                     "multipart/report; report-type=multi-device-sync".to_string(),
                 ))
+            } else if self.msg.param.get_cmd() == SystemMessage::WebxdcStatusUpdate {
+                PartBuilder::new().header((
+                    "Content-Type".to_string(),
+                    "multipart/report; report-type=status-update".to_string(),
+                ))
             } else {
                 PartBuilder::new().message_type(MimeMultipartType::Mixed)
             };
@@ -747,7 +777,7 @@ impl<'a> MimeFactory<'a> {
         } = self;
 
         Ok(RenderedEmail {
-            message: outer_message.build().as_string().into_bytes(),
+            message: outer_message.build().as_string(),
             // envelope: Envelope::new,
             is_encrypted,
             is_gossiped,
@@ -820,9 +850,12 @@ impl<'a> MimeFactory<'a> {
         }
 
         if chat.typ == Chattype::Group {
-            headers
-                .protected
-                .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
+            // Send group ID unless it is an ad hoc group that has no ID.
+            if !chat.grpid.is_empty() {
+                headers
+                    .protected
+                    .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
+            }
 
             let encoded = encode_words(&chat.name);
             headers
@@ -897,7 +930,9 @@ impl<'a> MimeFactory<'a> {
                     "ephemeral-timer-changed".to_string(),
                 ));
             }
-            SystemMessage::LocationOnly | SystemMessage::MultiDeviceSync => {
+            SystemMessage::LocationOnly
+            | SystemMessage::MultiDeviceSync
+            | SystemMessage::WebxdcStatusUpdate => {
                 // This should prevent automatic replies,
                 // such as non-delivery reports.
                 //
@@ -1098,7 +1133,7 @@ impl<'a> MimeFactory<'a> {
         }
 
         // add attachment part
-        if chat::msgtype_has_file(self.msg.viewtype) {
+        if self.msg.viewtype.has_file() {
             if !is_file_size_okay(context, self.msg).await? {
                 bail!(
                     "Message exceeds the recommended {} MB.",
@@ -1134,6 +1169,16 @@ impl<'a> MimeFactory<'a> {
             let ids = self.msg.param.get(Param::Arg2).unwrap_or_default();
             parts.push(context.build_sync_part(json.to_string()).await);
             self.sync_ids_to_delete = Some(ids.to_string());
+        } else if command == SystemMessage::WebxdcStatusUpdate {
+            let json = self.msg.param.get(Param::Arg).unwrap_or_default();
+            parts.push(context.build_status_update_part(json).await);
+        } else if self.msg.viewtype == Viewtype::Webxdc {
+            if let Some(json) = context
+                .render_webxdc_status_update_object(self.msg.id, None)
+                .await?
+            {
+                parts.push(context.build_status_update_part(&json).await);
+            }
         }
 
         if self.attach_selfavatar {
@@ -1265,7 +1310,7 @@ async fn build_body_file(
         .param
         .get_blob(Param::File, context, true)
         .await?
-        .ok_or_else(|| format_err!("msg has no filename"))?;
+        .context("msg has no filename")?;
     let suffix = blob.suffix().unwrap_or("dat");
 
     // Get file name to use for sending.  For privacy purposes, we do
@@ -1293,8 +1338,7 @@ async fn build_body_file(
             "video_{}.{}",
             chrono::Utc
                 .timestamp(msg.timestamp_sort, 0)
-                .format("%Y-%m-%d_%H-%M-%S")
-                .to_string(),
+                .format("%Y-%m-%d_%H-%M-%S"),
             &suffix
         ),
         _ => blob.as_file_name().to_string(),
@@ -1401,19 +1445,22 @@ fn maybe_encode_words(words: &str) -> String {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use async_std::fs::File;
     use async_std::prelude::*;
+    use mailparse::{addrparse_header, MailHeaderMap};
 
     use crate::chat::ChatId;
+    use crate::chat::{
+        self, add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg,
+        ProtectionStatus,
+    };
     use crate::chatlist::Chatlist;
     use crate::contact::Origin;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::mimeparser::MimeMessage;
     use crate::test_utils::{get_chat_msg, TestContext};
 
-    use async_std::fs::File;
-    use pretty_assertions::assert_eq;
-
+    use super::*;
     #[test]
     fn test_render_email_address() {
         let display_name = "ä space";
@@ -1510,7 +1557,7 @@ mod tests {
             msg_to_subject_str(
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: Bob <bob@example.com>\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Antw: Chat: hello\n\
                 Message-ID: <2222@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
@@ -1525,7 +1572,7 @@ mod tests {
             msg_to_subject_str(
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: Bob <bob@example.com>\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Infos: 42\n\
                 Message-ID: <2222@example.com>\n\
                 Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
@@ -1544,7 +1591,7 @@ mod tests {
             msg_to_subject_str(
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: bob@example.com\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
                 Message-ID: <2223@example.com>\n\
@@ -1562,7 +1609,7 @@ mod tests {
         // 3. Send the first message to a new contact
         let t = TestContext::new_alice().await;
 
-        assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.org");
 
         let t = TestContext::new_alice().await;
         t.set_config(Config::Displayname, Some("Alice"))
@@ -1577,7 +1624,7 @@ mod tests {
         msg_to_subject_str(
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
             From: bob@example.com\n\
-            To: alice@example.com\n\
+            To: alice@example.org\n\
             Subject: äääää\n\
             Chat-Version: 1.0\n\
             Message-ID: <2893@example.com>\n\
@@ -1591,7 +1638,7 @@ mod tests {
         msg_to_subject_str(
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
             From: bob@example.com\n\
-            To: alice@example.com\n\
+            To: alice@example.org\n\
             Subject: aäääää\n\
             Chat-Version: 1.0\n\
             Message-ID: <2893@example.com>\n\
@@ -1610,7 +1657,7 @@ mod tests {
         dc_receive_imf(
             &t,
             b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-            From: alice@example.com\n\
+            From: alice@example.org\n\
             To: bob@example.com\n\
             Subject: Hello, Bob\n\
             Chat-Version: 1.0\n\
@@ -1618,8 +1665,6 @@ mod tests {
             Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
             \n\
             hello\n",
-            "INBOX",
-            1,
             false,
         )
         .await
@@ -1627,7 +1672,7 @@ mod tests {
         let new_msg = incoming_msg_to_reply_msg(
             b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                  From: bob@example.com\n\
-                 To: alice@example.com\n\
+                 To: alice@example.org\n\
                  Subject: message opened\n\
                  Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
                  Chat-Version: 1.0\n\
@@ -1665,7 +1710,7 @@ mod tests {
             let mut new_msg = Message::new(Viewtype::Text);
             new_msg.set_text(Some("Hi".to_string()));
             if let Some(q) = quote {
-                new_msg.set_quote(t, q).await?;
+                new_msg.set_quote(t, Some(q)).await?;
             }
             let sent = t.send_msg(group_id, &mut new_msg).await;
             get_subject(t, sent).await
@@ -1702,7 +1747,7 @@ mod tests {
             format!(
                 "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: bob@example.com\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Different subject\n\
                 In-Reply-To: {}\n\
                 Message-ID: <2893@example.com>\n\
@@ -1712,8 +1757,6 @@ mod tests {
                 t.get_last_msg().await.rfc724_mid
             )
             .as_bytes(),
-            "INBOX",
-            5,
             false,
         )
         .await?;
@@ -1757,7 +1800,7 @@ mod tests {
         mf.subject_str(&t).await.unwrap()
     }
 
-    // In `imf_raw`, From has to be bob@example.com, To has to be alice@example.com
+    // In `imf_raw`, From has to be bob@example.com, To has to be alice@example.org
     async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
         let subject_str = msg_to_subject_str_inner(imf_raw, false, false, false).await;
 
@@ -1818,14 +1861,12 @@ mod tests {
                 &t,
                 b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                     From: Bob <bob@example.com>\n\
-                    To: alice@example.com\n\
+                    To: alice@example.org\n\
                     Subject: Some other, completely unrelated subject\n\
                     Message-ID: <3cl4@example.com>\n\
                     Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                     \n\
                     Some other, completely unrelated content\n",
-                "INBOX",
-                2,
                 false,
             )
             .await
@@ -1836,7 +1877,7 @@ mod tests {
         }
 
         if reply {
-            new_msg.set_quote(&t, &incoming_msg).await.unwrap();
+            new_msg.set_quote(&t, Some(&incoming_msg)).await.unwrap();
         }
 
         let mf = MimeFactory::from_msg(&t, &new_msg, false).await.unwrap();
@@ -1850,13 +1891,11 @@ mod tests {
             .await
             .unwrap();
 
-        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
-            .await
-            .unwrap();
+        dc_receive_imf(context, imf_raw, false).await.unwrap();
 
         let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
 
-        let chat_id = chats.get_chat_id(0);
+        let chat_id = chats.get_chat_id(0).unwrap();
         chat_id.accept(context).await.unwrap();
 
         let mut new_msg = Message::new(Viewtype::Text);
@@ -1878,7 +1917,7 @@ mod tests {
         let msg = incoming_msg_to_reply_msg(
             b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                 From: Charlie <charlie@example.com>\n\
-                To: alice@example.com\n\
+                To: alice@example.org\n\
                 Subject: Chat: hello\n\
                 Chat-Version: 1.0\n\
                 Message-ID: <2223@example.com>\n\
@@ -1896,7 +1935,7 @@ mod tests {
 
         let rendered_msg = mimefactory.render(context).await.unwrap();
 
-        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
+        let mail = mailparse::parse_mail(rendered_msg.message.as_bytes()).unwrap();
         assert_eq!(
             mail.headers
                 .iter()
@@ -1906,7 +1945,7 @@ mod tests {
             "1.0"
         );
 
-        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
+        let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes())
             .await
             .unwrap();
     }
@@ -1980,8 +2019,9 @@ mod tests {
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("this is the text!".to_string()));
 
-        let payload = t.send_msg(chat.id, &mut msg).await.payload();
-        let mut payload = payload.splitn(3, "\r\n\r\n");
+        let sent_msg = t.send_msg(chat.id, &mut msg).await;
+        let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
+
         let outer = payload.next().unwrap();
         let inner = payload.next().unwrap();
         let body = payload.next().unwrap();
@@ -1999,8 +2039,8 @@ mod tests {
 
         // if another message is sent, that one must not contain the avatar
         // and no artificial multipart/mixed nesting
-        let payload = t.send_msg(chat.id, &mut msg).await.payload();
-        let mut payload = payload.splitn(2, "\r\n\r\n");
+        let sent_msg = t.send_msg(chat.id, &mut msg).await;
+        let mut payload = sent_msg.payload().splitn(2, "\r\n\r\n");
         let outer = payload.next().unwrap();
         let body = payload.next().unwrap();
 
@@ -2017,4 +2057,58 @@ mod tests {
 
         Ok(())
     }
+
+    /// Test that removed member address does not go into the `To:` field.
+    #[async_std::test]
+    async fn test_remove_member_bcc() -> Result<()> {
+        // Alice creates a group with Bob and Claire and then removes Bob.
+        let alice = TestContext::new_alice().await;
+
+        let bob_id = Contact::create(&alice, "Bob", "bob@example.net").await?;
+        let claire_id = Contact::create(&alice, "Claire", "claire@foo.de").await?;
+
+        let alice_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "foo").await?;
+        add_contact_to_chat(&alice, alice_chat_id, bob_id).await?;
+        add_contact_to_chat(&alice, alice_chat_id, claire_id).await?;
+        send_text_msg(&alice, alice_chat_id, "Creating a group".to_string()).await?;
+
+        remove_contact_from_chat(&alice, alice_chat_id, claire_id).await?;
+        let remove = alice.pop_sent_msg().await;
+        let remove_payload = remove.payload();
+        let parsed = mailparse::parse_mail(remove_payload.as_bytes())?;
+        let to = parsed
+            .headers
+            .get_first_header("To")
+            .context("no To: header parsed")?;
+        let to = addrparse_header(to)?;
+        let mailbox = to
+            .extract_single_info()
+            .context("to: field does not contain exactly one address")?;
+        assert_eq!(mailbox.addr, "bob@example.net");
+
+        Ok(())
+    }
+
+    /// Tests that standard IMF header "From:" comes before non-standard "Autocrypt:" header.
+    #[async_std::test]
+    async fn test_from_before_autocrypt() -> Result<()> {
+        // create chat with bob
+        let t = TestContext::new_alice().await;
+        let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
+
+        // send message to bob: that should get multipart/mixed because of the avatar moved to inner header;
+        // make sure, `Subject:` stays in the outer header (imf header)
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("this is the text!".to_string()));
+
+        let sent_msg = t.send_msg(chat.id, &mut msg).await;
+        let payload = sent_msg.payload();
+
+        assert_eq!(payload.match_indices("Autocrypt:").count(), 1);
+        assert_eq!(payload.match_indices("From:").count(), 1);
+
+        assert!(payload.match_indices("From:").next() < payload.match_indices("Autocrypt:").next());
+
+        Ok(())
+    }
 }

src/mimeparser.rs

@@ -2,6 +2,7 @@
 
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
+use std::io::Cursor;
 use std::pin::Pin;
 
 use anyhow::{bail, Result};
@@ -12,10 +13,10 @@ use once_cell::sync::Lazy;
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
-use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
-use crate::contact::addr_normalize;
+use crate::constants::{DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
+use crate::contact::{addr_normalize, ContactId};
 use crate::context::Context;
-use crate::dc_tools::{dc_get_filemeta, dc_truncate};
+use crate::dc_tools::{dc_get_filemeta, dc_truncate, parse_receive_headers};
 use crate::dehtml::dehtml;
 use crate::e2ee;
 use crate::events::EventType;
@@ -23,7 +24,7 @@ use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
 use crate::location;
-use crate::message;
+use crate::message::{self, Viewtype};
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::simplify::simplify;
@@ -47,6 +48,7 @@ pub struct MimeMessage {
     /// Addresses are normalized and lowercased:
     pub recipients: Vec<SingleInfo>,
     pub from: Vec<SingleInfo>,
+    pub list_post: Option<String>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
 
@@ -65,6 +67,7 @@ pub struct MimeMessage {
     pub location_kml: Option<location::Kml>,
     pub message_kml: Option<location::Kml>,
     pub(crate) sync_items: Option<SyncItems>,
+    pub(crate) webxdc_status_update: Option<String>,
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
     pub(crate) mdn_reports: Vec<Report>,
@@ -82,6 +85,8 @@ pub struct MimeMessage {
     /// This is non-empty only if the message was actually encrypted.  It is used
     /// for e.g. late-parsing HTML.
     pub decoded_data: Vec<u8>,
+
+    pub(crate) hop_info: String,
 }
 
 #[derive(Debug, PartialEq)]
@@ -132,6 +137,10 @@ pub enum SystemMessage {
     /// Self-sent-message that contains only json used for multi-device-sync;
     /// if possible, we attach that to other messages as for locations.
     MultiDeviceSync = 20,
+
+    // Sync message that contains a json payload
+    // sent to the other webxdc instances
+    WebxdcStatusUpdate = 30,
 }
 
 impl Default for SystemMessage {
@@ -163,10 +172,12 @@ impl MimeMessage {
             .get_header_value(HeaderDef::Date)
             .and_then(|v| mailparse::dateparse(&v).ok())
             .unwrap_or_default();
+        let hop_info = parse_receive_headers(&mail.get_headers());
 
         let mut headers = Default::default();
         let mut recipients = Default::default();
         let mut from = Default::default();
+        let mut list_post = Default::default();
         let mut chat_disposition_notification_to = None;
 
         // Parse IMF headers.
@@ -175,6 +186,7 @@ impl MimeMessage {
             &mut headers,
             &mut recipients,
             &mut from,
+            &mut list_post,
             &mut chat_disposition_notification_to,
             &mail.headers,
         );
@@ -248,6 +260,7 @@ impl MimeMessage {
                             &mut headers,
                             &mut recipients,
                             &mut throwaway_from,
+                            &mut list_post,
                             &mut chat_disposition_notification_to,
                             &decrypted_mail.headers,
                         );
@@ -275,6 +288,7 @@ impl MimeMessage {
             parts: Vec::new(),
             header: headers,
             recipients,
+            list_post,
             from,
             chat_disposition_notification_to,
             decrypting_failed: false,
@@ -288,12 +302,14 @@ impl MimeMessage {
             location_kml: None,
             message_kml: None,
             sync_items: None,
+            webxdc_status_update: None,
             user_avatar: None,
             group_avatar: None,
             failure_report: None,
             footer: None,
             is_mime_modified: false,
             decoded_data: Vec::new(),
+            hop_info,
         };
 
         match partial {
@@ -353,6 +369,16 @@ impl MimeMessage {
             } else if value == "protection-disabled" {
                 self.is_system_message = SystemMessage::ChatProtectionDisabled;
             }
+        } else if self.get_header(HeaderDef::ChatGroupMemberRemoved).is_some() {
+            self.is_system_message = SystemMessage::MemberRemovedFromGroup;
+        } else if self.get_header(HeaderDef::ChatGroupMemberAdded).is_some() {
+            self.is_system_message = SystemMessage::MemberAddedToGroup;
+        } else if self.get_header(HeaderDef::ChatGroupNameChanged).is_some() {
+            self.is_system_message = SystemMessage::GroupNameChanged;
+        } else if let Some(value) = self.get_header(HeaderDef::ChatContent) {
+            if value == "group-avatar-changed" {
+                self.is_system_message = SystemMessage::GroupImageChanged;
+            }
         }
     }
 
@@ -388,16 +414,18 @@ impl MimeMessage {
     #[allow(clippy::indexing_slicing)]
     fn squash_attachment_parts(&mut self) {
         if let [textpart, filepart] = &self.parts[..] {
-            let need_drop = {
-                textpart.typ == Viewtype::Text
-                    && (filepart.typ == Viewtype::Image
-                        || filepart.typ == Viewtype::Gif
-                        || filepart.typ == Viewtype::Sticker
-                        || filepart.typ == Viewtype::Audio
-                        || filepart.typ == Viewtype::Voice
-                        || filepart.typ == Viewtype::Video
-                        || filepart.typ == Viewtype::File)
-            };
+            let need_drop = textpart.typ == Viewtype::Text
+                && match filepart.typ {
+                    Viewtype::Image
+                    | Viewtype::Gif
+                    | Viewtype::Sticker
+                    | Viewtype::Audio
+                    | Viewtype::Voice
+                    | Viewtype::Video
+                    | Viewtype::File
+                    | Viewtype::Webxdc => true,
+                    Viewtype::Unknown | Viewtype::Text | Viewtype::VideochatInvitation => false,
+                };
 
             if need_drop {
                 let mut filepart = self.parts.swap_remove(1);
@@ -529,7 +557,7 @@ impl MimeMessage {
             };
 
             if let Some(ref subject) = self.get_subject() {
-                if !self.has_chat_version() {
+                if !self.has_chat_version() && self.webxdc_status_update.is_none() {
                     part.msg = subject.to_string();
                 }
             }
@@ -828,6 +856,12 @@ impl MimeMessage {
                                     .await?;
                             }
                         }
+                        Some("status-update") => {
+                            if let Some(second) = mail.subparts.get(1) {
+                                self.add_single_part_if_known(context, second, is_related)
+                                    .await?;
+                            }
+                        }
                         Some(_) => {
                             if let Some(first) = mail.subparts.get(0) {
                                 any_part_added = self
@@ -997,8 +1031,14 @@ impl MimeMessage {
         if decoded_data.is_empty() {
             return;
         }
-        // treat location/message kml file attachments specially
-        if filename.ends_with(".kml") {
+        let reader = Cursor::new(decoded_data);
+        let msg_type = if context
+            .is_webxdc_file(filename, reader)
+            .await
+            .unwrap_or(false)
+        {
+            Viewtype::Webxdc
+        } else if filename.ends_with(".kml") {
             // XXX what if somebody sends eg an "location-highlights.kml"
             // attachment unrelated to location streaming?
             if filename.starts_with("location") || filename.starts_with("message") {
@@ -1014,6 +1054,7 @@ impl MimeMessage {
                 }
                 return;
             }
+            msg_type
         } else if filename == "multi-device-sync.json" {
             let serialized = String::from_utf8_lossy(decoded_data)
                 .parse()
@@ -1026,7 +1067,15 @@ impl MimeMessage {
                 })
                 .ok();
             return;
-        }
+        } else if filename == "status-update.json" {
+            let serialized = String::from_utf8_lossy(decoded_data)
+                .parse()
+                .unwrap_or_default();
+            self.webxdc_status_update = Some(serialized);
+            return;
+        } else {
+            msg_type
+        };
 
         /* we have a regular file attachment,
         write decoded data to new blob object */
@@ -1092,16 +1141,16 @@ impl MimeMessage {
         }
     }
 
-    pub fn repl_msg_by_error(&mut self, error_msg: impl AsRef<str>) {
+    pub fn repl_msg_by_error(&mut self, error_msg: &str) {
         self.is_system_message = SystemMessage::Unknown;
         if let Some(part) = self.parts.first_mut() {
             part.typ = Viewtype::Text;
-            part.msg = format!("[{}]", error_msg.as_ref());
+            part.msg = format!("[{}]", error_msg);
             self.parts.truncate(1);
         }
     }
 
-    pub fn get_rfc724_mid(&self) -> Option<String> {
+    pub(crate) fn get_rfc724_mid(&self) -> Option<String> {
         self.get_header(HeaderDef::XMicrosoftOriginalMessageId)
             .or_else(|| self.get_header(HeaderDef::MessageId))
             .and_then(|msgid| parse_message_id(msgid).ok())
@@ -1112,6 +1161,7 @@ impl MimeMessage {
         headers: &mut HashMap<String, String>,
         recipients: &mut Vec<SingleInfo>,
         from: &mut Vec<SingleInfo>,
+        list_post: &mut Option<String>,
         chat_disposition_notification_to: &mut Option<SingleInfo>,
         fields: &[mailparse::MailHeader<'_>],
     ) {
@@ -1142,6 +1192,10 @@ impl MimeMessage {
         if !from_new.is_empty() {
             *from = from_new;
         }
+        let list_post_new = get_list_post(fields);
+        if list_post_new.is_some() {
+            *list_post = list_post_new;
+        }
     }
 
     fn process_report(
@@ -1159,23 +1213,24 @@ impl MimeMessage {
 
         // must be present
         if let Some(_disposition) = report_fields.get_header_value(HeaderDef::Disposition) {
-            if let Some(original_message_id) = report_fields
+            let original_message_id = report_fields
                 .get_header_value(HeaderDef::OriginalMessageId)
-                .and_then(|v| parse_message_id(&v).ok())
-            {
-                let additional_message_ids = report_fields
-                    .get_header_value(HeaderDef::AdditionalMessageIds)
-                    .map_or_else(Vec::new, |v| {
-                        v.split(' ')
-                            .filter_map(|s| parse_message_id(s).ok())
-                            .collect()
-                    });
+                // MS Exchange doesn't add an Original-Message-Id header. Instead, they put
+                // the original message id into the In-Reply-To header:
+                .or_else(|| report.headers.get_header_value(HeaderDef::InReplyTo))
+                .and_then(|v| parse_message_id(&v).ok());
+            let additional_message_ids = report_fields
+                .get_header_value(HeaderDef::AdditionalMessageIds)
+                .map_or_else(Vec::new, |v| {
+                    v.split(' ')
+                        .filter_map(|s| parse_message_id(s).ok())
+                        .collect()
+                });
 
-                return Ok(Some(Report {
-                    original_message_id,
-                    additional_message_ids,
-                }));
-            }
+            return Ok(Some(Report {
+                original_message_id,
+                additional_message_ids,
+            }));
         }
         warn!(
             context,
@@ -1326,13 +1381,15 @@ impl MimeMessage {
     pub async fn handle_reports(
         &self,
         context: &Context,
-        from_id: u32,
+        from_id: ContactId,
         sent_timestamp: i64,
         parts: &[Part],
     ) {
         for report in &self.mdn_reports {
-            for original_message_id in
-                std::iter::once(&report.original_message_id).chain(&report.additional_message_ids)
+            for original_message_id in report
+                .original_message_id
+                .iter()
+                .chain(&report.additional_message_ids)
             {
                 match message::handle_mdn(context, from_id, original_message_id, sent_timestamp)
                     .await
@@ -1437,7 +1494,10 @@ async fn update_gossip_peerstates(
 #[derive(Debug)]
 pub(crate) struct Report {
     /// Original-Message-ID header
-    original_message_id: String,
+    ///
+    /// It MUST be present if the original message has a Message-ID according to RFC 8098.
+    /// In case we can't find it (shouldn't happen), this is None.
+    original_message_id: Option<String>,
     /// Additional-Message-IDs
     additional_message_ids: Vec<String>,
 }
@@ -1627,6 +1687,14 @@ pub(crate) fn get_from(headers: &[MailHeader]) -> Vec<SingleInfo> {
     get_all_addresses_from_header(headers, |header_key| header_key == "from")
 }
 
+/// Returned addresses are normalized and lowercased.
+pub(crate) fn get_list_post(headers: &[MailHeader]) -> Option<String> {
+    get_all_addresses_from_header(headers, |header_key| header_key == "list-post")
+        .into_iter()
+        .next()
+        .map(|s| s.addr)
+}
+
 fn get_all_addresses_from_header<F>(headers: &[MailHeader], pred: F) -> Vec<SingleInfo>
 where
     F: Fn(String) -> bool,
@@ -2346,7 +2414,7 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
         assert_eq!(message.mdn_reports.len(), 1);
         assert_eq!(
             message.mdn_reports[0].original_message_id,
-            "foo@example.org"
+            Some("foo@example.org".to_string())
         );
         assert_eq!(
             &message.mdn_reports[0].additional_message_ids,
@@ -2847,8 +2915,6 @@ On 2020-10-25, Bob wrote:
         dc_receive_imf(
             &t.ctx,
             include_bytes!("../test-data/message/subj_with_multimedia_msg.eml"),
-            "INBOX",
-            1,
             false,
         )
         .await
@@ -2997,7 +3063,7 @@ Subject: ...
 
 Some quote.
 "###;
-        dc_receive_imf(&t, raw, "INBOX", 1, false).await?;
+        dc_receive_imf(&t, raw, false).await?;
 
         // Delta Chat generates In-Reply-To with a starting tab when Message-ID is too long.
         let raw = br###"In-Reply-To:
@@ -3014,7 +3080,7 @@ Subject: ...
 Some reply
 "###;
 
-        dc_receive_imf(&t, raw, "INBOX", 2, false).await?;
+        dc_receive_imf(&t, raw, false).await?;
 
         let msg = t.get_last_msg().await;
         assert_eq!(msg.get_text().unwrap(), "Some reply");
@@ -3034,21 +3100,21 @@ Some reply
 Chat-Version: 1.0\n\
 Message-ID: <foobarbaz@example.org>
 To: Bob <bob@example.org>
-From: Alice <alice@example.com>
+From: Alice <alice@example.org>
 Subject: subject
-Chat-Disposition-Notification-To: alice@example.com
+Chat-Disposition-Notification-To: alice@example.org
 
 Message.
 "###;
 
         // Bob receives message.
-        dc_receive_imf(&bob, raw, "INBOX", 1, false).await?;
+        dc_receive_imf(&bob, raw, false).await?;
         let msg = bob.get_last_msg().await;
         // Message is incoming.
         assert!(msg.param.get_bool(Param::WantsMdn).unwrap());
 
         // Alice receives copy-to-self.
-        dc_receive_imf(&alice, raw, "INBOX", 1, false).await?;
+        dc_receive_imf(&alice, raw, false).await?;
         let msg = alice.get_last_msg().await;
         // Message is outgoing, don't send read receipt to self.
         assert!(msg.param.get_bool(Param::WantsMdn).is_none());
@@ -3064,18 +3130,16 @@ Message.
         dc_receive_imf(
             &alice,
             "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-                 From: alice@example.com\n\
+                 From: alice@example.org\n\
                  To: bob@example.net\n\
                  Subject: foo\n\
                  Message-ID: first@example.com\n\
                  Chat-Version: 1.0\n\
-                 Chat-Disposition-Notification-To: alice@example.com\n\
+                 Chat-Disposition-Notification-To: alice@example.org\n\
                  Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
                  \n\
                  hello\n"
                 .as_bytes(),
-            "INBOX",
-            1,
             false,
         )
         .await?;
@@ -3087,8 +3151,8 @@ Message.
         dc_receive_imf(
             &alice,
                 "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-                 From: alice@example.com\n\
-                 To: alice@example.com\n\
+                 From: alice@example.org\n\
+                 To: alice@example.org\n\
                  Subject: message opened\n\
                  Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
                  Chat-Version: 1.0\n\
@@ -3113,8 +3177,6 @@ Message.
                  \n\
                  --SNIPP--"
             .as_bytes(),
-            "INBOX",
-            2,
             false,
         )
         .await?;
@@ -3125,4 +3187,40 @@ Message.
 
         Ok(())
     }
+
+    /// Test parsing of MDN sent by MS Exchange.
+    ///
+    /// It does not have required Original-Message-ID field, so it is useless, but we want to
+    /// recognize it as MDN nevertheless to avoid displaying it in the chat as normal message.
+    #[async_std::test]
+    async fn test_ms_exchange_mdn() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        t.set_config(Config::ShowEmails, Some("2")).await?;
+
+        let original =
+            include_bytes!("../test-data/message/ms_exchange_report_original_message.eml");
+        dc_receive_imf(&t, original, false).await?;
+        let original_msg_id = t.get_last_msg().await.id;
+
+        // 1. Test mimeparser directly
+        let mdn =
+            include_bytes!("../test-data/message/ms_exchange_report_disposition_notification.eml");
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn).await?;
+        assert_eq!(mimeparser.mdn_reports.len(), 1);
+        assert_eq!(
+            mimeparser.mdn_reports[0].original_message_id.as_deref(),
+            Some("d5904dc344eeb5deaf9bb44603f0c716@posteo.de")
+        );
+        assert!(mimeparser.mdn_reports[0].additional_message_ids.is_empty());
+
+        // 2. Test that marking the original msg as read works
+        dc_receive_imf(&t, mdn, false).await?;
+
+        assert_eq!(
+            original_msg_id.get_state(&t).await?,
+            MessageState::OutMdnRcvd
+        );
+
+        Ok(())
+    }
 }

src/oauth2.rs

@@ -41,6 +41,7 @@ struct Oauth2 {
 
 /// OAuth 2 Access Token Response
 #[derive(Debug, Deserialize)]
+#[allow(dead_code)]
 struct Response {
     // Should always be there according to: <https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/>
     // but previous code handled its abscense.
@@ -58,7 +59,7 @@ pub async fn dc_get_oauth2_url(
     redirect_uri: &str,
 ) -> Result<Option<String>> {
     let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
-    if let Some(oauth2) = Oauth2::from_address(addr, socks5_enabled).await {
+    if let Some(oauth2) = Oauth2::from_address(context, addr, socks5_enabled).await {
         context
             .sql
             .set_raw_config("oauth2_pending_redirect_uri", Some(redirect_uri))
@@ -79,7 +80,7 @@ pub async fn dc_get_oauth2_access_token(
     regenerate: bool,
 ) -> Result<Option<String>> {
     let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
-    if let Some(oauth2) = Oauth2::from_address(addr, socks5_enabled).await {
+    if let Some(oauth2) = Oauth2::from_address(context, addr, socks5_enabled).await {
         let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
@@ -225,7 +226,7 @@ pub async fn dc_get_oauth2_addr(
     code: &str,
 ) -> Result<Option<String>> {
     let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
-    let oauth2 = match Oauth2::from_address(addr, socks5_enabled).await {
+    let oauth2 = match Oauth2::from_address(context, addr, socks5_enabled).await {
         Some(o) => o,
         None => return Ok(None),
     };
@@ -253,13 +254,13 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    async fn from_address(addr: &str, skip_mx: bool) -> Option<Self> {
+    async fn from_address(context: &Context, addr: &str, skip_mx: bool) -> Option<Self> {
         let addr_normalized = normalize_addr(addr);
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            if let Some(oauth2_authorizer) = provider::get_provider_info(domain, skip_mx)
+            if let Some(oauth2_authorizer) = provider::get_provider_info(context, domain, skip_mx)
                 .await
                 .and_then(|provider| provider.oauth2_authorizer.as_ref())
             {
@@ -356,32 +357,39 @@ mod tests {
 
     #[async_std::test]
     async fn test_oauth_from_address() {
+        let t = TestContext::new().await;
         assert_eq!(
-            Oauth2::from_address("hello@gmail.com", false).await,
+            Oauth2::from_address(&t, "hello@gmail.com", false).await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com", false).await,
+            Oauth2::from_address(&t, "hello@googlemail.com", false).await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com", false).await,
+            Oauth2::from_address(&t, "hello@yandex.com", false).await,
             Some(OAUTH2_YANDEX)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.ru", false).await,
+            Oauth2::from_address(&t, "hello@yandex.ru", false).await,
             Some(OAUTH2_YANDEX)
         );
-
-        assert_eq!(Oauth2::from_address("hello@web.de", false).await, None);
+        assert_eq!(Oauth2::from_address(&t, "hello@web.de", false).await, None);
     }
 
     #[async_std::test]
     async fn test_oauth_from_mx() {
+        // youtube staff seems to use "google workspace with oauth2", figures this out by MX lookup
+        let t = TestContext::new().await;
         assert_eq!(
-            Oauth2::from_address("hello@google.com", false).await,
+            Oauth2::from_address(&t, "hello@youtube.com", false).await,
             Some(OAUTH2_GMAIL)
         );
+        // without MX lookup, we would not know as youtube.com is not in our provider-db
+        assert_eq!(
+            Oauth2::from_address(&t, "hello@youtube.com", true).await,
+            None
+        );
     }
 
     #[async_std::test]

src/param.rs

@@ -4,7 +4,6 @@ use std::str;
 
 use anyhow::{bail, Error};
 use async_std::path::PathBuf;
-use itertools::Itertools;
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 
@@ -111,8 +110,8 @@ pub enum Param {
     /// For Jobs
     AlsoMove = b'M',
 
-    /// For Jobs: space-separated list of message recipients
-    Recipients = b'R',
+    /// For MDN-sending job
+    MsgId = b'I',
 
     /// For Groups
     ///
@@ -137,8 +136,17 @@ pub enum Param {
     /// For Chats
     Devicetalk = b'D',
 
-    /// For MDN-sending job
-    MsgId = b'I',
+    /// For Chats: If this is a mailing list chat, contains the List-Post address.
+    /// None if there simply is no `List-Post` header in the mailing list.
+    /// Some("") if the mailing list is using multiple different List-Post headers.
+    ///
+    /// The List-Post address is the email address where the user can write to in order to
+    /// post something to the mailing list.
+    ListPost = b'p',
+
+    /// For Contacts: If this is the List-Post address of a mailing list, contains
+    /// the List-Id of the mailing list (which is also used as the group id of the chat).
+    ListId = b's',
 
     /// For Contacts: timestamp of status (aka signature or footer) update.
     StatusTimestamp = b'j',
@@ -160,6 +168,12 @@ pub enum Param {
 
     /// For Chats: timestamp of protection settings update.
     ProtectionSettingsTimestamp = b'L',
+
+    /// For Webxdc Message Instances: Current summary
+    WebxdcSummary = b'N',
+
+    /// For Webxdc Message Instances: timestamp of summary update.
+    WebxdcSummaryTimestamp = b'Q',
 }
 
 /// An object for handling key=value parameter lists.
@@ -182,7 +196,7 @@ impl fmt::Display for Params {
                 f,
                 "{}={}",
                 *key as u8 as char,
-                value.split('\n').join("\n\n")
+                value.split('\n').collect::<Vec<&str>>().join("\n\n")
             )?;
         }
         Ok(())
@@ -192,6 +206,11 @@ impl fmt::Display for Params {
 impl str::FromStr for Params {
     type Err = Error;
 
+    /// Parse a raw string to Param.
+    ///
+    /// Silently ignore unknown keys:
+    /// they may come from a downgrade (when a shortly new version adds a key)
+    /// or from an upgrade (when a key is dropped but was used in the past)
     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
         let mut inner = BTreeMap::new();
         let mut lines = s.lines().peekable();
@@ -211,8 +230,6 @@ impl str::FromStr for Params {
 
                 if let Some(key) = key.as_bytes().first().and_then(|key| Param::from_u8(*key)) {
                     inner.insert(key, value);
-                } else {
-                    bail!("Unknown key: {}", key);
                 }
             } else {
                 bail!("Not a key-value pair: {:?}", line);
@@ -415,10 +432,12 @@ impl<'a> ParamsFile<'a> {
 mod tests {
     use super::*;
 
+    use anyhow::Result;
     use async_std::fs;
     use async_std::path::Path;
 
     use crate::test_utils::TestContext;
+    use std::str::FromStr;
 
     #[test]
     fn test_dc_param() {
@@ -521,4 +540,14 @@ mod tests {
         assert!(p.get_path(Param::File, &t).unwrap().is_none());
         assert!(p.get_blob(Param::File, &t, false).await.unwrap().is_none());
     }
+
+    #[async_std::test]
+    async fn test_params_unknown_key() -> Result<()> {
+        // 'Z' is used as a key that is known to be unused; these keys should be ignored silently by definition.
+        let p = Params::from_str("w=12\nZ=13\nh=14")?;
+        assert_eq!(p.len(), 2);
+        assert_eq!(p.get(Param::Width), Some("12"));
+        assert_eq!(p.get(Param::Height), Some("14"));
+        Ok(())
+    }
 }

src/peerstate.rs

@@ -4,11 +4,13 @@ use std::collections::HashSet;
 use std::fmt;
 
 use crate::aheader::{Aheader, EncryptPreference};
-use crate::chat::{self, ChatIdBlocked};
-use crate::constants::Blocked;
+use crate::chat::{self};
+use crate::chatlist::Chatlist;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
+use crate::message::Message;
+use crate::mimeparser::SystemMessage;
 use crate::sql::Sql;
 use crate::stock_str;
 use anyhow::{bail, Result};
@@ -271,14 +273,34 @@ impl Peerstate {
                 .query_get_value("SELECT id FROM contacts WHERE addr=?;", paramsv![self.addr])
                 .await?
             {
-                let chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Request)
-                    .await?
-                    .id;
-
+                let chats = Chatlist::try_load(context, 0, None, contact_id).await?;
                 let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
-
-                chat::add_info_msg(context, chat_id, msg, timestamp).await?;
-                context.emit_event(EventType::ChatModified(chat_id));
+                for (chat_id, msg_id) in chats.iter() {
+                    let timestamp_sort = if let Some(msg_id) = msg_id {
+                        let lastmsg = Message::load_from_db(context, *msg_id).await?;
+                        lastmsg.timestamp_sort
+                    } else {
+                        context
+                            .sql
+                            .query_get_value(
+                                "SELECT created_timestamp FROM chats WHERE id=?;",
+                                paramsv![chat_id],
+                            )
+                            .await?
+                            .unwrap_or(0)
+                    };
+                    chat::add_info_msg_with_cmd(
+                        context,
+                        *chat_id,
+                        &msg,
+                        SystemMessage::Unknown,
+                        timestamp_sort,
+                        Some(timestamp),
+                        None,
+                    )
+                    .await?;
+                    context.emit_event(EventType::ChatModified(*chat_id));
+                }
             } else {
                 bail!("contact with peerstate.addr {:?} not found", &self.addr);
             }
@@ -379,10 +401,9 @@ impl Peerstate {
     pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&SignedPublicKey> {
         match min_verified {
             PeerstateVerifiedStatus::BidirectVerified => self.verified_key.as_ref(),
-            PeerstateVerifiedStatus::Unverified => self
-                .public_key
-                .as_ref()
-                .or_else(|| self.gossip_key.as_ref()),
+            PeerstateVerifiedStatus::Unverified => {
+                self.public_key.as_ref().or(self.gossip_key.as_ref())
+            }
         }
     }
 
@@ -524,7 +545,6 @@ pub(crate) async fn deduplicate_peerstates(sql: &Sql) -> Result<()> {
 mod tests {
     use super::*;
     use crate::test_utils::alice_keypair;
-    use pretty_assertions::assert_eq;
 
     #[async_std::test]
     async fn test_peerstate_save_to_db() {

src/pgp.rs

@@ -4,11 +4,11 @@ use std::collections::{BTreeMap, HashSet};
 use std::io;
 use std::io::Cursor;
 
-use anyhow::{bail, ensure, format_err, Result};
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
-    SignedPublicSubKey, SignedSecretKey, SubkeyParamsBuilder,
+    SignedPublicSubKey, SignedSecretKey, StandaloneSignature, SubkeyParamsBuilder,
 };
 use pgp::crypto::{HashAlgorithm, SymmetricKeyAlgorithm};
 use pgp::types::{
@@ -248,7 +248,7 @@ pub async fn pk_encrypt(
         let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
             .keys()
             .iter()
-            .filter_map(|key| select_pk_for_encryption(key))
+            .filter_map(select_pk_for_encryption)
             .collect();
         let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
@@ -277,16 +277,17 @@ pub async fn pk_encrypt(
 /// Receiver private keys are provided in
 /// `private_keys_for_decryption`.
 ///
-/// If `ret_signature_fingerprints` is not `None`, stores fingerprints
+/// Returns decrypted message and fingerprints
 /// of all keys from the `public_keys_for_validation` keyring that
 /// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
 pub async fn pk_decrypt(
     ctext: Vec<u8>,
     private_keys_for_decryption: Keyring<SignedSecretKey>,
-    public_keys_for_validation: Keyring<SignedPublicKey>,
-    ret_signature_fingerprints: Option<&mut HashSet<Fingerprint>>,
-) -> Result<Vec<u8>> {
+    public_keys_for_validation: &Keyring<SignedPublicKey>,
+) -> Result<(Vec<u8>, HashSet<Fingerprint>)> {
+    let mut ret_signature_fingerprints: HashSet<Fingerprint> = Default::default();
+
     let msgs = async_std::task::spawn_blocking(move || {
         let cursor = Cursor::new(ctext);
         let (msg, _) = Message::from_armor_single(cursor)?;
@@ -308,33 +309,54 @@ pub async fn pk_decrypt(
             None => bail!("The decrypted message is empty"),
         };
 
-        if let Some(ret_signature_fingerprints) = ret_signature_fingerprints {
-            if !public_keys_for_validation.is_empty() {
-                let fingerprints = async_std::task::spawn_blocking(move || {
-                    let pkeys = public_keys_for_validation.keys();
+        if !public_keys_for_validation.is_empty() {
+            let pkeys = public_keys_for_validation.keys();
 
-                    let mut fingerprints: Vec<Fingerprint> = Vec::new();
-                    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
-                        for pkey in pkeys {
-                            if signed_msg.verify(&pkey.primary_key).is_ok() {
-                                let fp = DcKey::fingerprint(pkey);
-                                fingerprints.push(fp);
-                            }
-                        }
+            let mut fingerprints: Vec<Fingerprint> = Vec::new();
+            if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+                for pkey in pkeys {
+                    if signed_msg.verify(&pkey.primary_key).is_ok() {
+                        let fp = DcKey::fingerprint(pkey);
+                        fingerprints.push(fp);
                     }
-                    fingerprints
-                })
-                .await;
-
-                ret_signature_fingerprints.extend(fingerprints);
+                }
             }
+
+            ret_signature_fingerprints.extend(fingerprints);
         }
-        Ok(content)
+        Ok((content, ret_signature_fingerprints))
     } else {
         bail!("No valid messages found");
     }
 }
 
+/// Validates detached signature.
+pub async fn pk_validate(
+    content: &[u8],
+    signature: &[u8],
+    public_keys_for_validation: &Keyring<SignedPublicKey>,
+) -> Result<HashSet<Fingerprint>> {
+    let mut ret: HashSet<Fingerprint> = Default::default();
+
+    let standalone_signature = StandaloneSignature::from_armor_single(Cursor::new(signature))?.0;
+    let pkeys = public_keys_for_validation.keys();
+
+    // Remove trailing CRLF before the delimiter.
+    // According to RFC 3156 it is considered to be part of the MIME delimiter for the purpose of
+    // OpenPGP signature calculation.
+    let content = content
+        .get(..content.len().saturating_sub(2))
+        .context("index is out of range")?;
+
+    for pkey in pkeys {
+        if standalone_signature.verify(pkey, content).is_ok() {
+            let fp = DcKey::fingerprint(pkey);
+            ret.insert(fp);
+        }
+    }
+    Ok(ret)
+}
+
 /// Symmetric encryption.
 pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
@@ -492,15 +514,12 @@ mod tests {
         decrypt_keyring.add(KEYS.alice_secret.clone());
         let mut sig_check_keyring: Keyring<SignedPublicKey> = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
+        let (plain, valid_signatures) = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            sig_check_keyring,
-            Some(&mut valid_signatures),
+            &sig_check_keyring,
         )
         .await
-        .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
@@ -510,15 +529,12 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.alice_public.clone());
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
+        let (plain, valid_signatures) = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            sig_check_keyring,
-            Some(&mut valid_signatures),
+            &sig_check_keyring,
         )
         .await
-        .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
@@ -529,15 +545,10 @@ mod tests {
         let mut keyring = Keyring::new();
         keyring.add(KEYS.alice_secret.clone());
         let empty_keyring = Keyring::new();
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes().to_vec(),
-            keyring,
-            empty_keyring,
-            Some(&mut valid_signatures),
-        )
-        .await
-        .unwrap();
+        let (plain, valid_signatures) =
+            pk_decrypt(CTEXT_SIGNED.as_bytes().to_vec(), keyring, &empty_keyring)
+                .await
+                .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
@@ -549,12 +560,10 @@ mod tests {
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let mut sig_check_keyring = Keyring::new();
         sig_check_keyring.add(KEYS.bob_public.clone());
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
+        let (plain, valid_signatures) = pk_decrypt(
             CTEXT_SIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            sig_check_keyring,
-            Some(&mut valid_signatures),
+            &sig_check_keyring,
         )
         .await
         .unwrap();
@@ -567,34 +576,14 @@ mod tests {
         let mut decrypt_keyring = Keyring::new();
         decrypt_keyring.add(KEYS.bob_secret.clone());
         let sig_check_keyring = Keyring::new();
-        let mut valid_signatures: HashSet<Fingerprint> = Default::default();
-        let plain = pk_decrypt(
+        let (plain, valid_signatures) = pk_decrypt(
             CTEXT_UNSIGNED.as_bytes().to_vec(),
             decrypt_keyring,
-            sig_check_keyring,
-            Some(&mut valid_signatures),
+            &sig_check_keyring,
         )
         .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
     }
-
-    #[async_std::test]
-    async fn test_decrypt_signed_no_sigret() {
-        // Check decrypting signed cyphertext without providing the HashSet for signatures.
-        let mut decrypt_keyring = Keyring::new();
-        decrypt_keyring.add(KEYS.bob_secret.clone());
-        let mut sig_check_keyring = Keyring::new();
-        sig_check_keyring.add(KEYS.alice_public.clone());
-        let plain = pk_decrypt(
-            CTEXT_SIGNED.as_bytes().to_vec(),
-            decrypt_keyring,
-            sig_check_keyring,
-            None,
-        )
-        .await
-        .unwrap();
-        assert_eq!(plain, CLEARTEXT);
-    }
 }

src/plaintext.rs

@@ -41,7 +41,7 @@ impl PlainText {
             // as <http://example.org> cannot be handled correctly
             // (they would become &lt;http://example.org&gt; where the trailing &gt; would become a valid url part).
             // to avoid double encoding, we escape our html-entities by \r that must not be used in the string elsewhere.
-            let line = line.to_string().replace("\r", "");
+            let line = line.to_string().replace('\r', "");
 
             let mut line = LINKIFY_MAIL_RE
                 .replace_all(&*line, "\rLTa href=\rQUOTmailto:$1\rQUOT\rGT$1\rLT/a\rGT")

src/provider.rs

@@ -3,8 +3,10 @@
 mod data;
 
 use crate::config::Config;
+use crate::context::Context;
 use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED};
-use async_std_resolver::resolver_from_system_conf;
+use anyhow::Result;
+use async_std_resolver::{config, resolver, resolver_from_system_conf, AsyncStdResolver};
 use chrono::{NaiveDateTime, NaiveTime};
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
@@ -81,6 +83,23 @@ pub struct Provider {
     pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
+/// Get resolver to query MX records.
+///
+/// We first try resolver_from_system_conf() which reads the system's resolver from `/etc/resolv.conf`.
+/// This does not work at least on some Androids, therefore we use use ResolverConfig::default()
+/// which default eg. to google's 8.8.8.8 or 8.8.4.4 as a fallback.
+async fn get_resolver() -> Result<AsyncStdResolver> {
+    if let Ok(resolver) = resolver_from_system_conf().await {
+        return Ok(resolver);
+    }
+    let resolver = resolver(
+        config::ResolverConfig::default(),
+        config::ResolverOpts::default(),
+    )
+    .await?;
+    Ok(resolver)
+}
+
 /// Returns provider for the given domain.
 ///
 /// This function looks up domain in offline database first. If not
@@ -89,15 +108,19 @@ pub struct Provider {
 ///
 /// For compatibility, email address can be passed to this function
 /// instead of the domain.
-pub async fn get_provider_info(domain: &str, skip_mx: bool) -> Option<&'static Provider> {
-    let domain = domain.rsplitn(2, '@').next()?;
+pub async fn get_provider_info(
+    context: &Context,
+    domain: &str,
+    skip_mx: bool,
+) -> Option<&'static Provider> {
+    let domain = domain.rsplit('@').next()?;
 
     if let Some(provider) = get_provider_by_domain(domain) {
         return Some(provider);
     }
 
     if !skip_mx {
-        if let Some(provider) = get_provider_by_mx(domain).await {
+        if let Some(provider) = get_provider_by_mx(context, domain).await {
             return Some(provider);
         }
     }
@@ -117,8 +140,8 @@ pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
 /// Finds a provider based on MX record for the given domain.
 ///
 /// For security reasons, only Gmail can be configured this way.
-pub async fn get_provider_by_mx(domain: &str) -> Option<&'static Provider> {
-    if let Ok(resolver) = resolver_from_system_conf().await {
+pub async fn get_provider_by_mx(context: &Context, domain: &str) -> Option<&'static Provider> {
+    if let Ok(resolver) = get_resolver().await {
         let mut fqdn: String = domain.to_string();
         if !fqdn.ends_with('.') {
             fqdn.push('.');
@@ -143,6 +166,8 @@ pub async fn get_provider_by_mx(domain: &str) -> Option<&'static Provider> {
                 }
             }
         }
+    } else {
+        warn!(context, "cannot get a resolver to check MX records.");
     }
 
     None
@@ -169,6 +194,7 @@ mod tests {
 
     use super::*;
     use crate::dc_tools::time;
+    use crate::test_utils::TestContext;
     use chrono::NaiveDate;
 
     #[test]
@@ -218,12 +244,13 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_provider_info() {
-        assert!(get_provider_info("", false).await.is_none());
-        assert!(get_provider_info("google.com", false).await.unwrap().id == "gmail");
+        let t = TestContext::new().await;
+        assert!(get_provider_info(&t, "", false).await.is_none());
+        assert!(get_provider_info(&t, "google.com", false).await.unwrap().id == "gmail");
 
         // get_provider_info() accepts email addresses for backwards compatibility
         assert!(
-            get_provider_info("example@google.com", false)
+            get_provider_info(&t, "example@google.com", false)
                 .await
                 .unwrap()
                 .id
@@ -242,4 +269,10 @@ mod tests {
         assert!(get_provider_update_timestamp() <= time());
         assert!(get_provider_update_timestamp() > timestamp_past);
     }
+
+    #[async_std::test]
+    async fn test_get_resolver() -> Result<()> {
+        assert!(get_resolver().await.is_ok());
+        Ok(())
+    }
 }